From a91f6c5e03df2192a6e64cfc0453ca1c8ad77c6e Mon Sep 17 00:00:00 2001 From: James Maes Date: Tue, 13 Oct 2026 17:43:50 -0500 Subject: [PATCH 01/10] feat(api): /qqq/v1 data-analysis extension, core pushdown backend, analyze permission UI API extension data-analysis 1.0.0 (capabilities, column-profile, distinct-values, table-profile, duplicates, runs) with typed DataAnalysis*V1 schemas and the committed OpenAPI fragment. The analyze permission is registered through core TableFeaturePermissionsMetaData. CoreAnalysisBackend pushes percentiles, stddev, null/blank counts, numeric and date buckets down through the RC.6 aggregate extensions; estimates sample natively and scale counts, and the stand-in uses one primary-key bound instead of an IN list. Builds against QQQ 4.1.0-RC.6. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 13 + README.md | 48 +- docs/TODO.md | 7 + pom.xml | 2 +- .../DataAnalysisQBitProducer.java | 19 + .../dataanalysis/DataAnalysisRuntime.java | 2 +- .../dataanalysis/DataAnalysisService.java | 7 +- .../dataanalysis/api/DataAnalysisApi.java | 507 ++++++++ .../dataanalysis/api/DataAnalysisSchemas.java | 1149 ++++++++++++++++ .../dataanalysis/engine/AnalysisBackend.java | 44 +- .../dataanalysis/engine/ColumnProfiler.java | 80 +- .../engine/CoreAnalysisBackend.java | 475 +++++++ .../dataanalysis/engine/DrillFilters.java | 81 +- .../dataanalysis/engine/DuplicateFinder.java | 2 +- .../dataanalysis/engine/ProfileSupport.java | 20 +- .../engine/StandInAnalysisBackend.java | 173 ++- .../dataanalysis/engine/TableProfiler.java | 24 +- .../permissions/DataAnalysisPermissions.java | 6 +- .../result/DuplicatesRequest.java | 18 + .../qbits/dataanalysis/result/Metric.java | 18 + .../dataanalysis/run/AnalysisRunService.java | 4 +- .../openapi/qqq-v1/data-analysis.yaml | 1150 +++++++++++++++++ .../dataanalysis/CoreAnalysisBackendTest.java | 204 +++ .../dataanalysis/DataAnalysisApiTest.java | 290 +++++ .../DataAnalysisSecurityTest.java | 25 +- .../dataanalysis/EngineContractTest.java | 32 +- .../qbits/dataanalysis/EngineFixture.java | 10 +- .../H2CompatibilityEngineTest.java | 39 + .../qbits/dataanalysis/H2SamplingTest.java | 82 ++ .../dataanalysis/H2StreamedEngineTest.java | 48 + .../dataanalysis/OptionalMiddlewareTest.java | 77 ++ .../dataanalysis/PostgresEngineTest.java | 23 + 32 files changed, 4548 insertions(+), 131 deletions(-) create mode 100644 docs/TODO.md create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisApi.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/CoreAnalysisBackend.java create mode 100644 src/main/resources/openapi/qqq-v1/data-analysis.yaml create mode 100644 src/test/java/com/kingsrook/qbits/dataanalysis/CoreAnalysisBackendTest.java create mode 100644 src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisApiTest.java create mode 100644 src/test/java/com/kingsrook/qbits/dataanalysis/H2CompatibilityEngineTest.java create mode 100644 src/test/java/com/kingsrook/qbits/dataanalysis/H2SamplingTest.java create mode 100644 src/test/java/com/kingsrook/qbits/dataanalysis/H2StreamedEngineTest.java create mode 100644 src/test/java/com/kingsrook/qbits/dataanalysis/OptionalMiddlewareTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a69132..3424b6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,19 @@ Following [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and [Semantic ### Added +- `data-analysis` UI API extension 1.0.0: seven session-bound profile, capability, distinct-value, duplicate and run endpoints, typed schemas and a checked OpenAPI fragment. +- Core table-feature permission registration and capability-directed aggregate pushdown, with native sampling and portable streamed fallbacks. +- HTTP authorization, drill-to-core-count round trips, async polling, served OpenAPI and assistant discovery tests; H2 streamed and native-compatibility contract variants. + +### Changed + +- Scalable estimates sample each native aggregate without primary-key IN lists, scale counts using observed sample sizes, retain exact population totals and unscaled ratios, and drill over the full population. Portable estimates use one primary-key boundary and report unscaled prefix counts. + +- Target QQQ 4.1.0-RC.6; local verification uses `-Dqqq.snapshot.version=4.1.0-DATAANALYSIS-SNAPSHOT`. +- Empty-string and temporal drills round-trip through core HTTP filter interpretation. + +### Previously added + - Phase b host engine: typed column/table profiles, distinct pages, duplicates, count-equivalent drills, chart data, candidate keys and completeness heatmaps. - Internal `AnalysisBackend` port and RC.5 stand-in with aggregate pushdown, bounded query pages, deterministic prefix estimates and honest truncation metadata. - Personalized field/table authorization, enforced FK label lookup, runtime configuration overrides and minimum-group-size privacy before caching. diff --git a/README.md b/README.md index ba9f9ba..43e4329 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,13 @@ Statistical analysis and data dives on any QQQ table, filter or saved view: prof The host API provides column and table profiles, distinct-value pages, duplicate groups, drill-through filters, effective per-table configuration, privacy suppression, and asynchronous exact runs with a persistent per-user cache. Metadata, six visible tables, owner locks, PostgreSQL DDL generation and the Analyses home are also available. -HTTP endpoints require the QQQ RC.6 extension point and are next, along with the Analyze UI. Pivots, comparisons, anomalies, saved-analysis execution, scheduling and exports remain later phases. +Seven HTTP operations are available under `/qqq/v1/qbit/data-analysis`; the Analyze UI is a later phase. Pivots, comparisons, anomalies, saved-analysis execution, scheduling and exports remain later phases. ## Requirements -- Java 21; QQQ **4.1.0-RC.5+**. +- Java 21; QQQ **4.1.0-RC.6+**. - A configured QQQ backend for the qbit tables and authenticated sessions with permissions and security keys. -- The engine works on RC.5 through an internal stand-in. A future `CoreAnalysisBackend` can replace it when the core aggregate extensions ship. HTTP endpoints require the UI API extension point ([qqq#1128](https://github.com/QRun-IO/qqq/pull/1128), RC.6). +- Core aggregate and table-feature permission APIs ([qqq#1139](https://github.com/QRun-IO/qqq/pull/1139)) and the UI API extension point ([qqq#1128](https://github.com/QRun-IO/qqq/pull/1128)). Middleware Javalin remains optional; the producer registers endpoints only when it is present. ## Install @@ -58,7 +58,7 @@ Only Saved Analyses, Watches, Analysis Settings and Analysis Runs appear in navi | `data_analysis.admin` | The all-access key of the `data_analysis_user` record locks (see Security) | | `.analyze` | Analyze a source table, also requiring its READ permission | -`DataAnalysisPermissions.analyzePermissionName(table)` honors a custom permission base name and instance defaults. `canAnalyze(table)` is the legacy helper; its READ check follows the instance setting. The engine adds an explicit READ check and always uses `PermissionEnforcement.ENFORCE`, even when the instance default is off. The feature check is a stand-in for core `TableFeaturePermissionsMetaData` (§3.6); hosts must grant these analyze names explicitly until that hook registers them in the catalog. All qbit tables use standard permission rules (no custom checker), so their CRUD permissions appear in the core permission catalog and the qbit-authentik policy editor. Hosts explicitly grant source-table analyze permissions until the core feature hook ships. +`DataAnalysisPermissions.analyzePermissionName(table)` honors the effective table permission base. `canAnalyze(table)` delegates to core's table-feature permission check. The producer registers `analyze`, labelled **Analyze**, on configured source tables, excluding this qbit's own tables. No CRUD permission implies analyze. Core checks READ as well as the feature grant on protected tables. All analysis queries enforce permissions even when the instance default is off. Qbit CRUD permissions also appear in the core catalog. ## Security @@ -102,7 +102,7 @@ Cached results are scoped to the authenticated user, normalized request, effecti All properties have fluent `withX` setters. Global numeric defaults must be non-null and non-negative. Nullable per-table numeric overrides inherit global defaults; configured values must be non-negative. `TableAnalysisDefaults` also supports enabled, analyzable fields and excluded fields. Runtime `data_analysis_table_config` values override per-table code defaults, which override global defaults. Null row values inherit; blank field lists mean all readable fields or no exclusions. Include/exclude table lists remain an outer restriction, even if a runtime row enables the table. Only the config-row lookup uses a temporary system session, restored in `finally`. It selects configuration fields only and is memoized for 60 seconds. Settings editors can call `service.getConfigResolver().invalidate()` for immediate refresh. -`max_rows` bounds the stand-in's selected population and retained records. An exact request disables sampling but still respects this bound and reports `truncated: true` when necessary; increase the limit to obtain a complete larger profile. Zero selects no rows. Above `sample_threshold_rows`, estimates use the first `min(sample_rows, max_rows)` rows ordered by primary key. Counts describe that prefix, are not extrapolated, and carry `estimate`, `sampleRows`, `rowsScanned`, warnings and an `asOf` timestamp. This is deterministic prefix sampling, not a random sample. `cache_ttl_minutes: 0` disables cache reuse. +`max_rows` bounds the stand-in's selected population and retained records. An exact request disables sampling but still respects this bound and reports `truncated: true` when necessary; increase the limit to obtain a complete larger profile. Zero selects no rows in the stand-in. Above `sample_threshold_rows`, capable backends run each aggregate with `samplePercent = min(100, sample_rows * 100 / total)` on the user's filter. Counts, top values and histograms are scaled by the full population divided by the observed sample count for that call; ratios and descriptive statistics remain in sample units. The `rows` metric always reports the unsampled total for estimates. Other metrics carry `estimate: true` and an `≈ ` formatted prefix. Sampling never materializes a primary-key IN filter. Backends without sampling use one `pk <= boundary` criterion to compute an unscaled first-N prefix estimate. Estimate drills combine the original filter with the metric condition over the full population, so count/drill equivalence applies only to exact results. Warnings distinguish random samples from prefixes; `sampleRows`, `rowsScanned` and `asOf` describe coverage. `cache_ttl_minutes: 0` disables cache reuse. ## Host API @@ -136,9 +136,37 @@ The `result` package contains Serializable, Jackson-friendly request/result POJO Count drills combine an untouched copy of the request filter with their producing condition. Cardinality drills select one representative record per distinct non-null value; the privacy guard removes those representative drills. Truncated profiles retain their primary-key boundary in drills, so their counts remain reproducible. Serialized drill getters restore temporal operand types from the current instance metadata before core evaluates them. -## RC.5 stand-in +## HTTP endpoints -`engine.AnalysisBackend` isolates profilers from missing core operators. `StandInAnalysisBackend` pushes down COUNT, COUNT_DISTINCT, MIN, MAX, AVG, SUM, grouped counts, aggregate ordering and limits through core actions. RC.5 has no aggregate OFFSET, so pages over-fetch through the requested end and slice locally. Memory grouping pagination and scale-sensitive decimal equality have isolated compatibility handling. +Extension `data-analysis` version **1.0.0**, prefix `/qbit/data-analysis` inside `/qqq/v1`: + +| Method | Path after `/qqq/v1` | Body or purpose | +| --- | --- | --- | +| GET | `/qbit/data-analysis/tables/{table}/capabilities` | Availability, fields and charts, limits, backend capabilities, SQL Workbench presence | +| POST | `/qbit/data-analysis/tables/{table}/column-profile` | `{fieldName, filter, exact}` | +| POST | `/qbit/data-analysis/tables/{table}/distinct-values` | `{fieldName, filter, page, pageSize, search}` | +| POST | `/qbit/data-analysis/tables/{table}/table-profile` | `{filter, fields}` | +| POST | `/qbit/data-analysis/tables/{table}/duplicates` | `{fields, filter, limit}`; default limit 50, maximum 500 | +| POST | `/qbit/data-analysis/runs` | `{operation: COLUMN_PROFILE or TABLE_PROFILE, tableName, request}` returns `{runId, status}` | +| GET | `/qbit/data-analysis/runs/{runId}` | Status, timestamps, error, typed `columnProfile` or `tableProfile` | + +Every operation uses the request session through `DataAnalysisService`. Capabilities returns 200 with `canAnalyze: false` and no fields when denied. Other permission denials return 403, bad bodies or unavailable fields return 400, and unknown tables return 404. Core middleware supplies plain messages in its standard `{error: message}` JSON envelope. Run rows remain owner-locked. + +Requests and drills reuse core's `QueryFilter` JSON shape and schema reference. Versioned `DataAnalysis...V1` schemas cover profiles, metrics, drills, charts, series, points, field summaries, distinct and duplicate results, capabilities and runs. The [OpenAPI fragment](src/main/resources/openapi/qqq-v1/data-analysis.yaml) is checked against regeneration. The deployed `/qqq/v1/openapi.json` and `UiApiInvoker.listOperations()` include all seven operations. + +Regenerate deliberately after an API change: + +```bash +./mvnw -o -q test -Dtest=DataAnalysisApiTest#contract -Dopenapi.write=true -Dqqq.snapshot.version=4.1.0-DATAANALYSIS-SNAPSHOT +``` + +## Backend selection + +`DataAnalysisService` selects `CoreAnalysisBackend` when the source backend advertises aggregate capabilities, including memory. Supported standard deviation, percentiles, null/blank counts, numeric/date buckets and length statistics execute through core aggregates as the current session. Unsupported primitives delegate to `StandInAnalysisBackend`. Patterns and derived length histograms remain streamed; representative distinct drills and cyclic-date membership use bounded scans. + +The installed snapshot requires H2 PostgreSQL compatibility mode for native bucket SQL (`GROUP BY` positions). + +`engine.AnalysisBackend` isolates profilers from backend-specific operators. `StandInAnalysisBackend` pushes down COUNT, COUNT_DISTINCT, MIN, MAX, AVG, SUM, grouped counts, aggregate ordering and limits through core actions. RC.5 has no aggregate OFFSET, so pages over-fetch through the requested end and slice locally. Memory grouping pagination and scale-sensitive decimal equality have isolated compatibility handling. Variance, interpolated percentiles, string lengths/shapes, date classification, whitespace detection and representative-row drills use bounded primary-key query pages. Histogram/date membership counts use enforced range/count queries after their boundaries have been calculated. Numeric histograms have 20 interior bins between p1 and p99 plus below/above bins, and the last interior bin is closed. Profiles include Tukey outliers, date gaps, date-time weekday/hour patterns, chart alternatives, a seven-bucket moving average, table completeness and candidate keys (single fields and pairs among the eight most unique). @@ -152,15 +180,15 @@ Data versions use filtered COUNT plus MAX of a readable modification-date-like f 4. Review per-table include/exclude sets, limits and minimum group sizes before enabling analysis. Integrate Saved Analyses and Watches with host ESB/Quick Search when those optional host features are installed; this scaffold does not add those dependencies. 5. Use the Analyses home as the owner of analysis data. Other dashboards should link there until saved-analysis widgets are implemented. -Phase **b** profiles, drill-through and runs/cache are implemented through the host facade. Remaining work: **a** core aggregate extensions; **b** HTTP endpoints after QQQ RC.6 and analysis UI; **c** pivots; **d** comparisons and relationships; **e** watch evaluation and alerts; **f** saved-analysis execution, sharing, stars integration, schedules, chart widgets, full home and exports. The current home summarizes stored rows; the host facade executes profiles. The home does not send deliveries. +Phase **b** profiles, drill-through, runs/cache and HTTP endpoints are implemented. Remaining work: **b** analysis UI; **c** pivots; **d** comparisons and relationships; **e** watch evaluation and alerts; **f** saved-analysis execution, sharing, stars integration, schedules, chart widgets, full home and exports. The current home summarizes stored rows; the host facade executes profiles. The home does not send deliveries. ## Build ```bash -./mvnw verify +./mvnw -o -q verify -Dqqq.snapshot.version=4.1.0-DATAANALYSIS-SNAPSHOT ``` -The wrapper pins Maven 3.9.11. On Windows use `mvnw.cmd`. Verification runs Checkstyle, a shared JUnit 5/AssertJ contract suite on memory and H2, PostgreSQL via Testcontainers when Docker is available, and JaCoCo gates of 70% instructions and 90% classes. The suite checks numerical references, every emitted count/drill pair, security, guarded cache JSON, prefix sampling, asynchronous jobs and cache invalidation. For a populated local dependency cache use `./mvnw -o -q verify`. RDBMS, H2, Testcontainers PostgreSQL and the PostgreSQL driver are test-scoped; middleware Javalin is optional. +The wrapper pins Maven 3.9.11. On Windows use `mvnw.cmd`. Verification runs Checkstyle, a shared JUnit 5/AssertJ contract suite on memory and H2, PostgreSQL via Testcontainers when Docker is available, and JaCoCo gates of 70% instructions and 90% classes. The suite checks numerical references, every exact count/drill pair, full-population estimate drills, security, guarded cache JSON, native sampling, asynchronous jobs and cache invalidation. The POM targets RC.6; until its release, always supply the locally installed snapshot override shown above. RDBMS, H2, Testcontainers PostgreSQL and the PostgreSQL driver are test-scoped; middleware Javalin is optional. ## License diff --git a/docs/TODO.md b/docs/TODO.md new file mode 100644 index 0000000..e718b61 --- /dev/null +++ b/docs/TODO.md @@ -0,0 +1,7 @@ +# Phase b part 2 remaining work + +- Run the 21 PostgreSQL contract tests with accessible Docker, including the 50,000-row estimate regression at sample sizes 1,000 and 40,000. +- Review the existing endpoint work's core READ and H2 ordinal-grouping compatibility choices; this sampling task preserves them. +- Append the durable project/daily handoff in [SESSION-STATE.md](SESSION-STATE.md) to Second Brain from a workspace with vault write access. + +Completed: replace sampled-key IN scopes, scale native estimate counts, preserve raw ratios and full-population drills, update the OpenAPI contract and tests, and pass the requested offline Maven verification. diff --git a/pom.xml b/pom.xml index 4028e33..da25483 100644 --- a/pom.xml +++ b/pom.xml @@ -215,7 +215,7 @@ true - 4.1.0-RC.5 + 4.1.0-RC.6 diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitProducer.java b/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitProducer.java index bbc5caf..24e95aa 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitProducer.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitProducer.java @@ -21,9 +21,12 @@ package com.kingsrook.qbits.dataanalysis; +import com.kingsrook.qbits.dataanalysis.api.DataAnalysisApi; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.TableFeaturePermission; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.TableFeaturePermissionsMetaData; import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData; import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaDataProducer; @@ -129,6 +132,22 @@ public class DataAnalysisQBitProducer implements QBitMetaDataProducer !table.getName().startsWith(DataAnalysisQBitConfig.TABLE_NAME_PREFIX) + && (included.isEmpty() || included.contains(table.getName())) && !excluded.contains(table.getName()))); + try + { + Class.forName("io.javalin.Javalin"); + Class.forName("com.kingsrook.qqq.middleware.javalin.QJavalinImplementation"); + DataAnalysisApi.register(instance); + } + catch(ClassNotFoundException expected) + { + // The engine and metadata also work without the optional middleware. + } if(qBitConfig.getParentAppName() != null && instance.getApp(qBitConfig.getParentAppName()) != null) { instance.getApp(qBitConfig.getParentAppName()).withChild(instance.getApp(qBitConfig.getAppName())); diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisRuntime.java b/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisRuntime.java index 991d427..cf2542f 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisRuntime.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisRuntime.java @@ -25,7 +25,7 @@ import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.model.metadata.QSupplementalInstanceMetaData; /******************************************************************************* - ** Instance-scoped service holder for future endpoints and host integrations. + ** Instance-scoped service holder for UI API endpoints and host integrations. *******************************************************************************/ public class DataAnalysisRuntime implements QSupplementalInstanceMetaData { diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisService.java b/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisService.java index 252e0f1..c6b106d 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisService.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisService.java @@ -25,6 +25,7 @@ import java.util.concurrent.Executor; import com.kingsrook.qbits.dataanalysis.engine.AnalysisBackend; import com.kingsrook.qbits.dataanalysis.engine.AnalysisSecurity; import com.kingsrook.qbits.dataanalysis.engine.ColumnProfiler; +import com.kingsrook.qbits.dataanalysis.engine.CoreAnalysisBackend; import com.kingsrook.qbits.dataanalysis.engine.DistinctValues; import com.kingsrook.qbits.dataanalysis.engine.DuplicateFinder; import com.kingsrook.qbits.dataanalysis.engine.EffectiveConfigResolver; @@ -82,7 +83,7 @@ public class DataAnalysisService public Capabilities capabilities(String table) throws QException { return (new Capabilities().withTableName(table).withFields(security.analyzableFields(table).stream().map(QFieldMetaData::getName).toList()) - .withConfig(configs.resolve(table)).withBackend("RC.5 stand-in; deterministic primary-key prefix sampling")); + .withConfig(configs.resolve(table)).withBackend(CoreAnalysisBackend.capabilities(table).isEmpty() ? "Streamed analysis" : "Core aggregate pushdown")); } @@ -163,11 +164,11 @@ public class DataAnalysisService /******************************************************************************* - ** Fresh per-request backend, ready for a future core-backed implementation. + ** Select core pushdown when the backend advertises analysis capabilities. *******************************************************************************/ public AnalysisBackend backend(String table) throws QException { - return (new StandInAnalysisBackend(configs.resolve(table))); + return (CoreAnalysisBackend.capabilities(table).isEmpty() ? new StandInAnalysisBackend(configs.resolve(table)) : new CoreAnalysisBackend(configs.resolve(table))); } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisApi.java b/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisApi.java new file mode 100644 index 0000000..2aeb0df --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisApi.java @@ -0,0 +1,507 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.api; + +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.dataanalysis.DataAnalysisRuntime; +import com.kingsrook.qbits.dataanalysis.DataAnalysisService; +import com.kingsrook.qbits.dataanalysis.engine.CoreAnalysisBackend; +import com.kingsrook.qbits.dataanalysis.result.AnalysisRequest; +import com.kingsrook.qbits.dataanalysis.result.ChartKind; +import com.kingsrook.qbits.dataanalysis.result.ColumnProfileRequest; +import com.kingsrook.qbits.dataanalysis.result.DistinctValuesRequest; +import com.kingsrook.qbits.dataanalysis.result.DuplicatesRequest; +import com.kingsrook.qbits.dataanalysis.result.TableProfileRequest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; +import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException; +import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.middleware.javalin.specs.BasicOperation; +import com.kingsrook.qqq.middleware.javalin.specs.extensions.AbstractJsonEndpointSpec; +import com.kingsrook.qqq.middleware.javalin.specs.extensions.MiddlewareSpecExtension; +import com.kingsrook.qqq.middleware.javalin.specs.extensions.MiddlewareSpecExtensionsMetaData; +import com.kingsrook.qqq.openapi.model.HttpMethod; +import com.kingsrook.qqq.openapi.model.In; +import com.kingsrook.qqq.openapi.model.Parameter; +import com.kingsrook.qqq.openapi.model.Schema; +import com.kingsrook.qqq.openapi.model.Type; + +/******************************************************************************* + ** Session-bound profile operations registered only with optional Javalin present. + *******************************************************************************/ +public class DataAnalysisApi +{ + public static final String PREFIX = "/qbit/data-analysis"; + + /******************************************************************************* + ** Publish the extension through the core spec registry. + *******************************************************************************/ + public static void register(QInstance instance) + { + MiddlewareSpecExtensionsMetaData.ofOrWithNew(instance).withExtension(extension()); + } + + /******************************************************************************* + ** The same descriptor drives routes, OpenAPI and assistant tool discovery. + *******************************************************************************/ + public static MiddlewareSpecExtension extension() + { + return (new MiddlewareSpecExtension().withName("data-analysis").withSemanticVersion("1.0.0") + .withDescription("Profiles, distinct values, duplicate groups and exact analysis runs as the signed-in user.") + .withSpec(CapabilitiesSpec.class).withSpec(ColumnProfileSpec.class).withSpec(DistinctValuesSpec.class) + .withSpec(TableProfileSpec.class).withSpec(DuplicatesSpec.class).withSpec(StartRunSpec.class).withSpec(GetRunSpec.class)); + } + + /******************************************************************************* + ** Common schema catalog and path parameters for the seven JSON specs. + *******************************************************************************/ + public abstract static class ProfileSpec extends AbstractJsonEndpointSpec + { + private final String path; + private final HttpMethod method; + private final String summary; + private final String description; + private final String request; + private final String response; + + /******************************************************************************* + ** Bind immutable operation metadata. + *******************************************************************************/ + protected ProfileSpec(String path, HttpMethod method, String summary, String description, String request, String response) + { + this.path = path; + this.method = method; + this.summary = summary; + this.description = description; + this.request = request; + this.response = response; + } + + /******************************************************************************* + ** Assistant-facing operation purpose. + *******************************************************************************/ + @Override + public BasicOperation defineBasicOperation() + { + return (new BasicOperation().withPath(PREFIX + path).withHttpMethod(method).withShortSummary(summary).withLongDescription(description)); + } + + /******************************************************************************* + ** Required path parameter, when present. + *******************************************************************************/ + @Override + public List defineRequestParameters() + { + String name = path.contains("{table}") ? "table" : path.contains("{runId}") ? "runId" : null; + return (name == null ? List.of() : List.of(new Parameter().withName(name).withIn(In.PATH).withRequired(true) + .withDescription(name.equals("table") ? "QQQ table name to analyze." : "Owner-scoped analysis run id.").withSchema(new Schema().withType(Type.STRING)))); + } + + /******************************************************************************* + ** Versioned request schema. + *******************************************************************************/ + @Override + protected Schema defineRequestBodySchema() + { + return (request == null ? null : new Schema().withRef("#/components/schemas/DataAnalysis" + request + "V1")); + } + + /******************************************************************************* + ** Versioned response schema. + *******************************************************************************/ + @Override + protected String getResponseSchemaName() + { + return ("DataAnalysis" + response + "V1"); + } + + /******************************************************************************* + ** Include all referenced transport models and the shared core filter. + *******************************************************************************/ + @Override + public Map defineComponentSchemas() + { + return (this instanceof CapabilitiesSpec ? DataAnalysisSchemas.components() : Map.of()); + } + } + + /******************************************************************************* + ** Existing table lookup before invoking permission-sensitive operations. + *******************************************************************************/ + private static QTableMetaData table(String name) throws QNotFoundException + { + QTableMetaData table = QContext.getQInstance().getTable(name); + if(table == null) + { + throw new QNotFoundException("Unknown analysis table."); + } + return (table); + } + + /******************************************************************************* + ** Parse one JSON object with strict scalar/list types and core filter JSON. + *******************************************************************************/ + private static AnalysisRequest request(String operation, String table, Map body) throws Exception + { + table(table); + DataAnalysisService service = DataAnalysisRuntime.service(); + List readable = service.capabilities(table).getFields(); + try + { + for(String name : List.of("fieldName", "search")) + { + if(body.get(name) != null && !(body.get(name) instanceof String)) + { + throw new IllegalArgumentException(name + " must be a string."); + } + } + if(body.get("exact") != null && !(body.get("exact") instanceof Boolean)) + { + throw new IllegalArgumentException("exact must be a boolean."); + } + QQueryFilter filter = body.get("filter") == null ? null : JsonUtils.toObject(JsonUtils.toJson(body.get("filter")), QQueryFilter.class); + validateFilter(filter, readable); + List fields = null; + if(body.get("fields") != null) + { + if(!(body.get("fields") instanceof List values) || values.stream().anyMatch(v -> !(v instanceof String))) + { + throw new IllegalArgumentException("fields must be an array of field names."); + } + fields = ((List) body.get("fields")).stream().map(String.class::cast).toList(); + if(fields.isEmpty() || !readable.containsAll(fields)) + { + throw new IllegalArgumentException("Choose readable analysis fields."); + } + } + String field = (String) body.get("fieldName"); + if((operation.equals("COLUMN_PROFILE") || operation.equals("DISTINCT_VALUES")) && !readable.contains(field)) + { + throw new IllegalArgumentException("Choose a readable analysis field."); + } + boolean exact = Boolean.TRUE.equals(body.get("exact")); + return (switch(operation) + { + case "COLUMN_PROFILE" -> new ColumnProfileRequest().withTableName(table).withFilter(filter).withFieldName(field).withExact(exact); + case "TABLE_PROFILE" -> new TableProfileRequest().withTableName(table).withFilter(filter).withFields(fields); + case "DISTINCT_VALUES" -> new DistinctValuesRequest().withTableName(table).withFilter(filter).withFieldName(field) + .withPage(integer(body, "page", 0, 0, 1_000_000)).withPageSize(integer(body, "pageSize", 100, 1, 500)).withSearch((String) body.get("search")); + case "DUPLICATES" -> new DuplicatesRequest().withTableName(table).withFilter(filter) + .withFields(java.util.Objects.requireNonNull(fields, "Choose duplicate fields.")).withLimit(integer(body, "limit", 50, 1, 500)); + default -> throw new IllegalArgumentException("operation must be COLUMN_PROFILE or TABLE_PROFILE."); + }); + } + catch(Exception e) + { + throw new QBadRequestException("Invalid analysis request: " + e.getMessage()); + } + } + + /******************************************************************************* + ** Bad filter fields fail before any aggregate or cached result is consulted. + *******************************************************************************/ + private static void validateFilter(QQueryFilter filter, List readable) + { + if(filter == null) + { + return; + } + for(var criterion : filter.getCriteria()) + { + if(criterion.getOperator() == null || !readable.contains(criterion.getFieldName()) + || criterion.getOtherFieldName() != null && !readable.contains(criterion.getOtherFieldName())) + { + throw new IllegalArgumentException("Choose readable filter fields and a valid operator."); + } + } + filter.getSubFilters().forEach(sub -> validateFilter(sub, readable)); + } + + /******************************************************************************* + ** Reject fractional, null, overflowing and out-of-range numeric parameters. + *******************************************************************************/ + private static int integer(Map body, String name, int fallback, int min, int max) + { + if(!body.containsKey(name)) + { + return (fallback); + } + Object value = body.get(name); + if(!(value instanceof Number number) || number.doubleValue() != number.intValue() || number.intValue() < min || number.intValue() > max) + { + throw new IllegalArgumentException(name + " must be an integer from " + min + " to " + max + "."); + } + return (number.intValue()); + } + + /******************************************************************************* + ** Transport conversion avoids middleware dependencies in engine results. + *******************************************************************************/ + private static T transport(Object value, Class type) throws Exception + { + return (JsonUtils.toObject(JsonUtils.toJson(value), type)); + } + + /******************************************************************************* + ** Check analysis availability. + *******************************************************************************/ + public static class CapabilitiesSpec extends ProfileSpec + { + /******************************************************************************* + ** Public constructor used by the core extension registry. + *******************************************************************************/ + public CapabilitiesSpec() + { + super("/tables/{table}/capabilities", HttpMethod.GET, "Check analysis availability", + "Discover whether the current user can analyze this table, readable fields, chart defaults, limits and backend capabilities.", null, "Capabilities"); + } + + /******************************************************************************* + ** Execute in the request session established by core middleware. + *******************************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + return (capabilities(input.getPathParam("table"))); + } + } + + /******************************************************************************* + ** Profile one column. + *******************************************************************************/ + public static class ColumnProfileSpec extends ProfileSpec + { + /******************************************************************************* + ** Public constructor used by the core extension registry. + *******************************************************************************/ + public ColumnProfileSpec() + { + super("/tables/{table}/column-profile", HttpMethod.POST, "Profile one column", + "Profile one column of a table under a filter: counts, nulls, distinct values, top values, numeric statistics and histogram, date patterns; every number has a drill-through filter unless privacy suppresses it.", "ColumnProfileRequest", "ColumnProfile"); + } + + /******************************************************************************* + ** Execute in the request session established by core middleware. + *******************************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + return (transport(DataAnalysisRuntime.service().columnProfile((ColumnProfileRequest) request("COLUMN_PROFILE", input.getPathParam("table"), input.getBody())), DataAnalysisSchemas.DataAnalysisColumnProfileV1.class)); + } + } + + /******************************************************************************* + ** List distinct column values. + *******************************************************************************/ + public static class DistinctValuesSpec extends ProfileSpec + { + /******************************************************************************* + ** Public constructor used by the core extension registry. + *******************************************************************************/ + public DistinctValuesSpec() + { + super("/tables/{table}/distinct-values", HttpMethod.POST, "List distinct column values", + "Page through distinct stored values with display labels, counts and drill-through filters; search narrows stored values.", "DistinctValuesRequest", "DistinctValuesPage"); + } + + /******************************************************************************* + ** Execute in the request session established by core middleware. + *******************************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + return (transport(DataAnalysisRuntime.service().distinctValues((DistinctValuesRequest) request("DISTINCT_VALUES", input.getPathParam("table"), input.getBody())), DataAnalysisSchemas.DataAnalysisDistinctValuesPageV1.class)); + } + } + + /******************************************************************************* + ** Profile table completeness and keys. + *******************************************************************************/ + public static class TableProfileSpec extends ProfileSpec + { + /******************************************************************************* + ** Public constructor used by the core extension registry. + *******************************************************************************/ + public TableProfileSpec() + { + super("/tables/{table}/table-profile", HttpMethod.POST, "Profile table completeness and keys", + "Profile readable fields under a filter for completeness, uniqueness, constant or empty columns and candidate keys.", "TableProfileRequest", "TableProfile"); + } + + /******************************************************************************* + ** Execute in the request session established by core middleware. + *******************************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + return (transport(DataAnalysisRuntime.service().tableProfile((TableProfileRequest) request("TABLE_PROFILE", input.getPathParam("table"), input.getBody())), DataAnalysisSchemas.DataAnalysisTableProfileV1.class)); + } + } + + /******************************************************************************* + ** Find duplicate groups. + *******************************************************************************/ + public static class DuplicatesSpec extends ProfileSpec + { + /******************************************************************************* + ** Public constructor used by the core extension registry. + *******************************************************************************/ + public DuplicatesSpec() + { + super("/tables/{table}/duplicates", HttpMethod.POST, "Find duplicate groups", + "Group matching records by the selected readable fields and return repeated tuples with counts and drill-through filters.", "DuplicatesRequest", "DuplicatesResult"); + } + + /******************************************************************************* + ** Execute in the request session established by core middleware. + *******************************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + return (transport(DataAnalysisRuntime.service().duplicates((DuplicatesRequest) request("DUPLICATES", input.getPathParam("table"), input.getBody())), DataAnalysisSchemas.DataAnalysisDuplicatesResultV1.class)); + } + } + + /******************************************************************************* + ** Start an exact profile run. + *******************************************************************************/ + public static class StartRunSpec extends ProfileSpec + { + /******************************************************************************* + ** Public constructor used by the core extension registry. + *******************************************************************************/ + public StartRunSpec() + { + super("/runs", HttpMethod.POST, "Start an exact profile run", + "Start an owner-scoped exact column or table profile. Poll the returned runId until COMPLETE or FAILED; results honor the current session permissions.", "StartRunRequest", "RunStarted"); + } + + /******************************************************************************* + ** Execute in the request session established by core middleware. + *******************************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + Map body = input.getBody(); + if(body == null || !(body.get("tableName") instanceof String name) || !(body.get("operation") instanceof String operation) + || !List.of("COLUMN_PROFILE", "TABLE_PROFILE").contains(operation) || !(body.get("request") instanceof Map)) + { + throw new QBadRequestException("Supply operation, tableName and a profile request object."); + } + Map parameters = JsonUtils.toObject(JsonUtils.toJson(body.get("request")), java.util.LinkedHashMap.class); + String id = DataAnalysisRuntime.service().startExactRun(request(operation, name, parameters)); + DataAnalysisSchemas.DataAnalysisRunStartedV1 result = new DataAnalysisSchemas.DataAnalysisRunStartedV1(); + result.runId = id; + result.status = DataAnalysisRuntime.service().getRun(id).getStatus(); + return (result); + } + } + + /******************************************************************************* + ** Get an exact profile run. + *******************************************************************************/ + public static class GetRunSpec extends ProfileSpec + { + /******************************************************************************* + ** Public constructor used by the core extension registry. + *******************************************************************************/ + public GetRunSpec() + { + super("/runs/{runId}", HttpMethod.GET, "Get an exact profile run", + "Read status, timestamps and the typed columnProfile or tableProfile result of an owner-scoped run. Completed results are rechecked against current permissions.", null, "RunStatus"); + } + + /******************************************************************************* + ** Execute in the request session established by core middleware. + *******************************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + return (transport(DataAnalysisRuntime.service().getRun(input.getPathParam("runId")), DataAnalysisSchemas.DataAnalysisRunStatusV1.class)); + } + } + + /******************************************************************************* + ** Permission probing is a successful response with no disclosed field list. + *******************************************************************************/ + private static DataAnalysisSchemas.DataAnalysisCapabilitiesV1 capabilities(String name) throws Exception + { + QTableMetaData table = table(name); + DataAnalysisSchemas.DataAnalysisCapabilitiesV1 result = new DataAnalysisSchemas.DataAnalysisCapabilitiesV1(); + result.tableName = name; + result.tableLabel = table.getLabel(); + result.canAnalyze = false; + result.fields = List.of(); + result.backendCapabilities = List.of(); + result.sqlWorkbenchAvailable = QContext.getQInstance().getApp("sqlWorkbench") != null + || QContext.getQInstance().getTables().keySet().stream().anyMatch(t -> t.startsWith("sql_workbench_")); + try + { + var capabilities = DataAnalysisRuntime.service().capabilities(name); + result.canAnalyze = true; + result.backendCapabilities = CoreAnalysisBackend.capabilities(name).stream().map(Enum::name).sorted().toList(); + result.fields = capabilities.getFields().stream().map(f -> capabilityField(table.getField(f))).toList(); + var config = capabilities.getConfig(); + result.limits = new DataAnalysisSchemas.DataAnalysisLimitsV1(); + result.limits.maxRows = config.getMaxRows(); + result.limits.sampleThresholdRows = config.getSampleThresholdRows(); + result.limits.sampleRows = config.getSampleRows(); + result.limits.minGroupSize = config.getMinGroupSize(); + } + catch(QPermissionDeniedException expected) + { + // A UI capability probe deliberately reports denial without an error. + } + return (result); + } + + /******************************************************************************* + ** Semantic type and the default visualization, with PVS/FK taking priority. + *******************************************************************************/ + private static DataAnalysisSchemas.DataAnalysisCapabilityFieldV1 capabilityField(QFieldMetaData field) + { + DataAnalysisSchemas.DataAnalysisCapabilityFieldV1 result = new DataAnalysisSchemas.DataAnalysisCapabilityFieldV1(); + result.name = field.getName(); + result.label = field.getLabel(); + result.type = field.getType().name(); + result.kind = field.getPossibleValueSourceName() != null ? "CATEGORY" : switch(field.getType()) + { + case INTEGER, LONG, DECIMAL -> "NUMERIC"; + case DATE -> "DATE"; + case DATE_TIME -> "DATE_TIME"; + case BOOLEAN -> "BOOLEAN"; + default -> "TEXT"; + }; + result.defaultChartKind = switch(result.kind) + { + case "NUMERIC" -> ChartKind.HISTOGRAM; + case "DATE", "DATE_TIME" -> ChartKind.TIME_SERIES; + default -> ChartKind.BAR; + }; + return (result); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java b/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java new file mode 100644 index 0000000..794dbd5 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java @@ -0,0 +1,1149 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.api; + +import java.io.Serializable; +import java.math.BigDecimal; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.dataanalysis.result.ChartKind; +import com.kingsrook.qbits.dataanalysis.result.ValueType; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.middleware.javalin.schemabuilder.ToSchema; +import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIDescription; +import com.kingsrook.qqq.middleware.javalin.specs.v1.responses.components.QueryFilter; +import com.kingsrook.qqq.openapi.model.Schema; +import com.kingsrook.qqq.openapi.model.Type; + +/******************************************************************************* + ** Versioned transport models isolated from the optional middleware-free engine. + *******************************************************************************/ +public class DataAnalysisSchemas +{ + /******************************************************************************* + ** Core's Schema model lacks the standard OpenAPI required property. + *******************************************************************************/ + public static class RequiredSchema extends Schema + { + private final List required; + + /******************************************************************************* + ** Declare the fields the endpoint requires at runtime. + *******************************************************************************/ + public RequiredSchema(List required) + { + this.required = required; + } + + /******************************************************************************* + ** Standard JSON Schema required property serialized by core's generator. + *******************************************************************************/ + public List getRequired() + { + return (required); + } + } + + /******************************************************************************* + ** Shared component reference, including the core QueryFilter contract. + *******************************************************************************/ + private static Schema ref(String name) + { + return (new Schema().withRef("#/components/schemas/" + name)); + } + + /******************************************************************************* + ** Complete standalone extension component catalog. + *******************************************************************************/ + public static Map components() + { + Map result = new LinkedHashMap<>(); + result.put("QueryFilter", new QueryFilter().toSchema()); + result.put("FilterCriteria", new com.kingsrook.qqq.middleware.javalin.specs.v1.responses.components.FilterCriteria().toSchema()); + result.put("OrderBy", new com.kingsrook.qqq.middleware.javalin.specs.v1.responses.components.OrderBy().toSchema()); + result.put("DataAnalysisAxisV1", new DataAnalysisAxisV1().toSchema()); + result.put("DataAnalysisCandidateKeyV1", new DataAnalysisCandidateKeyV1().toSchema()); + result.put("DataAnalysisChartV1", new DataAnalysisChartV1().toSchema()); + result.put("DataAnalysisColumnProfileV1", new DataAnalysisColumnProfileV1().toSchema()); + result.put("DataAnalysisColumnProfileRequestV1", new DataAnalysisColumnProfileRequestV1().toSchema()); + result.put("DataAnalysisDateProfileV1", new DataAnalysisDateProfileV1().toSchema()); + result.put("DataAnalysisDistinctValuesPageV1", new DataAnalysisDistinctValuesPageV1().toSchema()); + result.put("DataAnalysisDistinctValuesRequestV1", new DataAnalysisDistinctValuesRequestV1().toSchema()); + result.put("DataAnalysisDrillV1", new DataAnalysisDrillV1().toSchema()); + result.put("DataAnalysisDuplicatesRequestV1", new DataAnalysisDuplicatesRequestV1().toSchema()); + result.put("DataAnalysisDuplicatesResultV1", new DataAnalysisDuplicatesResultV1().toSchema()); + result.put("DataAnalysisFieldSummaryV1", new DataAnalysisFieldSummaryV1().toSchema()); + result.put("DataAnalysisMetricV1", new DataAnalysisMetricV1().toSchema()); + result.put("DataAnalysisNumericProfileV1", new DataAnalysisNumericProfileV1().toSchema()); + result.put("DataAnalysisPointV1", new DataAnalysisPointV1().toSchema()); + result.put("DataAnalysisRunStatusV1", new DataAnalysisRunStatusV1().toSchema()); + result.put("DataAnalysisSeriesV1", new DataAnalysisSeriesV1().toSchema()); + result.put("DataAnalysisTableProfileV1", new DataAnalysisTableProfileV1().toSchema()); + result.put("DataAnalysisTableProfileRequestV1", new DataAnalysisTableProfileRequestV1().toSchema()); + result.put("DataAnalysisTextProfileV1", new DataAnalysisTextProfileV1().toSchema()); + result.put("DataAnalysisValueCountV1", new DataAnalysisValueCountV1().toSchema()); + result.put("DataAnalysisCapabilitiesV1", new DataAnalysisCapabilitiesV1().toSchema()); + result.put("DataAnalysisCapabilityFieldV1", new DataAnalysisCapabilityFieldV1().toSchema()); + result.put("DataAnalysisLimitsV1", new DataAnalysisLimitsV1().toSchema()); + result.put("DataAnalysisStartRunRequestV1", new DataAnalysisStartRunRequestV1().toSchema()); + result.put("DataAnalysisRunStartedV1", new DataAnalysisRunStartedV1().toSchema()); + return (result); + } + + /******************************************************************************* + ** Axis version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisAxisV1 implements ToSchema + { + @OpenAPIDescription("Field.") + public String field; + + @OpenAPIDescription("Label.") + public String label; + + @OpenAPIDescription("Value type.") + public ValueType valueType; + + /******************************************************************************* + ** Explicit component references keep the standalone contract reusable. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("field", new Schema().withType(Type.STRING).withDescription("Field.").withNullable(true)); + properties.put("label", new Schema().withType(Type.STRING).withDescription("Label.").withNullable(true)); + properties.put("valueType", new Schema().withType(Type.STRING).withEnumValues(java.util.Arrays.stream(ValueType.values()).map(Enum::name).toList()).withDescription("Value type.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withNullable(true).withProperties(properties)); + } + } + + /******************************************************************************* + ** CandidateKey version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisCandidateKeyV1 implements ToSchema + { + @OpenAPIDescription("Readable fields to include; omit for all analyzable fields on a table profile.") + public List fields; + + @OpenAPIDescription("Row count.") + public Long rowCount; + + @OpenAPIDescription("Open the matching records using this table and QueryFilter; null when suppressed.") + public DataAnalysisDrillV1 drill; + + /******************************************************************************* + ** Explicit component references keep the standalone contract reusable. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("fields", new Schema().withType(Type.ARRAY).withItems(new Schema().withType(Type.STRING)).withDescription("Readable fields to include; omit for all analyzable fields on a table profile.").withNullable(true)); + properties.put("rowCount", new Schema().withType(Type.INTEGER).withDescription("Row count.").withNullable(true)); + properties.put("drill", ref("DataAnalysisDrillV1").withDescription("Open the matching records using this table and QueryFilter; null when suppressed.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withNullable(true).withProperties(properties)); + } + } + + /******************************************************************************* + ** Chart version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisChartV1 implements ToSchema + { + @OpenAPIDescription("Semantic field kind: NUMERIC, DATE, DATE_TIME, TEXT, BOOLEAN or CATEGORY.") + public ChartKind kind; + + @OpenAPIDescription("Alternatives.") + public List alternatives; + + @OpenAPIDescription("X.") + public DataAnalysisAxisV1 x; + + @OpenAPIDescription("Y.") + public DataAnalysisAxisV1 y; + + @OpenAPIDescription("Series.") + public List series; + + @OpenAPIDescription("Annotations.") + public List annotations; + + /******************************************************************************* + ** Explicit component references keep the standalone contract reusable. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("kind", new Schema().withType(Type.STRING).withEnumValues(java.util.Arrays.stream(ChartKind.values()).map(Enum::name).toList()).withDescription("Semantic field kind: NUMERIC, DATE, DATE_TIME, TEXT, BOOLEAN or CATEGORY.").withNullable(true)); + properties.put("alternatives", new Schema().withType(Type.ARRAY).withItems(new Schema().withType(Type.STRING).withEnumValues(java.util.Arrays.stream(ChartKind.values()).map(Enum::name).toList())).withDescription("Alternatives.").withNullable(true)); + properties.put("x", ref("DataAnalysisAxisV1").withDescription("X.").withNullable(true)); + properties.put("y", ref("DataAnalysisAxisV1").withDescription("Y.").withNullable(true)); + properties.put("series", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisSeriesV1")).withDescription("Series.").withNullable(true)); + properties.put("annotations", new Schema().withType(Type.ARRAY).withItems(new Schema().withType(Type.STRING)).withDescription("Annotations.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withNullable(true).withProperties(properties)); + } + } + + /******************************************************************************* + ** ColumnProfile version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisColumnProfileV1 implements ToSchema + { + @OpenAPIDescription("Table name.") + public String tableName; + + @OpenAPIDescription("Core QueryFilter JSON, identical to the table query and count endpoints.") + public QQueryFilter filter; + + @OpenAPIDescription("True when statistics describe a sample rather than the entire filtered population.") + public Boolean estimate; + + @OpenAPIDescription("Actual number of rows in the sample, not an extrapolated count.") + public Long sampleRows; + + @OpenAPIDescription("True when the row limit prevented a complete scan.") + public Boolean truncated; + + @OpenAPIDescription("Number of source rows included in computation.") + public Long rowsScanned; + + @OpenAPIDescription("As of.") + public String asOf; + + @OpenAPIDescription("Data version.") + public String dataVersion; + + @OpenAPIDescription("Warnings.") + public List warnings; + + @OpenAPIDescription("Readable field name to profile or enumerate.") + public String fieldName; + + @OpenAPIDescription("Field label.") + public String fieldLabel; + + @OpenAPIDescription("Field type.") + public String fieldType; + + @OpenAPIDescription("Counts.") + public List counts; + + @OpenAPIDescription("Numeric.") + public DataAnalysisNumericProfileV1 numeric; + + @OpenAPIDescription("Dates.") + public DataAnalysisDateProfileV1 dates; + + @OpenAPIDescription("Text.") + public DataAnalysisTextProfileV1 text; + + @OpenAPIDescription("Booleans.") + public List booleans; + + @OpenAPIDescription("Top values.") + public List topValues; + + @OpenAPIDescription("Chart.") + public DataAnalysisChartV1 chart; + + /******************************************************************************* + ** Explicit component references keep the standalone contract reusable. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("tableName", new Schema().withType(Type.STRING).withDescription("Table name.").withNullable(true)); + properties.put("filter", ref("QueryFilter").withDescription("Core QueryFilter JSON, identical to the table query and count endpoints.").withNullable(true)); + properties.put("estimate", new Schema().withType(Type.BOOLEAN).withDescription("True when statistics describe a sample rather than the entire filtered population.").withNullable(true)); + properties.put("sampleRows", new Schema().withType(Type.INTEGER).withDescription("Actual number of rows in the sample, not an extrapolated count.").withNullable(true)); + properties.put("truncated", new Schema().withType(Type.BOOLEAN).withDescription("True when the row limit prevented a complete scan.").withNullable(true)); + properties.put("rowsScanned", new Schema().withType(Type.INTEGER).withDescription("Number of source rows included in computation.").withNullable(true)); + properties.put("asOf", new Schema().withType(Type.STRING).withFormat("date-time").withDescription("As of.").withNullable(true)); + properties.put("dataVersion", new Schema().withType(Type.STRING).withDescription("Data version.").withNullable(true)); + properties.put("warnings", new Schema().withType(Type.ARRAY).withItems(new Schema().withType(Type.STRING)).withDescription("Warnings.").withNullable(true)); + properties.put("fieldName", new Schema().withType(Type.STRING).withDescription("Readable field name to profile or enumerate.").withNullable(true)); + properties.put("fieldLabel", new Schema().withType(Type.STRING).withDescription("Field label.").withNullable(true)); + properties.put("fieldType", new Schema().withType(Type.STRING).withDescription("Field type.").withNullable(true)); + properties.put("counts", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisMetricV1")).withDescription("Counts.").withNullable(true)); + properties.put("numeric", ref("DataAnalysisNumericProfileV1").withDescription("Numeric.").withNullable(true)); + properties.put("dates", ref("DataAnalysisDateProfileV1").withDescription("Dates.").withNullable(true)); + properties.put("text", ref("DataAnalysisTextProfileV1").withDescription("Text.").withNullable(true)); + properties.put("booleans", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisMetricV1")).withDescription("Booleans.").withNullable(true)); + properties.put("topValues", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisValueCountV1")).withDescription("Top values.").withNullable(true)); + properties.put("chart", ref("DataAnalysisChartV1").withDescription("Chart.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withNullable(true).withProperties(properties)); + } + } + + /******************************************************************************* + ** ColumnProfileRequest version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisColumnProfileRequestV1 implements ToSchema + { + @OpenAPIDescription("Core QueryFilter JSON, identical to the table query and count endpoints.") + public QQueryFilter filter; + + @OpenAPIDescription("Request exact computation instead of the configured estimate.") + public Boolean exact; + + @OpenAPIDescription("Readable field name to profile or enumerate.") + public String fieldName; + + /******************************************************************************* + ** Explicit component references keep the standalone contract reusable. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("filter", ref("QueryFilter").withDescription("Core QueryFilter JSON, identical to the table query and count endpoints.")); + properties.put("exact", new Schema().withType(Type.BOOLEAN).withDescription("Request exact computation instead of the configured estimate.")); + properties.put("fieldName", new Schema().withType(Type.STRING).withDescription("Readable field name to profile or enumerate.")); + return (new RequiredSchema(List.of("fieldName")).withType(Type.OBJECT).withProperties(properties).withAdditionalProperties(false)); + } + } + + /******************************************************************************* + ** DateProfile version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisDateProfileV1 implements ToSchema + { + @OpenAPIDescription("Metrics.") + public List metrics; + + @OpenAPIDescription("Unit.") + public String unit; + + @OpenAPIDescription("Buckets.") + public List buckets; + + @OpenAPIDescription("Gaps.") + public List gaps; + + @OpenAPIDescription("Day of week.") + public List dayOfWeek; + + @OpenAPIDescription("Hour of day.") + public List hourOfDay; + + /******************************************************************************* + ** Explicit component references keep the standalone contract reusable. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("metrics", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisMetricV1")).withDescription("Metrics.").withNullable(true)); + properties.put("unit", new Schema().withType(Type.STRING).withDescription("Unit.").withNullable(true)); + properties.put("buckets", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisValueCountV1")).withDescription("Buckets.").withNullable(true)); + properties.put("gaps", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisValueCountV1")).withDescription("Gaps.").withNullable(true)); + properties.put("dayOfWeek", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisValueCountV1")).withDescription("Day of week.").withNullable(true)); + properties.put("hourOfDay", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisValueCountV1")).withDescription("Hour of day.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withNullable(true).withProperties(properties)); + } + } + + /******************************************************************************* + ** DistinctValuesPage version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisDistinctValuesPageV1 implements ToSchema + { + @OpenAPIDescription("Table name.") + public String tableName; + + @OpenAPIDescription("Core QueryFilter JSON, identical to the table query and count endpoints.") + public QQueryFilter filter; + + @OpenAPIDescription("True when statistics describe a sample rather than the entire filtered population.") + public Boolean estimate; + + @OpenAPIDescription("Actual number of rows in the sample, not an extrapolated count.") + public Long sampleRows; + + @OpenAPIDescription("True when the row limit prevented a complete scan.") + public Boolean truncated; + + @OpenAPIDescription("Number of source rows included in computation.") + public Long rowsScanned; + + @OpenAPIDescription("As of.") + public String asOf; + + @OpenAPIDescription("Data version.") + public String dataVersion; + + @OpenAPIDescription("Warnings.") + public List warnings; + + @OpenAPIDescription("Readable field name to profile or enumerate.") + public String fieldName; + + @OpenAPIDescription("Zero-based page number.") + public Integer page; + + @OpenAPIDescription("Maximum values per page, from 1 to 500.") + public Integer pageSize; + + @OpenAPIDescription("Has more.") + public Boolean hasMore; + + @OpenAPIDescription("Total distinct.") + public Long totalDistinct; + + @OpenAPIDescription("Values.") + public List values; + + /******************************************************************************* + ** Explicit component references keep the standalone contract reusable. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("tableName", new Schema().withType(Type.STRING).withDescription("Table name.").withNullable(true)); + properties.put("filter", ref("QueryFilter").withDescription("Core QueryFilter JSON, identical to the table query and count endpoints.").withNullable(true)); + properties.put("estimate", new Schema().withType(Type.BOOLEAN).withDescription("True when statistics describe a sample rather than the entire filtered population.").withNullable(true)); + properties.put("sampleRows", new Schema().withType(Type.INTEGER).withDescription("Actual number of rows in the sample, not an extrapolated count.").withNullable(true)); + properties.put("truncated", new Schema().withType(Type.BOOLEAN).withDescription("True when the row limit prevented a complete scan.").withNullable(true)); + properties.put("rowsScanned", new Schema().withType(Type.INTEGER).withDescription("Number of source rows included in computation.").withNullable(true)); + properties.put("asOf", new Schema().withType(Type.STRING).withFormat("date-time").withDescription("As of.").withNullable(true)); + properties.put("dataVersion", new Schema().withType(Type.STRING).withDescription("Data version.").withNullable(true)); + properties.put("warnings", new Schema().withType(Type.ARRAY).withItems(new Schema().withType(Type.STRING)).withDescription("Warnings.").withNullable(true)); + properties.put("fieldName", new Schema().withType(Type.STRING).withDescription("Readable field name to profile or enumerate.").withNullable(true)); + properties.put("page", new Schema().withType(Type.INTEGER).withDescription("Zero-based page number.").withNullable(true)); + properties.put("pageSize", new Schema().withType(Type.INTEGER).withDescription("Maximum values per page, from 1 to 500.").withNullable(true)); + properties.put("hasMore", new Schema().withType(Type.BOOLEAN).withDescription("Has more.").withNullable(true)); + properties.put("totalDistinct", new Schema().withType(Type.INTEGER).withDescription("Total distinct.").withNullable(true)); + properties.put("values", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisValueCountV1")).withDescription("Values.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withNullable(true).withProperties(properties)); + } + } + + /******************************************************************************* + ** DistinctValuesRequest version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisDistinctValuesRequestV1 implements ToSchema + { + @OpenAPIDescription("Core QueryFilter JSON, identical to the table query and count endpoints.") + public QQueryFilter filter; + + @OpenAPIDescription("Readable field name to profile or enumerate.") + public String fieldName; + + @OpenAPIDescription("Zero-based page number.") + public Integer page; + + @OpenAPIDescription("Maximum values per page, from 1 to 500.") + public Integer pageSize; + + @OpenAPIDescription("Optional search text applied to stored values.") + public String search; + + /******************************************************************************* + ** Explicit component references keep the standalone contract reusable. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("filter", ref("QueryFilter").withDescription("Core QueryFilter JSON, identical to the table query and count endpoints.")); + properties.put("fieldName", new Schema().withType(Type.STRING).withDescription("Readable field name to profile or enumerate.")); + properties.put("page", new Schema().withType(Type.INTEGER).withDescription("Zero-based page number.")); + properties.put("pageSize", new Schema().withType(Type.INTEGER).withDescription("Maximum values per page, from 1 to 500.")); + properties.put("search", new Schema().withType(Type.STRING).withDescription("Optional search text applied to stored values.")); + return (new RequiredSchema(List.of("fieldName")).withType(Type.OBJECT).withProperties(properties).withAdditionalProperties(false)); + } + } + + /******************************************************************************* + ** Drill version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisDrillV1 implements ToSchema + { + @OpenAPIDescription("Table name.") + public String tableName; + + @OpenAPIDescription("Core QueryFilter JSON, identical to the table query and count endpoints.") + public QQueryFilter filter; + + /******************************************************************************* + ** Explicit component references keep the standalone contract reusable. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("tableName", new Schema().withType(Type.STRING).withDescription("Table name.").withNullable(true)); + properties.put("filter", ref("QueryFilter").withDescription("Core QueryFilter JSON, identical to the table query and count endpoints.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withNullable(true).withProperties(properties)); + } + } + + /******************************************************************************* + ** DuplicatesRequest version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisDuplicatesRequestV1 implements ToSchema + { + @OpenAPIDescription("Core QueryFilter JSON, identical to the table query and count endpoints.") + public QQueryFilter filter; + + @OpenAPIDescription("Readable fields to include; omit for all analyzable fields on a table profile.") + public List fields; + + @OpenAPIDescription("Maximum duplicate groups to return, from 1 to 500; default 50.") + public Integer limit; + + /******************************************************************************* + ** Explicit component references keep the standalone contract reusable. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("filter", ref("QueryFilter").withDescription("Core QueryFilter JSON, identical to the table query and count endpoints.")); + properties.put("fields", new Schema().withType(Type.ARRAY).withItems(new Schema().withType(Type.STRING)).withDescription("Readable fields to include; omit for all analyzable fields on a table profile.")); + properties.put("limit", new Schema().withType(Type.INTEGER).withDescription("Maximum duplicate groups to return, from 1 to 500; default 50.")); + return (new RequiredSchema(List.of("fields")).withType(Type.OBJECT).withProperties(properties).withAdditionalProperties(false)); + } + } + + /******************************************************************************* + ** DuplicatesResult version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisDuplicatesResultV1 implements ToSchema + { + @OpenAPIDescription("Table name.") + public String tableName; + + @OpenAPIDescription("Core QueryFilter JSON, identical to the table query and count endpoints.") + public QQueryFilter filter; + + @OpenAPIDescription("True when statistics describe a sample rather than the entire filtered population.") + public Boolean estimate; + + @OpenAPIDescription("Actual number of rows in the sample, not an extrapolated count.") + public Long sampleRows; + + @OpenAPIDescription("True when the row limit prevented a complete scan.") + public Boolean truncated; + + @OpenAPIDescription("Number of source rows included in computation.") + public Long rowsScanned; + + @OpenAPIDescription("As of.") + public String asOf; + + @OpenAPIDescription("Data version.") + public String dataVersion; + + @OpenAPIDescription("Warnings.") + public List warnings; + + @OpenAPIDescription("Readable fields to include; omit for all analyzable fields on a table profile.") + public List fields; + + @OpenAPIDescription("Groups.") + public List groups; + + /******************************************************************************* + ** Explicit component references keep the standalone contract reusable. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("tableName", new Schema().withType(Type.STRING).withDescription("Table name.").withNullable(true)); + properties.put("filter", ref("QueryFilter").withDescription("Core QueryFilter JSON, identical to the table query and count endpoints.").withNullable(true)); + properties.put("estimate", new Schema().withType(Type.BOOLEAN).withDescription("True when statistics describe a sample rather than the entire filtered population.").withNullable(true)); + properties.put("sampleRows", new Schema().withType(Type.INTEGER).withDescription("Actual number of rows in the sample, not an extrapolated count.").withNullable(true)); + properties.put("truncated", new Schema().withType(Type.BOOLEAN).withDescription("True when the row limit prevented a complete scan.").withNullable(true)); + properties.put("rowsScanned", new Schema().withType(Type.INTEGER).withDescription("Number of source rows included in computation.").withNullable(true)); + properties.put("asOf", new Schema().withType(Type.STRING).withFormat("date-time").withDescription("As of.").withNullable(true)); + properties.put("dataVersion", new Schema().withType(Type.STRING).withDescription("Data version.").withNullable(true)); + properties.put("warnings", new Schema().withType(Type.ARRAY).withItems(new Schema().withType(Type.STRING)).withDescription("Warnings.").withNullable(true)); + properties.put("fields", new Schema().withType(Type.ARRAY).withItems(new Schema().withType(Type.STRING)).withDescription("Readable fields to include; omit for all analyzable fields on a table profile.").withNullable(true)); + properties.put("groups", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisValueCountV1")).withDescription("Groups.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withNullable(true).withProperties(properties)); + } + } + + /******************************************************************************* + ** FieldSummary version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisFieldSummaryV1 implements ToSchema + { + @OpenAPIDescription("Field.") + public String field; + + @OpenAPIDescription("Label.") + public String label; + + @OpenAPIDescription("Type.") + public String type; + + @OpenAPIDescription("Completeness.") + public BigDecimal completeness; + + @OpenAPIDescription("Distinct.") + public Long distinct; + + @OpenAPIDescription("Uniqueness.") + public BigDecimal uniqueness; + + @OpenAPIDescription("Constant.") + public Boolean constant; + + @OpenAPIDescription("Empty.") + public Boolean empty; + + @OpenAPIDescription("Nulls.") + public DataAnalysisMetricV1 nulls; + + @OpenAPIDescription("Blanks.") + public DataAnalysisMetricV1 blanks; + + /******************************************************************************* + ** Explicit component references keep the standalone contract reusable. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("field", new Schema().withType(Type.STRING).withDescription("Field.").withNullable(true)); + properties.put("label", new Schema().withType(Type.STRING).withDescription("Label.").withNullable(true)); + properties.put("type", new Schema().withType(Type.STRING).withDescription("Type.").withNullable(true)); + properties.put("completeness", new Schema().withType(Type.NUMBER).withDescription("Completeness.").withNullable(true)); + properties.put("distinct", new Schema().withType(Type.INTEGER).withDescription("Distinct.").withNullable(true)); + properties.put("uniqueness", new Schema().withType(Type.NUMBER).withDescription("Uniqueness.").withNullable(true)); + properties.put("constant", new Schema().withType(Type.BOOLEAN).withDescription("Constant.").withNullable(true)); + properties.put("empty", new Schema().withType(Type.BOOLEAN).withDescription("Empty.").withNullable(true)); + properties.put("nulls", ref("DataAnalysisMetricV1").withDescription("Nulls.").withNullable(true)); + properties.put("blanks", ref("DataAnalysisMetricV1").withDescription("Blanks.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withNullable(true).withProperties(properties)); + } + } + + /******************************************************************************* + ** Metric version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisMetricV1 implements ToSchema + { + @OpenAPIDescription("Key.") + public Boolean estimate; + public String key; + + @OpenAPIDescription("Label.") + public String label; + + @OpenAPIDescription("Value.") + public Serializable value; + + @OpenAPIDescription("Value type.") + public ValueType valueType; + + @OpenAPIDescription("Formatted.") + public String formatted; + + @OpenAPIDescription("Open the matching records using this table and QueryFilter; null when suppressed.") + public DataAnalysisDrillV1 drill; + + @OpenAPIDescription("Suppressed.") + public Boolean suppressed; + + @OpenAPIDescription("Suppressed reason.") + public String suppressedReason; + + /******************************************************************************* + ** Explicit component references keep the standalone contract reusable. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("key", new Schema().withType(Type.STRING).withDescription("Key.").withNullable(true)); + properties.put("label", new Schema().withType(Type.STRING).withDescription("Label.").withNullable(true)); + properties.put("value", new Schema().withOneOf(List.of(new Schema().withType(Type.STRING), new Schema().withType(Type.NUMBER), new Schema().withType(Type.BOOLEAN), new Schema().withType(Type.OBJECT).withNullable(true))).withDescription("Value.").withNullable(true)); + properties.put("valueType", new Schema().withType(Type.STRING).withEnumValues(java.util.Arrays.stream(ValueType.values()).map(Enum::name).toList()).withDescription("Value type.").withNullable(true)); + properties.put("estimate", new Schema().withType(Type.BOOLEAN).withDescription("Whether this metric is estimated; the population rows metric remains exact.")); + properties.put("formatted", new Schema().withType(Type.STRING).withDescription("Formatted.").withNullable(true)); + properties.put("drill", ref("DataAnalysisDrillV1").withDescription("Open the matching records using this table and QueryFilter; null when suppressed.").withNullable(true)); + properties.put("suppressed", new Schema().withType(Type.BOOLEAN).withDescription("Suppressed.").withNullable(true)); + properties.put("suppressedReason", new Schema().withType(Type.STRING).withDescription("Suppressed reason.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withNullable(true).withProperties(properties)); + } + } + + /******************************************************************************* + ** NumericProfile version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisNumericProfileV1 implements ToSchema + { + @OpenAPIDescription("Metrics.") + public List metrics; + + @OpenAPIDescription("Histogram.") + public List histogram; + + @OpenAPIDescription("Outliers.") + public List outliers; + + /******************************************************************************* + ** Explicit component references keep the standalone contract reusable. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("metrics", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisMetricV1")).withDescription("Metrics.").withNullable(true)); + properties.put("histogram", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisValueCountV1")).withDescription("Histogram.").withNullable(true)); + properties.put("outliers", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisValueCountV1")).withDescription("Outliers.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withNullable(true).withProperties(properties)); + } + } + + /******************************************************************************* + ** Point version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisPointV1 implements ToSchema + { + @OpenAPIDescription("X.") + public Serializable x; + + @OpenAPIDescription("Y.") + public BigDecimal y; + + @OpenAPIDescription("Label.") + public String label; + + @OpenAPIDescription("Open the matching records using this table and QueryFilter; null when suppressed.") + public DataAnalysisDrillV1 drill; + + /******************************************************************************* + ** Explicit component references keep the standalone contract reusable. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("x", new Schema().withOneOf(List.of(new Schema().withType(Type.STRING), new Schema().withType(Type.NUMBER), new Schema().withType(Type.BOOLEAN), new Schema().withType(Type.OBJECT).withNullable(true))).withDescription("X.").withNullable(true)); + properties.put("y", new Schema().withType(Type.NUMBER).withDescription("Y.").withNullable(true)); + properties.put("label", new Schema().withType(Type.STRING).withDescription("Label.").withNullable(true)); + properties.put("drill", ref("DataAnalysisDrillV1").withDescription("Open the matching records using this table and QueryFilter; null when suppressed.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withNullable(true).withProperties(properties)); + } + } + + /******************************************************************************* + ** RunStatus version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisRunStatusV1 implements ToSchema + { + @OpenAPIDescription("Run id.") + public String runId; + + @OpenAPIDescription("Status.") + public String status; + + @OpenAPIDescription("Error.") + public String error; + + @OpenAPIDescription("Completed column result; null for other operations or unfinished runs.") + public DataAnalysisColumnProfileV1 columnProfile; + + @OpenAPIDescription("Completed table result; null for other operations or unfinished runs.") + public DataAnalysisTableProfileV1 tableProfile; + + @OpenAPIDescription("Started at.") + public String startedAt; + + @OpenAPIDescription("Finished at.") + public String finishedAt; + + /******************************************************************************* + ** Explicit component references keep the standalone contract reusable. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("runId", new Schema().withType(Type.STRING).withDescription("Run id.").withNullable(true)); + properties.put("status", new Schema().withType(Type.STRING).withDescription("Status.").withNullable(true)); + properties.put("error", new Schema().withType(Type.STRING).withDescription("Error.").withNullable(true)); + properties.put("columnProfile", ref("DataAnalysisColumnProfileV1").withDescription("Completed column result; null for other operations or unfinished runs.").withNullable(true)); + properties.put("tableProfile", ref("DataAnalysisTableProfileV1").withDescription("Completed table result; null for other operations or unfinished runs.").withNullable(true)); + properties.put("startedAt", new Schema().withType(Type.STRING).withFormat("date-time").withDescription("Started at.").withNullable(true)); + properties.put("finishedAt", new Schema().withType(Type.STRING).withFormat("date-time").withDescription("Finished at.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withNullable(true).withProperties(properties)); + } + } + + /******************************************************************************* + ** Series version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisSeriesV1 implements ToSchema + { + @OpenAPIDescription("Name.") + public String name; + + @OpenAPIDescription("Points.") + public List points; + + /******************************************************************************* + ** Explicit component references keep the standalone contract reusable. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("name", new Schema().withType(Type.STRING).withDescription("Name.").withNullable(true)); + properties.put("points", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisPointV1")).withDescription("Points.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withNullable(true).withProperties(properties)); + } + } + + /******************************************************************************* + ** TableProfile version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisTableProfileV1 implements ToSchema + { + @OpenAPIDescription("Table name.") + public String tableName; + + @OpenAPIDescription("Core QueryFilter JSON, identical to the table query and count endpoints.") + public QQueryFilter filter; + + @OpenAPIDescription("True when statistics describe a sample rather than the entire filtered population.") + public Boolean estimate; + + @OpenAPIDescription("Actual number of rows in the sample, not an extrapolated count.") + public Long sampleRows; + + @OpenAPIDescription("True when the row limit prevented a complete scan.") + public Boolean truncated; + + @OpenAPIDescription("Number of source rows included in computation.") + public Long rowsScanned; + + @OpenAPIDescription("As of.") + public String asOf; + + @OpenAPIDescription("Data version.") + public String dataVersion; + + @OpenAPIDescription("Warnings.") + public List warnings; + + @OpenAPIDescription("Row count.") + public DataAnalysisMetricV1 rowCount; + + @OpenAPIDescription("Readable fields to include; omit for all analyzable fields on a table profile.") + public List fields; + + @OpenAPIDescription("Heatmap.") + public DataAnalysisChartV1 heatmap; + + @OpenAPIDescription("Candidate keys.") + public List candidateKeys; + + /******************************************************************************* + ** Explicit component references keep the standalone contract reusable. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("tableName", new Schema().withType(Type.STRING).withDescription("Table name.").withNullable(true)); + properties.put("filter", ref("QueryFilter").withDescription("Core QueryFilter JSON, identical to the table query and count endpoints.").withNullable(true)); + properties.put("estimate", new Schema().withType(Type.BOOLEAN).withDescription("True when statistics describe a sample rather than the entire filtered population.").withNullable(true)); + properties.put("sampleRows", new Schema().withType(Type.INTEGER).withDescription("Actual number of rows in the sample, not an extrapolated count.").withNullable(true)); + properties.put("truncated", new Schema().withType(Type.BOOLEAN).withDescription("True when the row limit prevented a complete scan.").withNullable(true)); + properties.put("rowsScanned", new Schema().withType(Type.INTEGER).withDescription("Number of source rows included in computation.").withNullable(true)); + properties.put("asOf", new Schema().withType(Type.STRING).withFormat("date-time").withDescription("As of.").withNullable(true)); + properties.put("dataVersion", new Schema().withType(Type.STRING).withDescription("Data version.").withNullable(true)); + properties.put("warnings", new Schema().withType(Type.ARRAY).withItems(new Schema().withType(Type.STRING)).withDescription("Warnings.").withNullable(true)); + properties.put("rowCount", ref("DataAnalysisMetricV1").withDescription("Row count.").withNullable(true)); + properties.put("fields", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisFieldSummaryV1")).withDescription("Readable fields to include; omit for all analyzable fields on a table profile.").withNullable(true)); + properties.put("heatmap", ref("DataAnalysisChartV1").withDescription("Heatmap.").withNullable(true)); + properties.put("candidateKeys", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisCandidateKeyV1")).withDescription("Candidate keys.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withNullable(true).withProperties(properties)); + } + } + + /******************************************************************************* + ** TableProfileRequest version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisTableProfileRequestV1 implements ToSchema + { + @OpenAPIDescription("Core QueryFilter JSON, identical to the table query and count endpoints.") + public QQueryFilter filter; + + @OpenAPIDescription("Readable fields to include; omit for all analyzable fields on a table profile.") + public List fields; + + /******************************************************************************* + ** Explicit component references keep the standalone contract reusable. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("filter", ref("QueryFilter").withDescription("Core QueryFilter JSON, identical to the table query and count endpoints.")); + properties.put("fields", new Schema().withType(Type.ARRAY).withItems(new Schema().withType(Type.STRING)).withDescription("Readable fields to include; omit for all analyzable fields on a table profile.")); + return (new Schema().withType(Type.OBJECT).withProperties(properties).withAdditionalProperties(false)); + } + } + + /******************************************************************************* + ** TextProfile version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisTextProfileV1 implements ToSchema + { + @OpenAPIDescription("Metrics.") + public List metrics; + + @OpenAPIDescription("Histogram.") + public List histogram; + + @OpenAPIDescription("Patterns.") + public List patterns; + + @OpenAPIDescription("Actual number of rows in the sample, not an extrapolated count.") + public Long sampleRows; + + /******************************************************************************* + ** Explicit component references keep the standalone contract reusable. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("metrics", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisMetricV1")).withDescription("Metrics.").withNullable(true)); + properties.put("histogram", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisValueCountV1")).withDescription("Histogram.").withNullable(true)); + properties.put("patterns", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisValueCountV1")).withDescription("Patterns.").withNullable(true)); + properties.put("sampleRows", new Schema().withType(Type.INTEGER).withDescription("Actual number of rows in the sample, not an extrapolated count.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withNullable(true).withProperties(properties)); + } + } + + /******************************************************************************* + ** ValueCount version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisValueCountV1 implements ToSchema + { + @OpenAPIDescription("Value.") + public Serializable value; + + @OpenAPIDescription("Label.") + public String label; + + @OpenAPIDescription("Count.") + public Long count; + + @OpenAPIDescription("Share.") + public BigDecimal share; + + @OpenAPIDescription("Open the matching records using this table and QueryFilter; null when suppressed.") + public DataAnalysisDrillV1 drill; + + /******************************************************************************* + ** Explicit component references keep the standalone contract reusable. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("value", new Schema().withOneOf(List.of(new Schema().withType(Type.STRING), new Schema().withType(Type.NUMBER), new Schema().withType(Type.BOOLEAN), new Schema().withType(Type.OBJECT).withNullable(true))).withDescription("Value.").withNullable(true)); + properties.put("label", new Schema().withType(Type.STRING).withDescription("Label.").withNullable(true)); + properties.put("count", new Schema().withType(Type.INTEGER).withDescription("Count.").withNullable(true)); + properties.put("share", new Schema().withType(Type.NUMBER).withDescription("Share.").withNullable(true)); + properties.put("drill", ref("DataAnalysisDrillV1").withDescription("Open the matching records using this table and QueryFilter; null when suppressed.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withNullable(true).withProperties(properties)); + } + } + + /******************************************************************************* + ** Capabilities version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisCapabilitiesV1 implements ToSchema + { + @OpenAPIDescription("Whether this session may analyze the table; false responses expose no fields.") + public Boolean canAnalyze; + + @OpenAPIDescription("Table name.") + public String tableName; + + @OpenAPIDescription("Table label.") + public String tableLabel; + + @OpenAPIDescription("Readable fields to include; omit for all analyzable fields on a table profile.") + public List fields; + + @OpenAPIDescription("Limits.") + public DataAnalysisLimitsV1 limits; + + @OpenAPIDescription("Backend capabilities.") + public List backendCapabilities; + + @OpenAPIDescription("Whether SQL Workbench metadata is installed in this instance.") + public Boolean sqlWorkbenchAvailable; + + /******************************************************************************* + ** Explicit component references keep the standalone contract reusable. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("canAnalyze", new Schema().withType(Type.BOOLEAN).withDescription("Whether this session may analyze the table; false responses expose no fields.").withNullable(true)); + properties.put("tableName", new Schema().withType(Type.STRING).withDescription("Table name.").withNullable(true)); + properties.put("tableLabel", new Schema().withType(Type.STRING).withDescription("Table label.").withNullable(true)); + properties.put("fields", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisCapabilityFieldV1")).withDescription("Readable fields to include; omit for all analyzable fields on a table profile.").withNullable(true)); + properties.put("limits", ref("DataAnalysisLimitsV1").withDescription("Limits.").withNullable(true)); + properties.put("backendCapabilities", new Schema().withType(Type.ARRAY).withItems(new Schema().withType(Type.STRING)).withDescription("Backend capabilities.").withNullable(true)); + properties.put("sqlWorkbenchAvailable", new Schema().withType(Type.BOOLEAN).withDescription("Whether SQL Workbench metadata is installed in this instance.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withNullable(true).withProperties(properties)); + } + } + + /******************************************************************************* + ** CapabilityField version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisCapabilityFieldV1 implements ToSchema + { + @OpenAPIDescription("Name.") + public String name; + + @OpenAPIDescription("Label.") + public String label; + + @OpenAPIDescription("Type.") + public String type; + + @OpenAPIDescription("Semantic field kind: NUMERIC, DATE, DATE_TIME, TEXT, BOOLEAN or CATEGORY.") + public String kind; + + @OpenAPIDescription("Default chart kind.") + public ChartKind defaultChartKind; + + /******************************************************************************* + ** Explicit component references keep the standalone contract reusable. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("name", new Schema().withType(Type.STRING).withDescription("Name.").withNullable(true)); + properties.put("label", new Schema().withType(Type.STRING).withDescription("Label.").withNullable(true)); + properties.put("type", new Schema().withType(Type.STRING).withDescription("Type.").withNullable(true)); + properties.put("kind", new Schema().withType(Type.STRING).withDescription("Semantic field kind: NUMERIC, DATE, DATE_TIME, TEXT, BOOLEAN or CATEGORY.").withNullable(true)); + properties.put("defaultChartKind", new Schema().withType(Type.STRING).withEnumValues(java.util.Arrays.stream(ChartKind.values()).map(Enum::name).toList()).withDescription("Default chart kind.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withNullable(true).withProperties(properties)); + } + } + + /******************************************************************************* + ** Limits version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisLimitsV1 implements ToSchema + { + @OpenAPIDescription("Max rows.") + public Integer maxRows; + + @OpenAPIDescription("Sample threshold rows.") + public Integer sampleThresholdRows; + + @OpenAPIDescription("Actual number of rows in the sample, not an extrapolated count.") + public Integer sampleRows; + + @OpenAPIDescription("Min group size.") + public Integer minGroupSize; + + /******************************************************************************* + ** Explicit component references keep the standalone contract reusable. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("maxRows", new Schema().withType(Type.INTEGER).withDescription("Max rows.").withNullable(true)); + properties.put("sampleThresholdRows", new Schema().withType(Type.INTEGER).withDescription("Sample threshold rows.").withNullable(true)); + properties.put("sampleRows", new Schema().withType(Type.INTEGER).withDescription("Actual number of rows in the sample, not an extrapolated count.").withNullable(true)); + properties.put("minGroupSize", new Schema().withType(Type.INTEGER).withDescription("Min group size.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withNullable(true).withProperties(properties)); + } + } + + /******************************************************************************* + ** StartRunRequest version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisStartRunRequestV1 implements ToSchema + { + @OpenAPIDescription("COLUMN_PROFILE or TABLE_PROFILE.") + public String operation; + + @OpenAPIDescription("Table name.") + public String tableName; + + @OpenAPIDescription("Profile parameters for the selected operation, without tableName.") + public Object request; + + /******************************************************************************* + ** Explicit component references keep the standalone contract reusable. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("operation", new Schema().withType(Type.STRING).withEnumValues(List.of("COLUMN_PROFILE", "TABLE_PROFILE")).withDescription("COLUMN_PROFILE or TABLE_PROFILE.")); + properties.put("tableName", new Schema().withType(Type.STRING).withDescription("Table name.")); + properties.put("request", new Schema().withOneOf(List.of(ref("DataAnalysisColumnProfileRequestV1"), ref("DataAnalysisTableProfileRequestV1"))).withDescription("Profile parameters for the selected operation, without tableName.")); + return (new RequiredSchema(List.of("operation", "tableName", "request")).withType(Type.OBJECT).withProperties(properties).withAdditionalProperties(false)); + } + } + + /******************************************************************************* + ** RunStarted version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisRunStartedV1 implements ToSchema + { + @OpenAPIDescription("Run id.") + public String runId; + + @OpenAPIDescription("Status.") + public String status; + + /******************************************************************************* + ** Explicit component references keep the standalone contract reusable. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("runId", new Schema().withType(Type.STRING).withDescription("Run id.").withNullable(true)); + properties.put("status", new Schema().withType(Type.STRING).withDescription("Status.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withNullable(true).withProperties(properties)); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisBackend.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisBackend.java index 54ec58a..14ebe27 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisBackend.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisBackend.java @@ -44,10 +44,27 @@ public interface AnalysisBackend /******************************************************************************* - ** Actual row coverage and deterministic selected population. + ** Computation scope, observed coverage and optional native sampling percentage. *******************************************************************************/ - record Scope(QQueryFilter filter, long total, long rowsScanned, boolean truncated, boolean estimate) + record Scope(QQueryFilter filter, long total, long rowsScanned, boolean truncated, boolean estimate, BigDecimal samplePercent) { + /******************************************************************************* + ** Exact and prefix scopes have no random sampling percentage. + *******************************************************************************/ + public Scope(QQueryFilter filter, long total, long rowsScanned, boolean truncated, boolean estimate) + { + this(filter, total, rowsScanned, truncated, estimate, null); + } + + /******************************************************************************* + ** Explain whether counts extrapolate a random sample or describe a prefix. + *******************************************************************************/ + public String warning() + { + return (samplePercent == null ? "Estimate from the first " + rowsScanned + " rows by id" + : "Estimate from a random sample of about " + rowsScanned + " rows (" + samplePercent.stripTrailingZeros().toPlainString() + + "%). Counts are scaled to the full " + total + " rows; run exact for exact numbers."); + } } @@ -55,8 +72,16 @@ public interface AnalysisBackend /******************************************************************************* ** Blank includes null and whitespace; distinct excludes null. *******************************************************************************/ - record FieldCounts(long nonNull, long nulls, long blanks, long distinct, QQueryFilter blankFilter, QQueryFilter distinctFilter) + record FieldCounts(long nonNull, long nulls, long blanks, long distinct, QQueryFilter blankFilter, QQueryFilter distinctFilter, BigDecimal uniqueness, BigDecimal completeness, boolean constant) { + /******************************************************************************* + ** Exact and prefix counters use their own population for ratios. + *******************************************************************************/ + public FieldCounts(long nonNull, long nulls, long blanks, long distinct, QQueryFilter blankFilter, QQueryFilter distinctFilter) + { + this(nonNull, nulls, blanks, distinct, blankFilter, distinctFilter, ProfileSupport.share(distinct, nonNull), + ProfileSupport.share(nonNull + nulls - blanks, nonNull + nulls), distinct == 1); + } } @@ -64,8 +89,15 @@ public interface AnalysisBackend /******************************************************************************* ** Group and exact predicate producing its members. *******************************************************************************/ - record GroupCount(Serializable value, String label, long count, QQueryFilter condition) + record GroupCount(Serializable value, String label, long count, QQueryFilter condition, BigDecimal share) { + /******************************************************************************* + ** Ordinary groups derive their share from the profiler population. + *******************************************************************************/ + public GroupCount(Serializable value, String label, long count, QQueryFilter condition) + { + this(value, label, count, condition, null); + } } @@ -89,12 +121,12 @@ public interface AnalysisBackend /******************************************************************************* - ** Exact filtered count. + ** Filtered count; a native estimate scales conditional counts to its population. *******************************************************************************/ long count(String table, QQueryFilter filter) throws QException; /******************************************************************************* - ** Select the bounded population used consistently by all operations. + ** Configure a request-local exact, prefix or native sampling scope. *******************************************************************************/ Scope scope(String table, QQueryFilter filter, boolean exact) throws QException; diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ColumnProfiler.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ColumnProfiler.java index 7710dcf..55edeb3 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ColumnProfiler.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ColumnProfiler.java @@ -68,38 +68,40 @@ public class ColumnProfiler String name = field.getName(); QQueryFilter filter = scope.filter(); long rows = backend.count(table, filter); + QQueryFilter drillFilter = scope.estimate() ? request.getFilter() : filter; AnalysisBackend.FieldCounts counts = backend.fieldCounts(table, filter, List.of(name)).get(name); ColumnProfile result = new ColumnProfile().withTableName(table).withFieldName(name).withFieldLabel(field.getLabel()).withFieldType(field.getType().name()) - .withFilter(DrillFilters.restoreTypes(request.getTableName(), request.getFilter())).withEstimate(scope.estimate()).withSampleRows(scope.estimate() ? rows : 0L) + .withFilter(DrillFilters.restoreTypes(request.getTableName(), request.getFilter())).withEstimate(scope.estimate()).withSampleRows(scope.estimate() ? scope.rowsScanned() : 0L) .withRowsScanned(scope.rowsScanned()).withTruncated(scope.truncated()).withAsOf(Instant.now()); - result.getCounts().add(ProfileSupport.count("rows", rows, table, filter, null)); - result.getCounts().add(ProfileSupport.count("nonNull", counts.nonNull(), table, filter, DrillFilters.nonNull(name, field.getType().isStringLike()))); - result.getCounts().add(ProfileSupport.count("nulls", counts.nulls(), table, filter, DrillFilters.nulls(name, field.getType().isStringLike()))); + result.getCounts().add(ProfileSupport.count("rows", scope.estimate() ? scope.total() : rows, table, drillFilter, null)); + result.getCounts().add(ProfileSupport.count("nonNull", counts.nonNull(), table, drillFilter, DrillFilters.nonNull(name, field.getType().isStringLike()))); + result.getCounts().add(ProfileSupport.count("nulls", counts.nulls(), table, drillFilter, DrillFilters.nulls(name, field.getType().isStringLike()))); if(field.getType().isStringLike()) { - result.getCounts().add(ProfileSupport.count("blanks", counts.blanks(), table, filter, counts.blankFilter())); + result.getCounts().add(ProfileSupport.count("blanks", counts.blanks(), table, drillFilter, counts.blankFilter())); } - result.getCounts().add(ProfileSupport.count("distinct", counts.distinct(), table, filter, counts.distinctFilter())); - result.getCounts().add(ProfileSupport.metric("uniqueness", ProfileSupport.share(counts.distinct(), counts.nonNull()), ValueType.PERCENT)); - result.withTopValues(ProfileSupport.values(table, filter, field, backend.topValues(table, filter, name, 20, 0, null), rows)); + result.getCounts().add(ProfileSupport.count("distinct", counts.distinct(), table, drillFilter, counts.distinctFilter())); + result.getCounts().add(ProfileSupport.metric("uniqueness", counts.uniqueness(), ValueType.PERCENT)); + result.withTopValues(ProfileSupport.values(table, drillFilter, field, backend.topValues(table, filter, name, 20, 0, null), rows)); QFieldType type = field.getType(); if(type == QFieldType.INTEGER || type == QFieldType.LONG || type == QFieldType.DECIMAL) { - numeric(result, filter, rows); + numeric(result, filter, drillFilter, rows); } else if(type == QFieldType.DATE || type == QFieldType.DATE_TIME) { - dates(result, filter, rows, type); + dates(result, filter, drillFilter, rows, type); } else if(type.isStringLike()) { AnalysisBackend.LengthSummary summary = backend.lengthSummary(table, filter, name); - TextProfile text = new TextProfile().withHistogram(ProfileSupport.values(table, filter, null, summary.histogram(), rows)) - .withPatterns(ProfileSupport.values(table, filter, null, backend.patterns(table, filter, name, 10_000), Math.min(rows, 10_000))) - .withSampleRows(Math.min(rows, 10_000)); + TextProfile text = new TextProfile().withHistogram(ProfileSupport.values(table, drillFilter, null, summary.histogram(), rows)) + .withPatterns(ProfileSupport.values(table, drillFilter, null, backend.patterns(table, filter, name, 10_000), Math.min(rows, 10_000))) + .withSampleRows(Math.min(scope.rowsScanned(), 10_000)); summary.metrics().forEach((key, value) -> text.getMetrics().add(ProfileSupport.metric(key, value, ValueType.DECIMAL))); result.withText(text); - result.getWarnings().add("Patterns use the first " + Math.min(rows, 10_000) + " rows by primary key."); + result.getWarnings().add(scope.samplePercent() == null ? "Patterns use the first " + Math.min(rows, 10_000) + " rows by primary key." + : "Patterns use up to " + Math.min(scope.rowsScanned(), 10_000) + " sampled rows."); } else if(type == QFieldType.BOOLEAN) { @@ -107,11 +109,28 @@ public class ColumnProfiler for(boolean value : List.of(true, false)) { QQueryFilter condition = DrillFilters.value(name, value, false); - result.getBooleans().add(ProfileSupport.count(Boolean.toString(value), backend.count(table, DrillFilters.and(filter, condition)), table, filter, condition)); + result.getBooleans().add(ProfileSupport.count(Boolean.toString(value), backend.count(table, DrillFilters.and(filter, condition)), table, drillFilter, condition)); } - result.getBooleans().add(ProfileSupport.count("nulls", counts.nulls(), table, filter, DrillFilters.nulls(name, false))); + result.getBooleans().add(ProfileSupport.count("nulls", counts.nulls(), table, drillFilter, DrillFilters.nulls(name, false))); } new MinGroupSizeGuard(config.getMinGroupSize()).column(result, rows); + if(scope.estimate()) + { + ProfileSupport.markEstimates(result.getCounts()); + ProfileSupport.markEstimates(result.getBooleans()); + if(result.getNumeric() != null) + { + ProfileSupport.markEstimates(result.getNumeric().getMetrics()); + } + if(result.getDates() != null) + { + ProfileSupport.markEstimates(result.getDates().getMetrics()); + } + if(result.getText() != null) + { + ProfileSupport.markEstimates(result.getText().getMetrics()); + } + } if(field.getPossibleValueSourceName() != null || result.getNumeric() == null && result.getDates() == null) { result.withChart(ProfileSupport.chart(ChartKind.BAR, ChartKind.PARETO, name, result.getTopValues())); @@ -128,17 +147,22 @@ public class ColumnProfiler { result.getWarnings().add("No readable records match this analysis scope."); } - else if(counts.nonNull() == 0) + else if(counts.nonNull() == 0 && (!scope.estimate() || scope.rowsScanned() > 0)) { result.getWarnings().add("All values in this analysis scope are null."); } if(scope.estimate()) { - result.getWarnings().add("Estimate: first " + rows + " rows by primary key; not a random sample."); + result.getWarnings().add(scope.warning()); + if(scope.rowsScanned() == 0) + { + result.getWarnings().add("No sample rows were observed; run exact for exact numbers."); + } } if(scope.truncated()) { - result.getWarnings().add("Partial result: max_rows reached; counts and drills cover only the selected prefix."); + result.getWarnings().add(scope.estimate() ? "Partial result: max_rows reached; counts cover the selected prefix and drills cover the full population." + : "Partial result: max_rows reached; counts and drills cover only the selected prefix."); } return (result); } @@ -148,7 +172,7 @@ public class ColumnProfiler /******************************************************************************* ** Numeric moments, twenty interior bins, exterior bins and Tukey outliers. *******************************************************************************/ - private void numeric(ColumnProfile result, QQueryFilter filter, long rows) throws QException + private void numeric(ColumnProfile result, QQueryFilter filter, QQueryFilter drillFilter, long rows) throws QException { String table = result.getTableName(); String field = result.getFieldName(); @@ -158,12 +182,12 @@ public class ColumnProfiler summary.metrics().forEach((key, value) -> numeric.getMetrics().add(ProfileSupport.metric(key, value, ValueType.DECIMAL))); for(AnalysisBackend.GroupCount count : summary.counts()) { - numeric.getMetrics().add(ProfileSupport.count(count.label(), count.count(), table, filter, count.condition())); + numeric.getMetrics().add(ProfileSupport.count(count.label(), count.count(), table, drillFilter, count.condition())); } - numeric.getMetrics().add(ProfileSupport.count("outliers", summary.outlierCount(), table, filter, summary.outlierFilter())); - numeric.withHistogram(ProfileSupport.values(table, filter, null, + numeric.getMetrics().add(ProfileSupport.count("outliers", summary.outlierCount(), table, drillFilter, summary.outlierFilter())); + numeric.withHistogram(ProfileSupport.values(table, drillFilter, null, backend.numericHistogram(table, filter, field, summary.metrics().get("p1"), summary.metrics().get("p99"), 20), rows)); - numeric.withOutliers(ProfileSupport.values(table, filter, null, summary.outliers(), rows)); + numeric.withOutliers(ProfileSupport.values(table, drillFilter, null, summary.outliers(), rows)); result.withNumeric(numeric); } @@ -172,7 +196,7 @@ public class ColumnProfiler /******************************************************************************* ** Date extrema, range, calendar volume, gaps and cyclic time patterns. *******************************************************************************/ - private void dates(ColumnProfile result, QQueryFilter filter, long rows, QFieldType type) throws QException + private void dates(ColumnProfile result, QQueryFilter filter, QQueryFilter drillFilter, long rows, QFieldType type) throws QException { String table = result.getTableName(); String field = result.getFieldName(); @@ -188,12 +212,12 @@ public class ColumnProfiler AnalysisBackend.DateUnit unit = autoBucket(first, last); dates.withUnit(unit.name()); dates.getMetrics().add(ProfileSupport.metric("range", ChronoUnit.DAYS.between(first, last), ValueType.DURATION)); - List buckets = ProfileSupport.values(table, filter, null, backend.dateBuckets(table, filter, field, unit, zone), rows); + List buckets = ProfileSupport.values(table, drillFilter, null, backend.dateBuckets(table, filter, field, unit, zone), rows); dates.withBuckets(buckets).withGaps(buckets.stream().filter(b -> b.getCount() == 0).toList()); if(type == QFieldType.DATE_TIME) { - dates.withDayOfWeek(ProfileSupport.values(table, filter, null, backend.dateBuckets(table, filter, field, AnalysisBackend.DateUnit.DAY_OF_WEEK, zone), rows)) - .withHourOfDay(ProfileSupport.values(table, filter, null, backend.dateBuckets(table, filter, field, AnalysisBackend.DateUnit.HOUR_OF_DAY, zone), rows)); + dates.withDayOfWeek(ProfileSupport.values(table, drillFilter, null, backend.dateBuckets(table, filter, field, AnalysisBackend.DateUnit.DAY_OF_WEEK, zone), rows)) + .withHourOfDay(ProfileSupport.values(table, drillFilter, null, backend.dateBuckets(table, filter, field, AnalysisBackend.DateUnit.HOUR_OF_DAY, zone), rows)); } } result.withDates(dates); diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/CoreAnalysisBackend.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/CoreAnalysisBackend.java new file mode 100644 index 0000000..e5f9201 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/CoreAnalysisBackend.java @@ -0,0 +1,475 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.io.Serializable; +import java.math.BigDecimal; +import java.math.MathContext; +import java.math.RoundingMode; +import java.time.LocalDate; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateCapability; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.DateBucket; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.DateBucketUnit; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.NumericBucket; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; + +/******************************************************************************* + ** Capability-directed aggregates; streamed primitives retain portable drills. + *******************************************************************************/ +public class CoreAnalysisBackend extends StandInAnalysisBackend +{ + private BigDecimal samplePercent; + private QQueryFilter populationFilter; + private long populationRows; + private long aggregateRows; + private BigDecimal lastCountShare; + + /******************************************************************************* + ** Bind the same bounded configuration as the streamed implementation. + *******************************************************************************/ + public CoreAnalysisBackend(EffectiveConfig config) + { + super(config); + } + + /******************************************************************************* + ** Ask the actual backend implementation, including dialect-specific support. + *******************************************************************************/ + public static Set capabilities(String table) throws QException + { + AggregateInput input = new AggregateInput(table); + return (new QBackendModuleDispatcher().getQBackendModule(input.getBackend()).getAggregateInterface().getSupportedCapabilities(input)); + } + + /******************************************************************************* + ** Sampling belongs to each aggregate, never to a materialized key filter. + *******************************************************************************/ + @Override + public Scope scope(String table, QQueryFilter filter, boolean exact) throws QException + { + samplePercent = null; + long total = super.count(table, filter); + if(exact || total <= config.getSampleThresholdRows() || !capabilities(table).contains(AggregateCapability.SAMPLING)) + { + return (super.scope(table, filter, exact)); + } + estimating = true; + populationRows = total; + populationFilter = DrillFilters.copy(filter); + samplePercent = BigDecimal.valueOf(config.getSampleRows()).multiply(BigDecimal.valueOf(100)) + .divide(BigDecimal.valueOf(total), MathContext.DECIMAL128).min(BigDecimal.valueOf(100)); + long seen = sampledCount(table, populationFilter); + return (new Scope(DrillFilters.copy(filter), total, seen, false, true, samplePercent)); + } + + /******************************************************************************* + ** Count the sampled call's actual rows rather than assuming the requested size. + *******************************************************************************/ + private long sampledCount(String table, QQueryFilter filter) throws QException + { + if(samplePercent.signum() == 0) + { + return (0); + } + List result = super.aggregate(new AggregateInput(table).withFilter(DrillFilters.copy(filter)) + .withAggregate(countOf(primaryKey(table), AggregateOperator.COUNT)).withSamplePercent(samplePercent)); + return (result.isEmpty() ? 0 : number(result.getFirst(), primaryKey(table), AggregateOperator.COUNT).longValue()); + } + + /******************************************************************************* + ** Unpaginated groups supply their own denominator, including null groups. + ** Limited groups use a separate sampled population count for this call. + *******************************************************************************/ + @Override + protected List aggregate(AggregateInput input) throws QException + { + if(samplePercent == null) + { + return (super.aggregate(input)); + } + if(samplePercent.signum() == 0) + { + aggregateRows = 0; + return (List.of()); + } + Aggregate count = countOf(primaryKey(input.getTableName()), AggregateOperator.COUNT); + if(input.getAggregates() == null || !input.getAggregates().contains(count)) + { + input.withAggregate(count); + } + input.withSamplePercent(samplePercent); + boolean fullPopulation = JsonUtils.toJson(DrillFilters.copy(input.getFilter())).equals(JsonUtils.toJson(populationFilter)); + List result = super.aggregate(input); + aggregateRows = fullPopulation && input.getLimit() == null + ? result.stream().mapToLong(r -> number(r, primaryKey(input.getTableName()), AggregateOperator.COUNT).longValue()).sum() + : sampledCount(input.getTableName(), populationFilter); + return (result); + } + + /******************************************************************************* + ** Population rows remain exact; conditional counters estimate their membership. + *******************************************************************************/ + @Override + public long count(String table, QQueryFilter filter) throws QException + { + if(samplePercent == null) + { + return (super.count(table, filter)); + } + if(JsonUtils.toJson(DrillFilters.copy(filter)).equals(JsonUtils.toJson(populationFilter))) + { + lastCountShare = ProfileSupport.share(populationRows, populationRows); + return (populationRows); + } + long value = sampledCount(table, filter); + aggregateRows = sampledCount(table, populationFilter); + lastCountShare = ProfileSupport.share(value, aggregateRows); + return (aggregateCount(value)); + } + + /******************************************************************************* + ** Round count estimates only after normalizing by observed sample rows. + *******************************************************************************/ + @Override + protected long aggregateCount(long value) + { + return (samplePercent == null ? value : aggregateRows == 0 ? 0 : BigDecimal.valueOf(value).multiply(BigDecimal.valueOf(populationRows)) + .divide(BigDecimal.valueOf(aggregateRows), 0, RoundingMode.HALF_UP).longValueExact()); + } + + /******************************************************************************* + ** Keep ratios in sample units, independent of count rounding. + *******************************************************************************/ + @Override + protected BigDecimal aggregateShare(long value) + { + return (samplePercent == null ? null : ProfileSupport.share(value, aggregateRows)); + } + + /******************************************************************************* + ** Conditional groups retain the ratio from their sampled count call. + *******************************************************************************/ + @Override + protected BigDecimal countShare() + { + return (samplePercent == null ? null : lastCountShare); + } + + /******************************************************************************* + ** Streamed fallbacks also observe native sampling, without binding sampled ids. + *******************************************************************************/ + @Override + protected List rows(String table, QQueryFilter filter, Set fields, int limit) throws QException + { + if(samplePercent == null) + { + return (super.rows(table, filter, fields, limit)); + } + if(limit == 0 || config.getMaxRows() == 0) + { + return (List.of()); + } + AggregateInput input = new AggregateInput(table).withFilter(DrillFilters.copy(filter)).withGroupBy(new GroupBy(field(table, primaryKey(table)))) + .withLimit(Math.min(limit, config.getMaxRows())); + fields.stream().filter(f -> !f.equals(primaryKey(table))).forEach(f -> input.withGroupBy(new GroupBy(field(table, f)))); + List result = new ArrayList<>(); + for(AggregateResult row : aggregate(input)) + { + QRecord record = new QRecord(); + row.getGroupByValues().forEach((key, value) -> record.setValue(key.getFieldName(), value)); + result.add(record); + } + return (result); + } + + /******************************************************************************* + ** Every field batch shares a COUNT denominator from the same sampled call. + *******************************************************************************/ + @Override + public Map fieldCounts(String table, QQueryFilter filter, List fields) throws QException + { + if(samplePercent == null) + { + return (super.fieldCounts(table, filter, fields)); + } + Map result = new LinkedHashMap<>(); + for(int start = 0; start < fields.size(); start += 20) + { + List batch = fields.subList(start, Math.min(start + 20, fields.size())); + AggregateInput input = new AggregateInput(table).withFilter(DrillFilters.copy(filter)); + for(String field : batch) + { + input.withAggregate(countOf(field, AggregateOperator.COUNT)).withAggregate(countOf(field, AggregateOperator.COUNT_DISTINCT)); + } + fieldCountAggregates(input, batch); + List stringFields = batch.stream().filter(f -> field(table, f).getType().isStringLike()).toList(); + List records = stringFields.isEmpty() ? List.of() : rows(table, filter, Set.copyOf(stringFields), config.getMaxRows()); + List results = aggregate(input); + long seen = aggregateRows; + for(String field : batch) + { + boolean string = field(table, field).getType().isStringLike(); + long nonNull = results.isEmpty() ? 0 : number(results.getFirst(), field, AggregateOperator.COUNT).longValue(); + long distinct = results.isEmpty() ? 0 : number(results.getFirst(), field, AggregateOperator.COUNT_DISTINCT).longValue(); + long blanks = string && capabilities(table).contains(AggregateCapability.NULL_BLANK_COUNTS) && !results.isEmpty() + ? number(results.getFirst(), field, AggregateOperator.COUNT_BLANK).longValue() : seen - nonNull; + QQueryFilter blankFilter = DrillFilters.nulls(field, string); + if(string) + { + blankFilter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR).withSubFilter(blankFilter); + for(String blank : records.stream().map(r -> r.getValueString(field)).filter(Objects::nonNull).filter(String::isBlank).distinct().toList()) + { + blankFilter.withSubFilter(DrillFilters.condition(field, QCriteriaOperator.EQUALS, blank)); + } + } + long blankCount = aggregateCount(blanks); + BigDecimal completeness = ProfileSupport.share(seen - blanks, seen); + if(string && !capabilities(table).contains(AggregateCapability.NULL_BLANK_COUNTS)) + { + blankCount = count(table, DrillFilters.and(filter, blankFilter)); + completeness = BigDecimal.ONE.subtract(countShare()); + aggregateRows = seen; + } + result.put(field, new FieldCounts(aggregateCount(nonNull), aggregateCount(seen - nonNull), blankCount, aggregateCount(distinct), + blankFilter, DrillFilters.nonNull(field, string), ProfileSupport.share(distinct, nonNull), completeness, distinct == 1)); + } + } + return (result); + } + + /******************************************************************************* + ** Add supported counters to the existing twenty-field aggregate batch. + *******************************************************************************/ + @Override + protected void fieldCountAggregates(AggregateInput input, List fields) throws QException + { + if(capabilities(input.getTableName()).contains(AggregateCapability.NULL_BLANK_COUNTS)) + { + for(String name : fields) + { + input.withAggregate(new Aggregate(name, AggregateOperator.COUNT_NULL)); + if(field(input.getTableName(), name).getType().isStringLike()) + { + input.withAggregate(new Aggregate(name, AggregateOperator.COUNT_BLANK)); + } + } + } + } + + /******************************************************************************* + ** Read native counters, falling back to the portable predicate counter. + *******************************************************************************/ + @Override + protected long fieldCount(String table, QQueryFilter filter, String field, AggregateOperator operator, + List results, QQueryFilter condition) throws QException + { + if(capabilities(table).contains(AggregateCapability.NULL_BLANK_COUNTS)) + { + return (results.isEmpty() ? 0 : number(results.getFirst(), field, operator).longValue()); + } + return (super.fieldCount(table, filter, field, operator, results, condition)); + } + + /******************************************************************************* + ** Push each supported statistic, streaming only unavailable operators. + *******************************************************************************/ + @Override + protected Map numericMetrics(String table, QQueryFilter filter, String field, List percentiles, List values) throws QException + { + Set supported = capabilities(table); + if(!supported.containsAll(Set.of(AggregateCapability.STDDEV, AggregateCapability.PERCENTILE))) + { + return (partialNumericMetrics(table, filter, field, percentiles, values, supported)); + } + return (nativeNumericMetrics(table, filter, field, percentiles, supported)); + } + + /******************************************************************************* + ** Existing streamed arithmetic supplies only unavailable metrics. + *******************************************************************************/ + private Map partialNumericMetrics(String table, QQueryFilter filter, String field, List percentiles, + List values, Set supported) throws QException + { + Map metrics = super.numericMetrics(table, filter, field, percentiles, values); + metrics.putAll(nativeNumericMetrics(table, filter, field, percentiles, supported)); + return (metrics); + } + + /******************************************************************************* + ** One aggregate call computes basic moments and all supported extensions. + *******************************************************************************/ + private Map nativeNumericMetrics(String table, QQueryFilter filter, String field, List percentiles, + Set supported) throws QException + { + Map metrics = new LinkedHashMap<>(); + AggregateInput input = new AggregateInput(table).withFilter(DrillFilters.copy(filter)); + Map names = Map.of(AggregateOperator.MIN, "min", AggregateOperator.MAX, "max", AggregateOperator.AVG, "mean", AggregateOperator.SUM, "sum", AggregateOperator.STDDEV_SAMP, "stddev"); + for(AggregateOperator op : names.keySet()) + { + if(op != AggregateOperator.STDDEV_SAMP || supported.contains(AggregateCapability.STDDEV)) + { + input.withAggregate(new Aggregate(field, op)); + } + } + if(supported.contains(AggregateCapability.PERCENTILE)) + { + percentiles.forEach(p -> input.withAggregate(new Aggregate(field, AggregateOperator.PERCENTILE_CONT).withPercentile(p))); + } + for(AggregateResult row : aggregate(input)) + { + row.getAggregateValues().entrySet().stream().filter(e -> e.getKey().getOperator() != AggregateOperator.COUNT).forEach(entry -> + { + Aggregate key = entry.getKey(); + Serializable value = entry.getValue(); + metrics.put(key.getOperator() == AggregateOperator.PERCENTILE_CONT + ? "p" + key.getPercentile().multiply(BigDecimal.valueOf(100)).stripTrailingZeros().toPlainString() : names.get(key.getOperator()), + value == null ? null : new BigDecimal(value.toString())); + }); + } + return (metrics); + } + + /******************************************************************************* + ** Convert native bin numbers into identical half-open drill predicates. + *******************************************************************************/ + @Override + public List numericHistogram(String table, QQueryFilter filter, String field, BigDecimal low, BigDecimal high, int bins) throws QException + { + if(low == null || high == null || low.compareTo(high) >= 0 || !capabilities(table).contains(AggregateCapability.NUMERIC_BUCKET)) + { + return (super.numericHistogram(table, filter, field, low, high, bins)); + } + GroupBy group = new GroupBy(field(table, field)).withBucket(new NumericBucket(low, high, bins)); + Map counts = bucketCounts(table, filter, group); + List values = !estimating && isMemory(table) ? rows(table, filter, Set.of(field), config.getMaxRows()).stream().map(r -> r.getValue(field)) + .filter(Objects::nonNull).map(v -> new BigDecimal(v.toString())).toList() : List.of(); + List result = new ArrayList<>(); + result.add(new GroupCount("Below " + low, "Below " + low, aggregateCount(counts.getOrDefault(0, 0L)), numericRange(table, field, values, null, low, false), aggregateShare(counts.getOrDefault(0, 0L)))); + BigDecimal width = high.subtract(low).divide(BigDecimal.valueOf(bins), MathContext.DECIMAL128); + for(int i = 0; i < bins; i++) + { + BigDecimal start = low.add(width.multiply(BigDecimal.valueOf(i))); + BigDecimal end = i == bins - 1 ? high : low.add(width.multiply(BigDecimal.valueOf(i + 1))); + String label = "[" + start + ", " + end + (i == bins - 1 ? "]" : ")"); + result.add(new GroupCount(label, label, aggregateCount(counts.getOrDefault(i + 1, 0L)), numericRange(table, field, values, start, end, i == bins - 1), aggregateShare(counts.getOrDefault(i + 1, 0L)))); + } + result.add(new GroupCount("Above " + high, "Above " + high, aggregateCount(counts.getOrDefault(bins + 1, 0L)), + DrillFilters.and(DrillFilters.condition(field, QCriteriaOperator.IS_NOT_BLANK), DrillFilters.condition(field, QCriteriaOperator.GREATER_THAN, high)), aggregateShare(counts.getOrDefault(bins + 1, 0L)))); + return (result); + } + + /******************************************************************************* + ** Native dates use the session zone; fill calendar gaps with zero counts. + *******************************************************************************/ + @Override + public List dateBuckets(String table, QQueryFilter filter, String field, DateUnit unit, ZoneId zone) throws QException + { + GroupBy group = new GroupBy(field(table, field)).withBucket(new DateBucket(DateBucketUnit.valueOf(unit.name()))); + AggregateInput probe = new AggregateInput(table).withGroupBy(group); + if(!new QBackendModuleDispatcher().getQBackendModule(probe.getBackend()).getAggregateInterface().getSupportedCapabilities(probe).contains(AggregateCapability.DATE_BUCKET)) + { + return (super.dateBuckets(table, filter, field, unit, zone)); + } + Map counts = bucketCounts(table, filter, group); + if(unit == DateUnit.DAY_OF_WEEK || unit == DateUnit.HOUR_OF_DAY) + { + long seen = aggregateRows; + List conditions = super.dateBuckets(table, filter, field, unit, zone); + aggregateRows = seen; + return (conditions.stream() + .map(g -> new GroupCount(g.value(), g.label(), aggregateCount(counts.getOrDefault(g.value(), 0L)), g.condition(), aggregateShare(counts.getOrDefault(g.value(), 0L)))).toList()); + } + List dates = counts.keySet().stream().filter(Objects::nonNull).map(v -> (LocalDate) v).sorted().toList(); + List result = new ArrayList<>(); + if(!dates.isEmpty()) + { + LocalDate start = dates.getFirst(); + while(!start.isAfter(dates.getLast())) + { + LocalDate end = next(start, unit); + String label = start + " / " + end; + result.add(new GroupCount(label, label, aggregateCount(counts.getOrDefault(start, 0L)), + DrillFilters.dateRange(field, start, end, field(table, field).getType() == QFieldType.DATE_TIME, zone), aggregateShare(counts.getOrDefault(start, 0L)))); + start = end; + } + } + return (result); + } + + /******************************************************************************* + ** Length moments are native; the derived length histogram remains streamed. + *******************************************************************************/ + @Override + public LengthSummary lengthSummary(String table, QQueryFilter filter, String field) throws QException + { + LengthSummary streamed = super.lengthSummary(table, filter, field); + if(capabilities(table).contains(AggregateCapability.STRING_LENGTH)) + { + AggregateInput input = new AggregateInput(table).withFilter(DrillFilters.copy(filter)); + Map names = Map.of(AggregateOperator.MIN_LENGTH, "minLength", AggregateOperator.MAX_LENGTH, "maxLength", AggregateOperator.AVG_LENGTH, "meanLength"); + names.keySet().forEach(op -> input.withAggregate(new Aggregate(field, op))); + for(AggregateResult row : aggregate(input)) + { + row.getAggregateValues().forEach((key, value) -> + { + if(value != null && names.containsKey(key.getOperator())) + { + streamed.metrics().put(names.get(key.getOperator()), new BigDecimal(value.toString())); + } + }); + } + } + return (streamed); + } + + /******************************************************************************* + ** Bucket result keys retain core's typed date or integer values. + *******************************************************************************/ + private Map bucketCounts(String table, QQueryFilter filter, GroupBy group) throws QException + { + Map result = new LinkedHashMap<>(); + AggregateInput input = new AggregateInput(table).withFilter(DrillFilters.copy(filter)).withGroupBy(group) + .withAggregate(countOf(primaryKey(table), AggregateOperator.COUNT)); + for(AggregateResult row : aggregate(input)) + { + result.put(row.getGroupByValues().get(group), number(row, primaryKey(table), AggregateOperator.COUNT).longValue()); + } + return (result); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/DrillFilters.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/DrillFilters.java index d860959..51a81a4 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/DrillFilters.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/DrillFilters.java @@ -75,11 +75,63 @@ public final class DrillFilters *******************************************************************************/ public static Drill drill(String table, QQueryFilter original, QQueryFilter condition) { - return (new Drill().withTableName(table).withFilter(and(original, condition))); + QQueryFilter filter = and(original, condition); + var instance = com.kingsrook.qqq.backend.core.context.QContext.getQInstance(); + if(instance != null && "memory".equals(instance.getBackend(instance.getTable(table).getBackendName()).getBackendType())) + { + filter = memoryTemporalBounds(restoreTypes(table, filter)); + } + return (new Drill().withTableName(table).withFilter(filter)); } + /******************************************************************************* + ** Memory's inclusive lower bound uses Java equality, which differs after + ** HTTP turns dates into strings. Strict lower bounds avoid that equality. + ** Only memory uses this adjustment: SQL timestamp precision may be coarser. + *******************************************************************************/ + private static QQueryFilter memoryTemporalBounds(QQueryFilter filter) + { + filter.setSubFilters(new java.util.ArrayList<>(filter.getSubFilters())); + for(var criterion : new java.util.ArrayList<>(filter.getCriteria())) + { + if(criterion.getValues() != null && !criterion.getValues().isEmpty() + && criterion.getValues().stream().allMatch(v -> v instanceof java.time.Instant || v instanceof LocalDate) + && (criterion.getOperator() == QCriteriaOperator.EQUALS || criterion.getOperator() == QCriteriaOperator.IN)) + { + QQueryFilter members = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR); + for(Serializable value : criterion.getValues()) + { + members.withSubFilter(range(criterion.getFieldName(), value, value, true)); + } + filter.getCriteria().remove(criterion); + filter.withSubFilter(members); + continue; + } + if(criterion.getValues() != null && criterion.getValues().size() == 1) + { + Serializable value = criterion.getValues().getFirst(); + if(value instanceof java.time.Instant || value instanceof LocalDate) + { + Serializable previous = value instanceof java.time.Instant instant ? instant.minusNanos(1) : ((LocalDate) value).minusDays(1); + if(criterion.getOperator() == QCriteriaOperator.GREATER_THAN_OR_EQUALS) + { + criterion.setOperator(QCriteriaOperator.GREATER_THAN); + criterion.setValues(List.of(previous)); + } + else if(criterion.getOperator() == QCriteriaOperator.LESS_THAN) + { + criterion.setOperator(QCriteriaOperator.LESS_THAN_OR_EQUALS); + criterion.setValues(List.of(previous)); + } + } + } + } + filter.getSubFilters().forEach(DrillFilters::memoryTemporalBounds); + return (filter); + } + /******************************************************************************* ** One criterion, including predicates without operands. *******************************************************************************/ @@ -116,7 +168,7 @@ public final class DrillFilters QQueryFilter result = condition(field, QCriteriaOperator.IS_NOT_BLANK); if(string) { - result.withBooleanOperator(QQueryFilter.BooleanOperator.OR).withCriteria(field, QCriteriaOperator.EQUALS, ""); + result.withBooleanOperator(QQueryFilter.BooleanOperator.OR).withSubFilter(emptyString(field)); } return (result); } @@ -128,7 +180,8 @@ public final class DrillFilters *******************************************************************************/ public static QQueryFilter value(String field, Serializable value, boolean string) { - return (value == null ? nulls(field, string) : condition(field, QCriteriaOperator.EQUALS, value)); + return (value == null ? nulls(field, string) : "".equals(value) ? emptyString(field) + : value instanceof java.time.Instant || value instanceof LocalDate ? range(field, value, value, true) : condition(field, QCriteriaOperator.EQUALS, value)); } @@ -181,11 +234,33 @@ public final class DrillFilters *******************************************************************************/ public static QQueryFilter in(String field, List values) { + if(values.stream().anyMatch(v -> v instanceof java.time.Instant || v instanceof LocalDate)) + { + QQueryFilter result = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR); + values.forEach(value -> result.withSubFilter(range(field, value, value, true))); + return (result); + } + if(values.contains("")) + { + List nonEmpty = values.stream().filter(v -> !"".equals(v)).toList(); + return (new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR).withSubFilter(emptyString(field)).withSubFilter(in(field, nonEmpty))); + } return (values.isEmpty() ? condition(field, QCriteriaOperator.FALSE) : new QQueryFilter().withCriteria(field, QCriteriaOperator.IN, values)); } + /******************************************************************************* + ** Core HTTP interpretation treats empty string operands as missing inputs. + ** A zero-length predicate expresses the same set without an empty operand. + *******************************************************************************/ + private static QQueryFilter emptyString(String field) + { + return (new QQueryFilter().withCriteria(new com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria(field, QCriteriaOperator.EQUALS, 0) + .withFieldFunction(new com.kingsrook.qqq.backend.core.model.metadata.fields.functions.FieldFunction().withFieldName(field) + .withFunctionTypeIdentifier(com.kingsrook.qqq.backend.core.model.metadata.fields.functions.implementations.StringLengthFunction.IDENTIFIER)))); + } + /******************************************************************************* ** JSON has no temporal scalar types. Restore them from current metadata before ** a cached drill is handed back to core, without mutating the stored filter. diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/DuplicateFinder.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/DuplicateFinder.java index e8fe3e1..fb251c0 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/DuplicateFinder.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/DuplicateFinder.java @@ -51,6 +51,6 @@ public class DuplicateFinder return (new DuplicatesResult().withTableName(request.getTableName()).withFields(request.getFields()).withFilter(DrillFilters.copy(request.getFilter())) .withRowsScanned(scope.rowsScanned()).withTruncated(scope.truncated()).withAsOf(Instant.now()) .withGroups(new MinGroupSizeGuard(config.getMinGroupSize()).groups(ProfileSupport.values(request.getTableName(), scope.filter(), null, - backend.duplicates(request.getTableName(), scope.filter(), request.getFields(), 50), scope.rowsScanned())))); + backend.duplicates(request.getTableName(), scope.filter(), request.getFields(), request.getLimit()), scope.rowsScanned())))); } } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ProfileSupport.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ProfileSupport.java index 0092e3f..83c8b59 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ProfileSupport.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ProfileSupport.java @@ -81,6 +81,24 @@ public final class ProfileSupport + /******************************************************************************* + ** The population row total is exact even when other metrics are estimates. + *******************************************************************************/ + public static void markEstimates(List metrics) + { + if(metrics != null) + { + metrics.stream().filter(m -> !"rows".equals(m.getKey())).forEach(m -> + { + m.withEstimate(true); + if(m.getFormatted() != null) + { + m.withFormatted("≈ " + m.getFormatted()); + } + }); + } + } + /******************************************************************************* ** PVS translations run as the current user; drill values always remain ids. *******************************************************************************/ @@ -98,7 +116,7 @@ public final class ProfileSupport label = translated; } } - result.add(new ValueCount().withValue(group.value()).withLabel(label).withCount(group.count()).withShare(share(group.count(), total)) + result.add(new ValueCount().withValue(group.value()).withLabel(label).withCount(group.count()).withShare(group.share() == null ? share(group.count(), total) : group.share()) .withDrill(DrillFilters.drill(table, filter, group.condition()))); } return (result); diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/StandInAnalysisBackend.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/StandInAnalysisBackend.java index 121d9bd..a8ee186 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/StandInAnalysisBackend.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/StandInAnalysisBackend.java @@ -68,7 +68,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; public class StandInAnalysisBackend implements AnalysisBackend { private static final MathContext MATH = MathContext.DECIMAL128; - private final EffectiveConfig config; + protected final EffectiveConfig config; + protected boolean estimating; /******************************************************************************* ** Bind one request's detached configuration. @@ -94,13 +95,14 @@ public class StandInAnalysisBackend implements AnalysisBackend /******************************************************************************* - ** A boundary query selects the prefix retained in every drill. + ** A single boundary parameter bounds the prefix used for computation. *******************************************************************************/ @Override public Scope scope(String table, QQueryFilter filter, boolean exact) throws QException { long total = count(table, filter); boolean estimate = !exact && total > config.getSampleThresholdRows(); + estimating = estimate; int limit = estimate ? Math.min(config.getSampleRows(), config.getMaxRows()) : config.getMaxRows(); QQueryFilter scoped = DrillFilters.copy(filter); if(total > limit) @@ -108,7 +110,7 @@ public class StandInAnalysisBackend implements AnalysisBackend String pk = primaryKey(table); if(limit == 0) { - scoped = DrillFilters.and(scoped, DrillFilters.in(pk, List.of())); + scoped = DrillFilters.and(scoped, DrillFilters.condition(pk, QCriteriaOperator.IS_BLANK)); } else { @@ -141,7 +143,7 @@ public class StandInAnalysisBackend implements AnalysisBackend /******************************************************************************* ** Bound both page size and retained records. *******************************************************************************/ - private List rows(String table, QQueryFilter filter, Set fields, int limit) throws QException + protected List rows(String table, QQueryFilter filter, Set fields, int limit) throws QException { List records = new ArrayList<>(); int bound = Math.min(limit, config.getMaxRows()); @@ -163,7 +165,7 @@ public class StandInAnalysisBackend implements AnalysisBackend /******************************************************************************* ** Core applies table permissions, field locks and record locks. *******************************************************************************/ - private List aggregate(AggregateInput input) throws QException + protected List aggregate(AggregateInput input) throws QException { QQueryFilter restored = DrillFilters.restoreTypes(input.getTableName(), input.getFilter()); restored.setOrderBys(input.getFilter().getOrderBys()); @@ -191,6 +193,7 @@ public class StandInAnalysisBackend implements AnalysisBackend { input.withAggregate(countOf(field, AggregateOperator.COUNT)).withAggregate(countOf(field, AggregateOperator.COUNT_DISTINCT)); } + fieldCountAggregates(input, batch); List aggregates = aggregate(input); Set selection = new LinkedHashSet<>(batch); selection.add(primaryKey(table)); @@ -198,7 +201,7 @@ public class StandInAnalysisBackend implements AnalysisBackend for(String field : batch) { boolean string = field(table, field).getType().isStringLike(); - long nulls = count(table, DrillFilters.and(filter, DrillFilters.nulls(field, string))); + long nulls = fieldCount(table, filter, field, AggregateOperator.COUNT_NULL, aggregates, DrillFilters.nulls(field, string)); Map representatives = new LinkedHashMap<>(); Set blanks = new LinkedHashSet<>(); for(QRecord record : records) @@ -220,12 +223,12 @@ public class StandInAnalysisBackend implements AnalysisBackend .withSubFilter(DrillFilters.in(field, new ArrayList<>(blanks))); } long distinct = aggregates.isEmpty() ? 0 : number(aggregates.getFirst(), field, AggregateOperator.COUNT_DISTINCT).longValue(); - if(isMemory(table) && field(table, field).getType() == QFieldType.DECIMAL) + if(!estimating && isMemory(table) && field(table, field).getType() == QFieldType.DECIMAL) { distinct = representatives.size(); } - result.put(field, new FieldCounts(total - nulls, nulls, string ? count(table, DrillFilters.and(filter, blankFilter)) : nulls, - distinct, blankFilter, DrillFilters.in(primaryKey(table), new ArrayList<>(representatives.values())))); + result.put(field, new FieldCounts(total - nulls, nulls, string ? fieldCount(table, filter, field, AggregateOperator.COUNT_BLANK, aggregates, blankFilter) : nulls, + distinct, blankFilter, estimating ? DrillFilters.nonNull(field, string) : DrillFilters.in(primaryKey(table), new ArrayList<>(representatives.values())))); } } return (result); @@ -233,6 +236,23 @@ public class StandInAnalysisBackend implements AnalysisBackend + /******************************************************************************* + ** Subclasses may extend the existing batch with native null/blank counts. + *******************************************************************************/ + protected void fieldCountAggregates(AggregateInput input, List fields) throws QException + { + // The portable backend counts null and blank predicates separately. + } + + /******************************************************************************* + ** Count the exact drill predicate when native counters are unavailable. + *******************************************************************************/ + protected long fieldCount(String table, QQueryFilter filter, String field, AggregateOperator operator, + List results, QQueryFilter condition) throws QException + { + return (count(table, DrillFilters.and(filter, condition))); + } + /******************************************************************************* ** Push down group, count, ordering and pagination. *******************************************************************************/ @@ -254,6 +274,30 @@ public class StandInAnalysisBackend implements AnalysisBackend + /******************************************************************************* + ** Random backends scale a count using the current aggregate's sample size. + *******************************************************************************/ + protected long aggregateCount(long value) + { + return (value); + } + + /******************************************************************************* + ** Random backends retain the raw sample ratio before rounding counts. + *******************************************************************************/ + protected BigDecimal aggregateShare(long value) + { + return (null); + } + + /******************************************************************************* + ** Exact and prefix groups derive shares from the profiler's row count. + *******************************************************************************/ + protected BigDecimal countShare() + { + return (null); + } + /******************************************************************************* ** Tuple drills use IN for non-null values and null-only predicates otherwise. *******************************************************************************/ @@ -270,7 +314,8 @@ public class StandInAnalysisBackend implements AnalysisBackend Map merged = new LinkedHashMap<>(); for(AggregateResult row : aggregate(input)) { - long size = number(row, primaryKey(table), AggregateOperator.COUNT).longValue(); + long rawSize = number(row, primaryKey(table), AggregateOperator.COUNT).longValue(); + long size = aggregateCount(rawSize); LinkedHashMap tuple = new LinkedHashMap<>(); QQueryFilter condition = new QQueryFilter(); for(String field : fields) @@ -278,7 +323,7 @@ public class StandInAnalysisBackend implements AnalysisBackend Serializable value = row.getGroupByValues().entrySet().stream().filter(e -> field.equals(e.getKey().getFieldName())).findFirst().orElseThrow().getValue(); tuple.put(field, value instanceof BigDecimal d ? new BigDecimal(d.stripTrailingZeros().toPlainString()) : value); condition.withSubFilter(value == null ? DrillFilters.nulls(field, field(table, field).getType().isStringLike()) - : value instanceof BigDecimal ? DrillFilters.in(field, List.of(value.toString())) + : value instanceof BigDecimal && !estimating ? DrillFilters.in(field, List.of(value.toString())) : duplicates ? DrillFilters.in(field, List.of(value)) : DrillFilters.value(field, value, field(table, field).getType().isStringLike())); } Serializable value = duplicates ? tuple : tuple.get(fields.getFirst()); @@ -288,7 +333,7 @@ public class StandInAnalysisBackend implements AnalysisBackend size += previous.count(); condition = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR).withSubFilter(previous.condition()).withSubFilter(condition); } - merged.put(value, new GroupCount(value, value == null ? "(null)" : value.toString(), size, condition)); + merged.put(value, new GroupCount(value, value == null ? "(null)" : value.toString(), size, condition, aggregateShare(rawSize))); } List groups = new ArrayList<>(merged.values()); if(isMemory(table)) @@ -308,14 +353,8 @@ public class StandInAnalysisBackend implements AnalysisBackend { List values = rows(table, filter, Set.of(field), config.getMaxRows()).stream().map(r -> r.getValue(field)) .filter(Objects::nonNull).map(v -> new BigDecimal(v.toString())).sorted().toList(); - Map metrics = new LinkedHashMap<>(); - AggregateInput input = new AggregateInput(table).withFilter(DrillFilters.copy(filter)); - for(AggregateOperator op : List.of(AggregateOperator.MIN, AggregateOperator.MAX, AggregateOperator.AVG, AggregateOperator.SUM)) - { - input.withAggregate(new Aggregate(field, op)); - } - List results = aggregate(input); - if(values.isEmpty()) + Map metrics = numericMetrics(table, filter, field, percentiles, values); + if(metrics.isEmpty() || metrics.get("min") == null) { for(String key : List.of("min", "max", "mean", "stddev", "sum")) { @@ -325,23 +364,12 @@ public class StandInAnalysisBackend implements AnalysisBackend { metrics.put("p" + p.multiply(BigDecimal.valueOf(100)).stripTrailingZeros().toPlainString(), null); } - QQueryFilter empty = DrillFilters.in(primaryKey(table), List.of()); - return (new NumericSummary(metrics, List.of(), 0, empty, List.of(new GroupCount("zeros", "zeros", 0, empty), new GroupCount("negatives", "negatives", 0, empty)))); + QQueryFilter empty = DrillFilters.condition(primaryKey(table), QCriteriaOperator.IS_BLANK); + return (new NumericSummary(metrics, List.of(), 0, empty, List.of(new GroupCount("zeros", "zeros", 0, estimating ? DrillFilters.condition(field, QCriteriaOperator.EQUALS, BigDecimal.ZERO) : empty), + new GroupCount("negatives", "negatives", 0, estimating ? DrillFilters.range(field, null, BigDecimal.ZERO, false) : empty)))); } - AggregateResult result = results.getFirst(); - metrics.put("min", number(result, field, AggregateOperator.MIN)); - metrics.put("max", number(result, field, AggregateOperator.MAX)); - metrics.put("mean", number(result, field, AggregateOperator.AVG)); - metrics.put("sum", number(result, field, AggregateOperator.SUM)); - BigDecimal mean = values.stream().reduce(BigDecimal.ZERO, BigDecimal::add).divide(BigDecimal.valueOf(values.size()), MATH); - BigDecimal variance = values.stream().map(v -> v.subtract(mean).pow(2)).reduce(BigDecimal.ZERO, BigDecimal::add); - metrics.put("stddev", values.size() < 2 ? null : variance.divide(BigDecimal.valueOf(values.size() - 1), MATH).sqrt(MATH)); - for(BigDecimal p : percentiles) - { - metrics.put("p" + p.multiply(BigDecimal.valueOf(100)).stripTrailingZeros().toPlainString(), percentile(values, p)); - } - BigDecimal q1 = percentile(values, new BigDecimal("0.25")); - BigDecimal q3 = percentile(values, new BigDecimal("0.75")); + BigDecimal q1 = metrics.containsKey("p25") ? metrics.get("p25") : percentile(values, new BigDecimal("0.25")); + BigDecimal q3 = metrics.containsKey("p75") ? metrics.get("p75") : percentile(values, new BigDecimal("0.75")); BigDecimal span = q3.subtract(q1).multiply(new BigDecimal("1.5")); BigDecimal low = q1.subtract(span); BigDecimal high = q3.add(span); @@ -355,10 +383,10 @@ public class StandInAnalysisBackend implements AnalysisBackend List extremeGroups = new ArrayList<>(); for(BigDecimal value : extremes) { - QQueryFilter condition = DrillFilters.in(field, equalNumbers(values, value)); - extremeGroups.add(new GroupCount(value, value.toPlainString(), count(table, DrillFilters.and(filter, condition)), condition)); + QQueryFilter condition = estimating ? DrillFilters.condition(field, QCriteriaOperator.EQUALS, value) : DrillFilters.in(field, equalNumbers(values, value)); + extremeGroups.add(new GroupCount(value, value.toPlainString(), count(table, DrillFilters.and(filter, condition)), condition, countShare())); } - QQueryFilter zeros = DrillFilters.in(field, equalNumbers(values, BigDecimal.ZERO)); + QQueryFilter zeros = estimating ? DrillFilters.condition(field, QCriteriaOperator.EQUALS, BigDecimal.ZERO) : DrillFilters.in(field, equalNumbers(values, BigDecimal.ZERO)); QQueryFilter negatives = numericRange(table, field, values, null, BigDecimal.ZERO, false); return (new NumericSummary(metrics, extremeGroups, count(table, DrillFilters.and(filter, outliers)), outliers, List.of(bucket(table, filter, "zeros", zeros), bucket(table, filter, "negatives", negatives)))); @@ -366,6 +394,39 @@ public class StandInAnalysisBackend implements AnalysisBackend + /******************************************************************************* + ** Portable numeric moments; core-backed subclasses replace supported work. + *******************************************************************************/ + protected Map numericMetrics(String table, QQueryFilter filter, String field, List percentiles, List values) throws QException + { + Map metrics = new LinkedHashMap<>(); + if(values.isEmpty()) + { + return (metrics); + } + AggregateInput input = new AggregateInput(table).withFilter(DrillFilters.copy(filter)); + for(AggregateOperator op : List.of(AggregateOperator.MIN, AggregateOperator.MAX, AggregateOperator.AVG, AggregateOperator.SUM)) + { + input.withAggregate(new Aggregate(field, op)); + } + List results = aggregate(input); + AggregateResult result = results.getFirst(); + metrics.put("min", number(result, field, AggregateOperator.MIN)); + metrics.put("max", number(result, field, AggregateOperator.MAX)); + metrics.put("mean", number(result, field, AggregateOperator.AVG)); + metrics.put("sum", number(result, field, AggregateOperator.SUM)); + BigDecimal mean = values.stream().reduce(BigDecimal.ZERO, BigDecimal::add).divide(BigDecimal.valueOf(values.size()), MATH); + BigDecimal variance = values.stream().map(v -> v.subtract(mean).pow(2)).reduce(BigDecimal.ZERO, BigDecimal::add); + metrics.put("stddev", values.size() < 2 ? null : variance.divide(BigDecimal.valueOf(values.size() - 1), MATH).sqrt(MATH)); + for(BigDecimal p : percentiles) + { + metrics.put("p" + p.multiply(BigDecimal.valueOf(100)).stripTrailingZeros().toPlainString(), percentile(values, p)); + } + return (metrics); + } + + + /******************************************************************************* ** PostgreSQL percentile_cont interpolation at p * (n - 1). *******************************************************************************/ @@ -415,7 +476,7 @@ public class StandInAnalysisBackend implements AnalysisBackend *******************************************************************************/ private GroupCount bucket(String table, QQueryFilter filter, String label, QQueryFilter condition) throws QException { - return (new GroupCount(label, label, count(table, DrillFilters.and(filter, condition)), condition)); + return (new GroupCount(label, label, count(table, DrillFilters.and(filter, condition)), condition, countShare())); } @@ -464,7 +525,7 @@ public class StandInAnalysisBackend implements AnalysisBackend /******************************************************************************* ** ISO weeks begin Monday. *******************************************************************************/ - private static LocalDate floor(LocalDate date, DateUnit unit) + protected static LocalDate floor(LocalDate date, DateUnit unit) { return (switch(unit) { @@ -480,7 +541,7 @@ public class StandInAnalysisBackend implements AnalysisBackend /******************************************************************************* ** Calendar arithmetic handles leap years and daylight savings. *******************************************************************************/ - private static LocalDate next(LocalDate date, DateUnit unit) + protected static LocalDate next(LocalDate date, DateUnit unit) { return (switch(unit) { @@ -506,7 +567,13 @@ public class StandInAnalysisBackend implements AnalysisBackend result.put("max", null); for(AggregateResult row : aggregate(input)) { - row.getAggregateValues().forEach((key, value) -> result.put(key.getOperator().name().toLowerCase(java.util.Locale.ROOT), value)); + row.getAggregateValues().forEach((key, value) -> + { + if(key.getOperator() == AggregateOperator.MIN || key.getOperator() == AggregateOperator.MAX) + { + result.put(key.getOperator().name().toLowerCase(java.util.Locale.ROOT), value); + } + }); } return (result); } @@ -543,7 +610,7 @@ public class StandInAnalysisBackend implements AnalysisBackend List records = rows(table, filter, Set.of(field, primaryKey(table)), sampleRows); QQueryFilter sampled = records.isEmpty() ? DrillFilters.in(primaryKey(table), List.of()) : DrillFilters.condition(primaryKey(table), QCriteriaOperator.LESS_THAN_OR_EQUALS, records.getLast().getValue(primaryKey(table))); - return (mappedGroups(table, DrillFilters.and(filter, sampled), field, records, v -> shape(v.toString()), 10)); + return (mappedGroups(table, estimating ? filter : DrillFilters.and(filter, sampled), field, records, v -> shape(v.toString()), 10)); } @@ -575,8 +642,8 @@ public class StandInAnalysisBackend implements AnalysisBackend List groups = new ArrayList<>(); for(Map.Entry> entry : members.entrySet()) { - QQueryFilter condition = DrillFilters.and(filter, DrillFilters.in(field, new ArrayList<>(entry.getValue()))); - groups.add(new GroupCount(entry.getKey(), entry.getKey().toString(), count(table, condition), condition)); + QQueryFilter condition = DrillFilters.in(field, new ArrayList<>(entry.getValue())); + groups.add(new GroupCount(entry.getKey(), entry.getKey().toString(), count(table, DrillFilters.and(filter, condition)), estimating ? condition : DrillFilters.and(filter, condition), countShare())); } return (groups.stream().sorted(Comparator.comparingLong(GroupCount::count).reversed().thenComparing(GroupCount::label)).limit(limit).toList()); } @@ -597,7 +664,7 @@ public class StandInAnalysisBackend implements AnalysisBackend /******************************************************************************* ** Metadata lookup follows facade security checks. *******************************************************************************/ - private static QFieldMetaData field(String table, String field) + protected static QFieldMetaData field(String table, String field) { return (QContext.getQInstance().getTable(table).getField(field)); } @@ -607,7 +674,7 @@ public class StandInAnalysisBackend implements AnalysisBackend /******************************************************************************* ** Deterministic paging requires an addressable primary key. *******************************************************************************/ - private static String primaryKey(String table) + protected static String primaryKey(String table) { return (QContext.getQInstance().getTable(table).getPrimaryKeyField()); } @@ -618,7 +685,7 @@ public class StandInAnalysisBackend implements AnalysisBackend ** COUNT and COUNT_DISTINCT always read as integers: RC.5's RDBMS reader ** otherwise reads them as the field's type (a boolean count fails). *******************************************************************************/ - private static Aggregate countOf(String field, AggregateOperator op) + protected static Aggregate countOf(String field, AggregateOperator op) { return (new Aggregate(field, op).withFieldType(QFieldType.INTEGER)); } @@ -628,7 +695,7 @@ public class StandInAnalysisBackend implements AnalysisBackend /******************************************************************************* ** The value of one aggregate of a result row (0 when absent). *******************************************************************************/ - private static BigDecimal number(AggregateResult row, String field, AggregateOperator op) + protected static BigDecimal number(AggregateResult row, String field, AggregateOperator op) { Serializable value = row.getAggregateValues().entrySet().stream().filter(e -> field.equals(e.getKey().getFieldName()) && op == e.getKey().getOperator()) .map(Map.Entry::getValue).filter(Objects::nonNull).findFirst().orElse(0); @@ -642,9 +709,9 @@ public class StandInAnalysisBackend implements AnalysisBackend ** Use strict comparison plus actual boundary spellings to preserve half-open ** membership. SQL backends use ordinary range predicates. *******************************************************************************/ - private static QQueryFilter numericRange(String table, String field, List values, BigDecimal low, BigDecimal high, boolean closed) + protected QQueryFilter numericRange(String table, String field, List values, BigDecimal low, BigDecimal high, boolean closed) { - if(!isMemory(table)) + if(estimating || !isMemory(table)) { return (DrillFilters.range(field, low, high, closed)); } @@ -682,7 +749,7 @@ public class StandInAnalysisBackend implements AnalysisBackend /******************************************************************************* ** Isolate compatibility fixes to the RC.5 memory implementation. *******************************************************************************/ - private static boolean isMemory(String table) + protected static boolean isMemory(String table) { return ("memory".equals(QContext.getQInstance().getBackend(QContext.getQInstance().getTable(table).getBackendName()).getBackendType())); } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/TableProfiler.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/TableProfiler.java index fd55ba8..2560174 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/TableProfiler.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/TableProfiler.java @@ -60,22 +60,23 @@ public class TableProfiler String table = request.getTableName(); QQueryFilter filter = scope.filter(); long rows = backend.count(table, filter); + QQueryFilter drillFilter = scope.estimate() ? request.getFilter() : filter; TableProfile result = new TableProfile().withTableName(table).withFilter(DrillFilters.restoreTypes(request.getTableName(), request.getFilter())).withEstimate(scope.estimate()) - .withSampleRows(scope.estimate() ? rows : 0L).withRowsScanned(scope.rowsScanned()).withTruncated(scope.truncated()).withAsOf(Instant.now()) - .withRowCount(ProfileSupport.count("rows", rows, table, filter, null)); + .withSampleRows(scope.estimate() ? scope.rowsScanned() : 0L).withRowsScanned(scope.rowsScanned()).withTruncated(scope.truncated()).withAsOf(Instant.now()) + .withRowCount(ProfileSupport.count("rows", scope.estimate() ? scope.total() : rows, table, drillFilter, null)); Map counts = backend.fieldCounts(table, filter, fields.stream().map(QFieldMetaData::getName).toList()); for(QFieldMetaData field : fields) { String name = field.getName(); AnalysisBackend.FieldCounts c = counts.get(name); result.getFields().add(new FieldSummary().withField(name).withLabel(field.getLabel()).withType(field.getType().name()) - .withCompleteness(ProfileSupport.share(rows - c.blanks(), rows)).withDistinct(c.distinct()).withUniqueness(ProfileSupport.share(c.distinct(), c.nonNull())) - .withConstant(c.distinct() == 1).withEmpty(c.blanks() == rows) - .withNulls(ProfileSupport.count("nulls", c.nulls(), table, filter, DrillFilters.nulls(name, field.getType().isStringLike()))) - .withBlanks(ProfileSupport.count("blanks", c.blanks(), table, filter, c.blankFilter()))); + .withCompleteness(c.completeness()).withDistinct(c.distinct()).withUniqueness(c.uniqueness()) + .withConstant(c.constant()).withEmpty(c.blanks() == rows) + .withNulls(ProfileSupport.count("nulls", c.nulls(), table, drillFilter, DrillFilters.nulls(name, field.getType().isStringLike()))) + .withBlanks(ProfileSupport.count("blanks", c.blanks(), table, drillFilter, c.blankFilter()))); if(rows > 0 && c.distinct() == rows && c.nulls() == 0) { - result.getCandidateKeys().add(key(table, filter, List.of(name), rows)); + result.getCandidateKeys().add(key(table, drillFilter, List.of(name), rows)); } } List ranked = result.getFields().stream().sorted(Comparator.comparing(FieldSummary::getUniqueness).reversed().thenComparing(FieldSummary::getField)).limit(8).toList(); @@ -86,7 +87,7 @@ public class TableProfiler List pair = List.of(ranked.get(i).getField(), ranked.get(j).getField()); if(rows > 0 && pair.stream().allMatch(f -> counts.get(f).nulls() == 0) && backend.duplicates(table, filter, pair, 1).isEmpty()) { - result.getCandidateKeys().add(key(table, filter, pair, rows)); + result.getCandidateKeys().add(key(table, drillFilter, pair, rows)); } } } @@ -97,7 +98,12 @@ public class TableProfiler { result.getHeatmap().getSeries().getFirst().getPoints().add(new Point().withX(field.getField()).withLabel(field.getLabel()).withY(field.getCompleteness())); } - if(scope.estimate() || scope.truncated()) + if(scope.estimate()) + { + result.getFields().forEach(f -> ProfileSupport.markEstimates(List.of(f.getNulls(), f.getBlanks()))); + result.getWarnings().add(scope.warning()); + } + else if(scope.truncated()) { result.getWarnings().add("Profiles and candidate keys describe only the first " + rows + " rows by primary key."); } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/permissions/DataAnalysisPermissions.java b/src/main/java/com/kingsrook/qbits/dataanalysis/permissions/DataAnalysisPermissions.java index d3a5c31..5bda962 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/permissions/DataAnalysisPermissions.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/permissions/DataAnalysisPermissions.java @@ -27,12 +27,11 @@ import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* - ** Data Analysis permissions and the temporary per-table feature check. + ** Data Analysis permissions through the core table feature registry. *******************************************************************************/ public final class DataAnalysisPermissions { @@ -68,9 +67,8 @@ public final class DataAnalysisPermissions *******************************************************************************/ public static boolean canAnalyze(QTableMetaData table) { - QSession session = QContext.getQSession(); boolean read = !Boolean.TRUE.equals(QContext.getQInstance().getEnforcePermissionsInCoreActions()) || PermissionsHelper.hasTablePermission(new QueryInput(table.getName()), table.getName(), TablePermissionSubType.READ); - return (read && session != null && session.hasPermission(analyzePermissionName(table))); + return (read && PermissionsHelper.hasTableFeaturePermission(QContext.getQInstance(), QContext.getQSession(), table, "analyze")); } } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/DuplicatesRequest.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/DuplicatesRequest.java index c1bae39..d1b0172 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/result/DuplicatesRequest.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/DuplicatesRequest.java @@ -36,6 +36,7 @@ public class DuplicatesRequest implements AnalysisRequest private QQueryFilter filter; private Boolean exact = false; private List fields; + private Integer limit = 50; @@ -120,4 +121,21 @@ public class DuplicatesRequest implements AnalysisRequest this.fields = fields; return (this); } + + /******************************************************************************* + ** Maximum duplicate groups requested. + *******************************************************************************/ + public Integer getLimit() + { + return (limit); + } + + /******************************************************************************* + ** Fluent duplicate group limit. + *******************************************************************************/ + public DuplicatesRequest withLimit(Integer limit) + { + this.limit = limit; + return (this); + } } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/Metric.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/Metric.java index 456f77c..b82fefa 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/result/Metric.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/Metric.java @@ -31,6 +31,7 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; @JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) public class Metric implements Serializable { + private Boolean estimate = false; private String key; private String label; private Serializable value; @@ -42,6 +43,23 @@ public class Metric implements Serializable + /******************************************************************************* + ** Whether the metric describes a sampled estimate. + *******************************************************************************/ + public Boolean getEstimate() + { + return (estimate); + } + + /******************************************************************************* + ** Mark an estimated metric independently of the exact population row total. + *******************************************************************************/ + public Metric withEstimate(Boolean estimate) + { + this.estimate = estimate; + return (this); + } + /******************************************************************************* ** Getter for key. *******************************************************************************/ diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/run/AnalysisRunService.java b/src/main/java/com/kingsrook/qbits/dataanalysis/run/AnalysisRunService.java index ddecca4..f9abd27 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/run/AnalysisRunService.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/run/AnalysisRunService.java @@ -96,7 +96,7 @@ public class AnalysisRunService AnalysisBackend.Scope scope = backend.scope(request.getTableName(), request.getFilter(), Boolean.TRUE.equals(request.getExact())); if(scope.estimate()) { - return (service.compute(request, backend, scope).withDataVersion(DataVersion.of(backend, request.getTableName(), request.getFilter()))); + return (service.compute(request, backend, scope).withDataVersion(DataVersion.of(service.backend(request.getTableName()), request.getTableName(), request.getFilter()))); } return (exact(request).getColumnProfile()); } @@ -113,7 +113,7 @@ public class AnalysisRunService AnalysisBackend.Scope scope = backend.scope(request.getTableName(), request.getFilter(), Boolean.TRUE.equals(request.getExact())); if(scope.estimate()) { - return (service.compute(request, backend, scope).withDataVersion(DataVersion.of(backend, request.getTableName(), request.getFilter()))); + return (service.compute(request, backend, scope).withDataVersion(DataVersion.of(service.backend(request.getTableName()), request.getTableName(), request.getFilter()))); } return (exact(request).getTableProfile()); } diff --git a/src/main/resources/openapi/qqq-v1/data-analysis.yaml b/src/main/resources/openapi/qqq-v1/data-analysis.yaml new file mode 100644 index 0000000..7f0b3df --- /dev/null +++ b/src/main/resources/openapi/qqq-v1/data-analysis.yaml @@ -0,0 +1,1150 @@ +components: + schemas: + DataAnalysisAxisV1: + nullable: true + properties: + field: + description: "Field." + nullable: true + type: "string" + label: + description: "Label." + nullable: true + type: "string" + valueType: + description: "Value type." + enum: + - "COUNT" + - "DECIMAL" + - "PERCENT" + - "DATE" + - "DATE_TIME" + - "DURATION" + - "TEXT" + nullable: true + type: "string" + type: "object" + DataAnalysisCandidateKeyV1: + nullable: true + properties: + fields: + description: "Readable fields to include; omit for all analyzable fields\ + \ on a table profile." + items: + type: "string" + nullable: true + type: "array" + rowCount: + description: "Row count." + nullable: true + type: "integer" + drill: + $ref: "#/components/schemas/DataAnalysisDrillV1" + description: "Open the matching records using this table and QueryFilter;\ + \ null when suppressed." + nullable: true + type: "object" + DataAnalysisCapabilitiesV1: + nullable: true + properties: + canAnalyze: + description: "Whether this session may analyze the table; false responses\ + \ expose no fields." + nullable: true + type: "boolean" + tableName: + description: "Table name." + nullable: true + type: "string" + tableLabel: + description: "Table label." + nullable: true + type: "string" + fields: + description: "Readable fields to include; omit for all analyzable fields\ + \ on a table profile." + items: + $ref: "#/components/schemas/DataAnalysisCapabilityFieldV1" + nullable: true + type: "array" + limits: + $ref: "#/components/schemas/DataAnalysisLimitsV1" + description: "Limits." + nullable: true + backendCapabilities: + description: "Backend capabilities." + items: + type: "string" + nullable: true + type: "array" + sqlWorkbenchAvailable: + description: "Whether SQL Workbench metadata is installed in this instance." + nullable: true + type: "boolean" + type: "object" + DataAnalysisCapabilityFieldV1: + nullable: true + properties: + name: + description: "Name." + nullable: true + type: "string" + label: + description: "Label." + nullable: true + type: "string" + type: + description: "Type." + nullable: true + type: "string" + kind: + description: "Semantic field kind: NUMERIC, DATE, DATE_TIME, TEXT, BOOLEAN\ + \ or CATEGORY." + nullable: true + type: "string" + defaultChartKind: + description: "Default chart kind." + enum: + - "HISTOGRAM" + - "BOX" + - "BAR" + - "PARETO" + - "TIME_SERIES" + - "SCATTER" + - "HEATMAP" + nullable: true + type: "string" + type: "object" + DataAnalysisChartV1: + nullable: true + properties: + kind: + description: "Semantic field kind: NUMERIC, DATE, DATE_TIME, TEXT, BOOLEAN\ + \ or CATEGORY." + enum: + - "HISTOGRAM" + - "BOX" + - "BAR" + - "PARETO" + - "TIME_SERIES" + - "SCATTER" + - "HEATMAP" + nullable: true + type: "string" + alternatives: + description: "Alternatives." + items: + enum: + - "HISTOGRAM" + - "BOX" + - "BAR" + - "PARETO" + - "TIME_SERIES" + - "SCATTER" + - "HEATMAP" + type: "string" + nullable: true + type: "array" + x: + $ref: "#/components/schemas/DataAnalysisAxisV1" + description: "X." + nullable: true + "y": + $ref: "#/components/schemas/DataAnalysisAxisV1" + description: "Y." + nullable: true + series: + description: "Series." + items: + $ref: "#/components/schemas/DataAnalysisSeriesV1" + nullable: true + type: "array" + annotations: + description: "Annotations." + items: + type: "string" + nullable: true + type: "array" + type: "object" + DataAnalysisColumnProfileRequestV1: + additionalProperties: false + properties: + filter: + $ref: "#/components/schemas/QueryFilter" + description: "Core QueryFilter JSON, identical to the table query and count\ + \ endpoints." + exact: + description: "Request exact computation instead of the configured estimate." + type: "boolean" + fieldName: + description: "Readable field name to profile or enumerate." + type: "string" + required: + - "fieldName" + type: "object" + DataAnalysisColumnProfileV1: + nullable: true + properties: + tableName: + description: "Table name." + nullable: true + type: "string" + filter: + $ref: "#/components/schemas/QueryFilter" + description: "Core QueryFilter JSON, identical to the table query and count\ + \ endpoints." + nullable: true + estimate: + description: "True when statistics describe a sample rather than the entire\ + \ filtered population." + nullable: true + type: "boolean" + sampleRows: + description: "Actual number of rows in the sample, not an extrapolated count." + nullable: true + type: "integer" + truncated: + description: "True when the row limit prevented a complete scan." + nullable: true + type: "boolean" + rowsScanned: + description: "Number of source rows included in computation." + nullable: true + type: "integer" + asOf: + description: "As of." + format: "date-time" + nullable: true + type: "string" + dataVersion: + description: "Data version." + nullable: true + type: "string" + warnings: + description: "Warnings." + items: + type: "string" + nullable: true + type: "array" + fieldName: + description: "Readable field name to profile or enumerate." + nullable: true + type: "string" + fieldLabel: + description: "Field label." + nullable: true + type: "string" + fieldType: + description: "Field type." + nullable: true + type: "string" + counts: + description: "Counts." + items: + $ref: "#/components/schemas/DataAnalysisMetricV1" + nullable: true + type: "array" + numeric: + $ref: "#/components/schemas/DataAnalysisNumericProfileV1" + description: "Numeric." + nullable: true + dates: + $ref: "#/components/schemas/DataAnalysisDateProfileV1" + description: "Dates." + nullable: true + text: + $ref: "#/components/schemas/DataAnalysisTextProfileV1" + description: "Text." + nullable: true + booleans: + description: "Booleans." + items: + $ref: "#/components/schemas/DataAnalysisMetricV1" + nullable: true + type: "array" + topValues: + description: "Top values." + items: + $ref: "#/components/schemas/DataAnalysisValueCountV1" + nullable: true + type: "array" + chart: + $ref: "#/components/schemas/DataAnalysisChartV1" + description: "Chart." + nullable: true + type: "object" + DataAnalysisDateProfileV1: + nullable: true + properties: + metrics: + description: "Metrics." + items: + $ref: "#/components/schemas/DataAnalysisMetricV1" + nullable: true + type: "array" + unit: + description: "Unit." + nullable: true + type: "string" + buckets: + description: "Buckets." + items: + $ref: "#/components/schemas/DataAnalysisValueCountV1" + nullable: true + type: "array" + gaps: + description: "Gaps." + items: + $ref: "#/components/schemas/DataAnalysisValueCountV1" + nullable: true + type: "array" + dayOfWeek: + description: "Day of week." + items: + $ref: "#/components/schemas/DataAnalysisValueCountV1" + nullable: true + type: "array" + hourOfDay: + description: "Hour of day." + items: + $ref: "#/components/schemas/DataAnalysisValueCountV1" + nullable: true + type: "array" + type: "object" + DataAnalysisDistinctValuesPageV1: + nullable: true + properties: + tableName: + description: "Table name." + nullable: true + type: "string" + filter: + $ref: "#/components/schemas/QueryFilter" + description: "Core QueryFilter JSON, identical to the table query and count\ + \ endpoints." + nullable: true + estimate: + description: "True when statistics describe a sample rather than the entire\ + \ filtered population." + nullable: true + type: "boolean" + sampleRows: + description: "Actual number of rows in the sample, not an extrapolated count." + nullable: true + type: "integer" + truncated: + description: "True when the row limit prevented a complete scan." + nullable: true + type: "boolean" + rowsScanned: + description: "Number of source rows included in computation." + nullable: true + type: "integer" + asOf: + description: "As of." + format: "date-time" + nullable: true + type: "string" + dataVersion: + description: "Data version." + nullable: true + type: "string" + warnings: + description: "Warnings." + items: + type: "string" + nullable: true + type: "array" + fieldName: + description: "Readable field name to profile or enumerate." + nullable: true + type: "string" + page: + description: "Zero-based page number." + nullable: true + type: "integer" + pageSize: + description: "Maximum values per page, from 1 to 500." + nullable: true + type: "integer" + hasMore: + description: "Has more." + nullable: true + type: "boolean" + totalDistinct: + description: "Total distinct." + nullable: true + type: "integer" + values: + description: "Values." + items: + $ref: "#/components/schemas/DataAnalysisValueCountV1" + nullable: true + type: "array" + type: "object" + DataAnalysisDistinctValuesRequestV1: + additionalProperties: false + properties: + filter: + $ref: "#/components/schemas/QueryFilter" + description: "Core QueryFilter JSON, identical to the table query and count\ + \ endpoints." + fieldName: + description: "Readable field name to profile or enumerate." + type: "string" + page: + description: "Zero-based page number." + type: "integer" + pageSize: + description: "Maximum values per page, from 1 to 500." + type: "integer" + search: + description: "Optional search text applied to stored values." + type: "string" + required: + - "fieldName" + type: "object" + DataAnalysisDrillV1: + nullable: true + properties: + tableName: + description: "Table name." + nullable: true + type: "string" + filter: + $ref: "#/components/schemas/QueryFilter" + description: "Core QueryFilter JSON, identical to the table query and count\ + \ endpoints." + nullable: true + type: "object" + DataAnalysisDuplicatesRequestV1: + additionalProperties: false + properties: + filter: + $ref: "#/components/schemas/QueryFilter" + description: "Core QueryFilter JSON, identical to the table query and count\ + \ endpoints." + fields: + description: "Readable fields to include; omit for all analyzable fields\ + \ on a table profile." + items: + type: "string" + type: "array" + limit: + description: "Maximum duplicate groups to return, from 1 to 500; default\ + \ 50." + type: "integer" + required: + - "fields" + type: "object" + DataAnalysisDuplicatesResultV1: + nullable: true + properties: + tableName: + description: "Table name." + nullable: true + type: "string" + filter: + $ref: "#/components/schemas/QueryFilter" + description: "Core QueryFilter JSON, identical to the table query and count\ + \ endpoints." + nullable: true + estimate: + description: "True when statistics describe a sample rather than the entire\ + \ filtered population." + nullable: true + type: "boolean" + sampleRows: + description: "Actual number of rows in the sample, not an extrapolated count." + nullable: true + type: "integer" + truncated: + description: "True when the row limit prevented a complete scan." + nullable: true + type: "boolean" + rowsScanned: + description: "Number of source rows included in computation." + nullable: true + type: "integer" + asOf: + description: "As of." + format: "date-time" + nullable: true + type: "string" + dataVersion: + description: "Data version." + nullable: true + type: "string" + warnings: + description: "Warnings." + items: + type: "string" + nullable: true + type: "array" + fields: + description: "Readable fields to include; omit for all analyzable fields\ + \ on a table profile." + items: + type: "string" + nullable: true + type: "array" + groups: + description: "Groups." + items: + $ref: "#/components/schemas/DataAnalysisValueCountV1" + nullable: true + type: "array" + type: "object" + DataAnalysisFieldSummaryV1: + nullable: true + properties: + field: + description: "Field." + nullable: true + type: "string" + label: + description: "Label." + nullable: true + type: "string" + type: + description: "Type." + nullable: true + type: "string" + completeness: + description: "Completeness." + nullable: true + type: "number" + distinct: + description: "Distinct." + nullable: true + type: "integer" + uniqueness: + description: "Uniqueness." + nullable: true + type: "number" + constant: + description: "Constant." + nullable: true + type: "boolean" + empty: + description: "Empty." + nullable: true + type: "boolean" + nulls: + $ref: "#/components/schemas/DataAnalysisMetricV1" + description: "Nulls." + nullable: true + blanks: + $ref: "#/components/schemas/DataAnalysisMetricV1" + description: "Blanks." + nullable: true + type: "object" + DataAnalysisLimitsV1: + nullable: true + properties: + maxRows: + description: "Max rows." + nullable: true + type: "integer" + sampleThresholdRows: + description: "Sample threshold rows." + nullable: true + type: "integer" + sampleRows: + description: "Actual number of rows in the sample, not an extrapolated count." + nullable: true + type: "integer" + minGroupSize: + description: "Min group size." + nullable: true + type: "integer" + type: "object" + DataAnalysisMetricV1: + nullable: true + properties: + key: + description: "Key." + nullable: true + type: "string" + label: + description: "Label." + nullable: true + type: "string" + value: + description: "Value." + nullable: true + oneOf: + - type: "string" + - type: "number" + - type: "boolean" + - nullable: true + type: "object" + valueType: + description: "Value type." + enum: + - "COUNT" + - "DECIMAL" + - "PERCENT" + - "DATE" + - "DATE_TIME" + - "DURATION" + - "TEXT" + nullable: true + type: "string" + estimate: + description: "Whether this metric is estimated; the population rows metric\ + \ remains exact." + type: "boolean" + formatted: + description: "Formatted." + nullable: true + type: "string" + drill: + $ref: "#/components/schemas/DataAnalysisDrillV1" + description: "Open the matching records using this table and QueryFilter;\ + \ null when suppressed." + nullable: true + suppressed: + description: "Suppressed." + nullable: true + type: "boolean" + suppressedReason: + description: "Suppressed reason." + nullable: true + type: "string" + type: "object" + DataAnalysisNumericProfileV1: + nullable: true + properties: + metrics: + description: "Metrics." + items: + $ref: "#/components/schemas/DataAnalysisMetricV1" + nullable: true + type: "array" + histogram: + description: "Histogram." + items: + $ref: "#/components/schemas/DataAnalysisValueCountV1" + nullable: true + type: "array" + outliers: + description: "Outliers." + items: + $ref: "#/components/schemas/DataAnalysisValueCountV1" + nullable: true + type: "array" + type: "object" + DataAnalysisPointV1: + nullable: true + properties: + x: + description: "X." + nullable: true + oneOf: + - type: "string" + - type: "number" + - type: "boolean" + - nullable: true + type: "object" + "y": + description: "Y." + nullable: true + type: "number" + label: + description: "Label." + nullable: true + type: "string" + drill: + $ref: "#/components/schemas/DataAnalysisDrillV1" + description: "Open the matching records using this table and QueryFilter;\ + \ null when suppressed." + nullable: true + type: "object" + DataAnalysisRunStartedV1: + nullable: true + properties: + runId: + description: "Run id." + nullable: true + type: "string" + status: + description: "Status." + nullable: true + type: "string" + type: "object" + DataAnalysisRunStatusV1: + nullable: true + properties: + runId: + description: "Run id." + nullable: true + type: "string" + status: + description: "Status." + nullable: true + type: "string" + error: + description: "Error." + nullable: true + type: "string" + columnProfile: + $ref: "#/components/schemas/DataAnalysisColumnProfileV1" + description: "Completed column result; null for other operations or unfinished\ + \ runs." + nullable: true + tableProfile: + $ref: "#/components/schemas/DataAnalysisTableProfileV1" + description: "Completed table result; null for other operations or unfinished\ + \ runs." + nullable: true + startedAt: + description: "Started at." + format: "date-time" + nullable: true + type: "string" + finishedAt: + description: "Finished at." + format: "date-time" + nullable: true + type: "string" + type: "object" + DataAnalysisSeriesV1: + nullable: true + properties: + name: + description: "Name." + nullable: true + type: "string" + points: + description: "Points." + items: + $ref: "#/components/schemas/DataAnalysisPointV1" + nullable: true + type: "array" + type: "object" + DataAnalysisStartRunRequestV1: + additionalProperties: false + properties: + operation: + description: "COLUMN_PROFILE or TABLE_PROFILE." + enum: + - "COLUMN_PROFILE" + - "TABLE_PROFILE" + type: "string" + tableName: + description: "Table name." + type: "string" + request: + description: "Profile parameters for the selected operation, without tableName." + oneOf: + - $ref: "#/components/schemas/DataAnalysisColumnProfileRequestV1" + - $ref: "#/components/schemas/DataAnalysisTableProfileRequestV1" + required: + - "operation" + - "tableName" + - "request" + type: "object" + DataAnalysisTableProfileRequestV1: + additionalProperties: false + properties: + filter: + $ref: "#/components/schemas/QueryFilter" + description: "Core QueryFilter JSON, identical to the table query and count\ + \ endpoints." + fields: + description: "Readable fields to include; omit for all analyzable fields\ + \ on a table profile." + items: + type: "string" + type: "array" + type: "object" + DataAnalysisTableProfileV1: + nullable: true + properties: + tableName: + description: "Table name." + nullable: true + type: "string" + filter: + $ref: "#/components/schemas/QueryFilter" + description: "Core QueryFilter JSON, identical to the table query and count\ + \ endpoints." + nullable: true + estimate: + description: "True when statistics describe a sample rather than the entire\ + \ filtered population." + nullable: true + type: "boolean" + sampleRows: + description: "Actual number of rows in the sample, not an extrapolated count." + nullable: true + type: "integer" + truncated: + description: "True when the row limit prevented a complete scan." + nullable: true + type: "boolean" + rowsScanned: + description: "Number of source rows included in computation." + nullable: true + type: "integer" + asOf: + description: "As of." + format: "date-time" + nullable: true + type: "string" + dataVersion: + description: "Data version." + nullable: true + type: "string" + warnings: + description: "Warnings." + items: + type: "string" + nullable: true + type: "array" + rowCount: + $ref: "#/components/schemas/DataAnalysisMetricV1" + description: "Row count." + nullable: true + fields: + description: "Readable fields to include; omit for all analyzable fields\ + \ on a table profile." + items: + $ref: "#/components/schemas/DataAnalysisFieldSummaryV1" + nullable: true + type: "array" + heatmap: + $ref: "#/components/schemas/DataAnalysisChartV1" + description: "Heatmap." + nullable: true + candidateKeys: + description: "Candidate keys." + items: + $ref: "#/components/schemas/DataAnalysisCandidateKeyV1" + nullable: true + type: "array" + type: "object" + DataAnalysisTextProfileV1: + nullable: true + properties: + metrics: + description: "Metrics." + items: + $ref: "#/components/schemas/DataAnalysisMetricV1" + nullable: true + type: "array" + histogram: + description: "Histogram." + items: + $ref: "#/components/schemas/DataAnalysisValueCountV1" + nullable: true + type: "array" + patterns: + description: "Patterns." + items: + $ref: "#/components/schemas/DataAnalysisValueCountV1" + nullable: true + type: "array" + sampleRows: + description: "Actual number of rows in the sample, not an extrapolated count." + nullable: true + type: "integer" + type: "object" + DataAnalysisValueCountV1: + nullable: true + properties: + value: + description: "Value." + nullable: true + oneOf: + - type: "string" + - type: "number" + - type: "boolean" + - nullable: true + type: "object" + label: + description: "Label." + nullable: true + type: "string" + count: + description: "Count." + nullable: true + type: "integer" + share: + description: "Share." + nullable: true + type: "number" + drill: + $ref: "#/components/schemas/DataAnalysisDrillV1" + description: "Open the matching records using this table and QueryFilter;\ + \ null when suppressed." + nullable: true + type: "object" + FilterCriteria: + properties: + fieldName: + description: "Field name that this criteria applies to" + type: "string" + operator: + description: "Logical operator that applies to this criteria" + enum: + - "EQUALS" + - "NOT_EQUALS" + - "NOT_EQUALS_OR_IS_NULL" + - "IN" + - "NOT_IN" + - "IS_NULL_OR_IN" + - "LIKE" + - "NOT_LIKE" + - "STARTS_WITH" + - "ENDS_WITH" + - "CONTAINS" + - "NOT_STARTS_WITH" + - "NOT_ENDS_WITH" + - "NOT_CONTAINS" + - "LESS_THAN" + - "LESS_THAN_OR_EQUALS" + - "GREATER_THAN" + - "GREATER_THAN_OR_EQUALS" + - "IS_BLANK" + - "IS_NOT_BLANK" + - "BETWEEN" + - "NOT_BETWEEN" + - "TRUE" + - "FALSE" + type: "string" + values: + description: "Values to apply via the operator to the field" + items: + type: "string" + type: "array" + type: "object" + OrderBy: + properties: + fieldName: + description: "Field name that this order-by applies to" + type: "string" + isAscending: + description: "Whether this order-by is ascending or descending" + type: "boolean" + type: "object" + QueryFilter: + properties: + booleanOperator: + description: "Boolean operator to apply between criteria" + enum: + - "AND" + - "OR" + type: "string" + criteria: + description: "Field level criteria that make up the query filter" + items: + $ref: "#/components/schemas/FilterCriteria" + type: "array" + limit: + description: "Maximum number of results to return." + type: "number" + orderBys: + description: "How the query's results should be ordered (sorted)." + items: + $ref: "#/components/schemas/OrderBy" + type: "array" + skip: + description: "Records to skip (e.g., to implement pagination)" + type: "number" + type: "object" + securitySchemes: + sessionUuidCookie: + in: "cookie" + name: "sessionUUID" + type: "apiKey" +info: + contact: + email: "contact@kingsrook.com" + description: "Profiles, distinct values, duplicate groups and exact analysis runs\ + \ as the signed-in user." + title: "QQQ Middleware API extension: data-analysis" + version: "1.0.0" +openapi: "3.0.3" +paths: + /qqq/v1/qbit/data-analysis/tables/{table}/capabilities: + get: + description: "Discover whether the current user can analyze this table, readable\ + \ fields, chart defaults, limits and backend capabilities." + parameters: + - description: "QQQ table name to analyze." + in: "path" + name: "table" + required: true + schema: + type: "string" + responses: + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisCapabilitiesV1" + description: "Successful Response" + security: + - sessionUuidCookie: + - "N/A" + summary: "Check analysis availability" + /qqq/v1/qbit/data-analysis/tables/{table}/column-profile: + post: + description: "Profile one column of a table under a filter: counts, nulls, distinct\ + \ values, top values, numeric statistics and histogram, date patterns; every\ + \ number has a drill-through filter unless privacy suppresses it." + parameters: + - description: "QQQ table name to analyze." + in: "path" + name: "table" + required: true + schema: + type: "string" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisColumnProfileRequestV1" + required: true + responses: + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisColumnProfileV1" + description: "Successful Response" + security: + - sessionUuidCookie: + - "N/A" + summary: "Profile one column" + /qqq/v1/qbit/data-analysis/tables/{table}/distinct-values: + post: + description: "Page through distinct stored values with display labels, counts\ + \ and drill-through filters; search narrows stored values." + parameters: + - description: "QQQ table name to analyze." + in: "path" + name: "table" + required: true + schema: + type: "string" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisDistinctValuesRequestV1" + required: true + responses: + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisDistinctValuesPageV1" + description: "Successful Response" + security: + - sessionUuidCookie: + - "N/A" + summary: "List distinct column values" + /qqq/v1/qbit/data-analysis/tables/{table}/table-profile: + post: + description: "Profile readable fields under a filter for completeness, uniqueness,\ + \ constant or empty columns and candidate keys." + parameters: + - description: "QQQ table name to analyze." + in: "path" + name: "table" + required: true + schema: + type: "string" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisTableProfileRequestV1" + required: true + responses: + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisTableProfileV1" + description: "Successful Response" + security: + - sessionUuidCookie: + - "N/A" + summary: "Profile table completeness and keys" + /qqq/v1/qbit/data-analysis/tables/{table}/duplicates: + post: + description: "Group matching records by the selected readable fields and return\ + \ repeated tuples with counts and drill-through filters." + parameters: + - description: "QQQ table name to analyze." + in: "path" + name: "table" + required: true + schema: + type: "string" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisDuplicatesRequestV1" + required: true + responses: + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisDuplicatesResultV1" + description: "Successful Response" + security: + - sessionUuidCookie: + - "N/A" + summary: "Find duplicate groups" + /qqq/v1/qbit/data-analysis/runs: + post: + description: "Start an owner-scoped exact column or table profile. Poll the\ + \ returned runId until COMPLETE or FAILED; results honor the current session\ + \ permissions." + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisStartRunRequestV1" + required: true + responses: + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisRunStartedV1" + description: "Successful Response" + security: + - sessionUuidCookie: + - "N/A" + summary: "Start an exact profile run" + /qqq/v1/qbit/data-analysis/runs/{runId}: + get: + description: "Read status, timestamps and the typed columnProfile or tableProfile\ + \ result of an owner-scoped run. Completed results are rechecked against current\ + \ permissions." + parameters: + - description: "Owner-scoped analysis run id." + in: "path" + name: "runId" + required: true + schema: + type: "string" + responses: + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisRunStatusV1" + description: "Successful Response" + security: + - sessionUuidCookie: + - "N/A" + summary: "Get an exact profile run" diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/CoreAnalysisBackendTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/CoreAnalysisBackendTest.java new file mode 100644 index 0000000..324d20e --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/CoreAnalysisBackendTest.java @@ -0,0 +1,204 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis; + +import java.util.Set; +import com.kingsrook.qbits.dataanalysis.engine.CoreAnalysisBackend; +import com.kingsrook.qbits.dataanalysis.engine.StandInAnalysisBackend; +import com.kingsrook.qbits.dataanalysis.result.ColumnProfileRequest; +import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateCapability; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryAggregateAction; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + +/******************************************************************************* + ** Native sample membership and capability-directed fallback behavior. + *******************************************************************************/ +class CoreAnalysisBackendTest extends EngineFixture +{ + /******************************************************************************* + ** Restore the global dispatcher after a capability-specific fixture. + *******************************************************************************/ + @AfterEach + void restoreModule() + { + QBackendModuleDispatcher.registerBackendModule(new MemoryBackendModule()); + } + + /******************************************************************************* + ** Deterministic memory sampling scales every-k-th counts to the population. + *******************************************************************************/ + @Test + void sampledPopulation() throws Exception + { + config.withDefaultSampleThresholdRows(5).withDefaultSampleRows(4); + assertThat(service.backend("orders")).isInstanceOf(CoreAnalysisBackend.class); + var profile = service.columnProfile(new ColumnProfileRequest().withTableName("orders").withFieldName("amount")); + assertThat(profile.getEstimate()).isTrue(); + assertThat(profile.getSampleRows()).isEqualTo(4); + assertThat(profile.getNumeric().getMetrics().stream().filter(m -> m.getKey().equals("sum")).findFirst().orElseThrow().getValue().toString()).isEqualTo("1030"); + var counts = profile.getCounts().stream().collect(java.util.stream.Collectors.toMap(m -> m.getKey(), m -> m)); + assertThat(counts.get("rows").getValue()).isEqualTo(12L); + assertThat(counts.get("rows").getEstimate()).isFalse(); + assertThat(counts.get("nonNull").getValue()).isEqualTo(Math.round(4.0 * 12 / 4)); + assertThat(profile.getTopValues()).allSatisfy(v -> + { + assertThat(v.getCount()).isEqualTo(Math.round(1.0 * 12 / 4)); + assertThat(v.getShare()).isEqualByComparingTo("0.25"); + }); + assertThat(com.kingsrook.qqq.backend.core.utils.JsonUtils.toJson(profile)).doesNotContain("\"operator\":\"IN\""); + new MemoryEngineTest().assertDrills(profile); + config.withDefaultMaxRows(0); + service.getConfigResolver().invalidate(); + var empty = service.columnProfile(new ColumnProfileRequest().withTableName("orders").withFieldName("amount")); + assertThat(empty.getEstimate()).isTrue(); + assertThat(empty.getSampleRows()).isEqualTo(4); + assertThat(empty.getTruncated()).isFalse(); + } + + /******************************************************************************* + ** Rounding counts must not alter raw shares, and drills retain the user filter. + *******************************************************************************/ + @Test + void filteredSampleRatios() throws Exception + { + config.withDefaultSampleThresholdRows(2).withDefaultSampleRows(4); + var filter = com.kingsrook.qbits.dataanalysis.engine.DrillFilters.condition("id", + com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.GREATER_THAN, 1); + var profile = service.columnProfile(new ColumnProfileRequest().withTableName("orders").withFieldName("amount").withFilter(filter)); + // The filtered ids 2..12, sampled every third row, yield 2, 5, 8, 11. + var sampledAmounts = java.util.Arrays.asList(0, 20, 40, null); + long nonNull = sampledAmounts.stream().filter(java.util.Objects::nonNull).count(); + var counts = profile.getCounts().stream().collect(java.util.stream.Collectors.toMap(m -> m.getKey(), m -> m)); + assertThat(profile.getSampleRows()).isEqualTo(sampledAmounts.size()); + assertThat(counts.get("rows").getValue()).isEqualTo(11L); + assertThat(counts.get("nonNull").getValue()).isEqualTo(Math.round(nonNull * 11.0 / sampledAmounts.size())); + assertThat(counts.get("nulls").getValue()).isEqualTo(Math.round((sampledAmounts.size() - nonNull) * 11.0 / sampledAmounts.size())); + assertThat(profile.getTopValues()).hasSize(4).allSatisfy(v -> + { + assertThat(v.getCount()).isEqualTo(Math.round(11.0 / sampledAmounts.size())); + assertThat(v.getShare()).isEqualByComparingTo("0.25"); + }); + var exact = new StandInAnalysisBackend(service.getConfigResolver().resolve("orders")); + assertThat(exact.count("orders", counts.get("rows").getDrill().getFilter())).isEqualTo(11); + assertThat(exact.count("orders", counts.get("nonNull").getDrill().getFilter())).isEqualTo(10); + var twenty = profile.getTopValues().stream().filter(v -> v.getValue() != null && v.getValue().toString().equals("20")).findFirst().orElseThrow(); + assertThat(exact.count("orders", twenty.getDrill().getFilter())).isEqualTo(2); + } + + /******************************************************************************* + ** Zero and capped percentages never divide by zero or restrict the row drill. + *******************************************************************************/ + @Test + void samplingBoundaries() throws Exception + { + config.withDefaultSampleThresholdRows(2).withDefaultSampleRows(0); + var empty = service.columnProfile(new ColumnProfileRequest().withTableName("orders").withFieldName("amount")); + assertThat(empty.getSampleRows()).isZero(); + assertThat(empty.getCounts().getFirst().getValue()).isEqualTo(12L); + assertThat(empty.getCounts().get(1).getValue()).isEqualTo(0L); + config.withDefaultSampleRows(100); + service.getConfigResolver().invalidate(); + var scope = service.backend("orders").scope("orders", null, false); + assertThat(scope.samplePercent()).isEqualByComparingTo("100"); + assertThat(scope.rowsScanned()).isEqualTo(12); + } + + /******************************************************************************* + ** A non-random estimate has one boundary bind and population-wide drills. + *******************************************************************************/ + @Test + void prefixBoundary() throws Exception + { + capabilities(Set.of()); + config.withDefaultSampleThresholdRows(2).withDefaultSampleRows(4); + var filter = com.kingsrook.qbits.dataanalysis.engine.DrillFilters.condition("id", + com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.GREATER_THAN, 2); + var backend = service.backend("orders"); + var scope = backend.scope("orders", filter, false); + assertThat(scope.filter().getSubFilters().get(1).getCriteria()).singleElement().satisfies(c -> + { + assertThat(c.getFieldName()).isEqualTo("id"); + assertThat(c.getOperator()).isEqualTo(com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.LESS_THAN_OR_EQUALS); + assertThat(c.getValues()).containsExactly(6); + }); + var profile = service.columnProfile(new ColumnProfileRequest().withTableName("orders").withFieldName("quantity").withFilter(filter)); + assertThat(profile.getCounts().getFirst().getValue()).isEqualTo(10L); + assertThat(profile.getCounts().stream().filter(m -> m.getKey().equals("nonNull")).findFirst().orElseThrow().getValue()).isEqualTo(4L); + assertThat(profile.getWarnings()).contains("Estimate from the first 4 rows by id"); + var drill = profile.getCounts().getFirst().getDrill(); + assertThat(new StandInAnalysisBackend(service.getConfigResolver().resolve("orders")).count("orders", drill.getFilter())).isEqualTo(10); + assertThat(com.kingsrook.qqq.backend.core.utils.JsonUtils.toJson(profile)).doesNotContain("\"operator\":\"IN\""); + } + + /******************************************************************************* + ** Empty capabilities select the stand-in; partial support uses core plus fallback. + *******************************************************************************/ + @Test + void fallbackSelection() throws Exception + { + capabilities(Set.of()); + assertThat(service.backend("orders")).isExactlyInstanceOf(StandInAnalysisBackend.class); + capabilities(Set.of(AggregateCapability.STDDEV)); + assertThat(service.backend("orders")).isInstanceOf(CoreAnalysisBackend.class); + var profile = service.columnProfile(new ColumnProfileRequest().withTableName("orders").withFieldName("amount")); + assertThat(profile.getNumeric().getMetrics().stream().filter(m -> m.getKey().equals("p50")).findFirst().orElseThrow().getValue().toString()).isEqualTo("20.0"); + new MemoryEngineTest().assertDrills(profile); + } + + /******************************************************************************* + ** Advertise selected capabilities while retaining the fixture's actual actions. + *******************************************************************************/ + private void capabilities(Set supported) + { + LimitedMemoryBackend.supported = supported; + QBackendModuleDispatcher.registerBackendModule(new LimitedMemoryBackend()); + } + + /******************************************************************************* + ** Public no-argument backend class required by the module dispatcher. + *******************************************************************************/ + public static class LimitedMemoryBackend extends MemoryBackendModule + { + private static Set supported = Set.of(); + + /******************************************************************************* + ** Retain real memory execution with a deliberately reduced feature set. + *******************************************************************************/ + @Override + public AggregateInterface getAggregateInterface() + { + return (new MemoryAggregateAction() + { + @Override + public Set getSupportedCapabilities(AggregateInput input) + { + return (supported); + } + }); + } + } +} diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisApiTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisApiTest.java new file mode 100644 index 0000000..1c6d144 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisApiTest.java @@ -0,0 +1,290 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.kingsrook.qbits.dataanalysis.api.DataAnalysisApi; +import com.kingsrook.qqq.backend.core.instances.AbstractQQQApplication; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleCustomizerInterface; +import com.kingsrook.qqq.middleware.javalin.QApplicationJavalinServer; +import com.kingsrook.qqq.middleware.javalin.QJavalinImplementation; +import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiInvoker; +import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiRequest; +import com.kingsrook.qqq.middleware.javalin.specs.v1.MiddlewareVersionV1; +import com.kingsrook.qqq.middleware.javalin.tools.UiApiContract; +import com.kingsrook.qqq.openapi.model.HttpMethod; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + +/******************************************************************************* + ** Real HTTP session, schema and drill round-trip acceptance tests. + *******************************************************************************/ +class DataAnalysisApiTest extends EngineFixture +{ + private QApplicationJavalinServer server; + private UiApiInvoker invoker; + private String caller = "analysis-user"; + + /******************************************************************************* + ** Give MOCK sessions only the grants represented by their caller cookie. + *******************************************************************************/ + public static class Sessions implements QAuthenticationModuleCustomizerInterface + { + /******************************************************************************* + ** Build the host's ordinary user permission and record-lock policy. + *******************************************************************************/ + @Override + public void customizeSession(QInstance instance, QSession session, Map context) + { + QSession grants = TestSupport.admin().withPermissions("orders.read", "customer.read").withSecurityKeyValue("orders.all", true); + session.setPermissions(grants.getPermissions()); + session.setSecurityKeyValues(grants.getSecurityKeyValues()); + if(!"denied".equals(session.getUuid())) + { + session.withPermission("orders.analyze"); + } + session.withSecurityKeyValue("data_analysis_user", session.getUser().getIdReference()); + } + } + + /******************************************************************************* + ** Start on an OS-selected port using the actual spec router and authentication. + *******************************************************************************/ + private void start() throws Exception + { + instance.withInstanceDefaultAuthentication(new QAuthenticationMetaData().withName("http-test").withType(QAuthenticationType.MOCK) + .withCustomizer(new QCodeReference(Sessions.class))); + QInstance fixture = instance; + server = new QApplicationJavalinServer(new AbstractQQQApplication() + { + @Override + public QInstance defineQInstance() + { + return (fixture); + } + }).withPort(0).withServeFrontendNext(false).withServeFrontendMaterialDashboard(false) + .withServeLegacyUnversionedMiddlewareAPI(false).withStrictUiApiRoutes(true); + server.start(); + invoker = UiApiInvoker.forServer(server); + } + + /******************************************************************************* + ** Stop the server before the fixture releases context. + *******************************************************************************/ + @AfterEach + void stop() + { + if(server != null) + { + server.stop(); + } + QJavalinImplementation.setSessionCookieHttpOnly(false); + } + + /******************************************************************************* + ** Invoke through loopback, forwarding the actual session cookie. + *******************************************************************************/ + private JsonNode call(HttpMethod method, String path, Object body, int status) throws Exception + { + var request = new UiApiRequest().withMethod(method).withPath(DataAnalysisApi.PREFIX + path) + .withPathParam("table", "orders").withSessionUUID(caller).withUserTimezone("America/Chicago"); + if(body != null) + { + request.withJsonBody(body); + } + var response = invoker.invoke(request); + assertThat(response.getStatus()).as(response.getBody()).isEqualTo(status); + return (response.getJson()); + } + + /******************************************************************************* + ** All operations, assistant discovery, OpenAPI and count-drill equivalence. + *******************************************************************************/ + @Test + void happyPathsAndDrills() throws Exception + { + start(); + JsonNode capabilities = call(HttpMethod.GET, "/tables/{table}/capabilities", null, 200); + assertThat(capabilities.path("canAnalyze").asBoolean()).isTrue(); + assertThat(capabilities.path("backendCapabilities").toString()).contains("NUMERIC_BUCKET"); + assertThat(capabilities.path("fields").toString()).doesNotContain("secret").contains("CATEGORY", "HISTOGRAM"); + for(String field : List.of("amount", "note", "created", "order_date", "paid", "status")) + { + checkDrills(call(HttpMethod.POST, "/tables/{table}/column-profile", Map.of("fieldName", field), 200)); + } + checkDrills(call(HttpMethod.POST, "/tables/{table}/column-profile", Map.of("fieldName", "amount", "filter", + Map.of("criteria", List.of(Map.of("fieldName", "created", "operator", "EQUALS", "values", List.of("2026-01-01T12:00:00Z"))))), 200)); + checkDrills(call(HttpMethod.POST, "/tables/{table}/distinct-values", Map.of("fieldName", "status", "pageSize", 1), 200)); + checkDrills(call(HttpMethod.POST, "/tables/{table}/table-profile", Map.of("fields", List.of("amount", "note")), 200)); + checkDrills(call(HttpMethod.POST, "/tables/{table}/duplicates", Map.of("fields", List.of("quantity"), "limit", 2), 200)); + for(String operation : List.of("COLUMN_PROFILE", "TABLE_PROFILE")) + { + JsonNode started = call(HttpMethod.POST, "/runs", Map.of("operation", operation, "tableName", "orders", "request", + operation.equals("COLUMN_PROFILE") ? Map.of("fieldName", "quantity") : Map.of("fields", List.of("paid"))), 200); + JsonNode run = null; + for(int i = 0; i < 100; i++) + { + var response = invoker.invoke(new UiApiRequest().withMethod(HttpMethod.GET).withPath(DataAnalysisApi.PREFIX + "/runs/{runId}") + .withPathParam("runId", started.path("runId").asText()).withSessionUUID(caller).withUserTimezone("America/Chicago")); + assertThat(response.getStatus()).as(response.getBody()).isEqualTo(200); + run = response.getJson(); + if(!run.path("status").asText().equals("RUNNING")) + { + break; + } + Thread.sleep(20); + } + assertThat(run.path("status").asText()).isEqualTo("COMPLETE"); + var denied = invoker.invoke(new UiApiRequest().withMethod(HttpMethod.GET).withPath(DataAnalysisApi.PREFIX + "/runs/{runId}") + .withPathParam("runId", started.path("runId").asText()).withSessionUUID("another-owner")); + assertThat(denied.getStatus()).isEqualTo(403); + assertThat(run.path(operation.equals("COLUMN_PROFILE") ? "columnProfile" : "tableProfile").isObject()).isTrue(); + } + assertThat(invoker.listOperations().stream().filter(o -> o.pathTemplate().startsWith(DataAnalysisApi.PREFIX)).toList()).hasSize(7); + var client = java.net.http.HttpClient.newHttpClient(); + var response = client.send(java.net.http.HttpRequest.newBuilder(java.net.URI.create("http://127.0.0.1:" + server.getBoundPort() + "/qqq/v1/openapi.json")).build(), java.net.http.HttpResponse.BodyHandlers.ofString()); + assertThat(response.statusCode()).isEqualTo(200); + JsonNode api = new ObjectMapper().readTree(response.body()); + assertThat(api.path("paths").has("/qqq/v1/qbit/data-analysis/tables/{table}/column-profile")).isTrue(); + assertThat(api.path("components").path("schemas").has("DataAnalysisMetricV1")).isTrue(); + } + + /******************************************************************************* + ** Estimate payloads preserve exact totals and send drills to all matching rows. + *******************************************************************************/ + @Test + void estimateDrills() throws Exception + { + config.withDefaultSampleThresholdRows(2).withDefaultSampleRows(4); + instance.withSupplementalMetaData(new DataAnalysisRuntime(config)); + start(); + JsonNode profile = call(HttpMethod.POST, "/tables/{table}/column-profile", Map.of("fieldName", "amount"), 200); + assertThat(profile.path("estimate").asBoolean()).isTrue(); + assertThat(profile.path("sampleRows").asLong()).isEqualTo(4); + JsonNode rows = profile.path("counts").get(0); + assertThat(rows.path("value").asLong()).isEqualTo(12); + assertThat(rows.path("estimate").asBoolean()).isFalse(); + assertThat(profile.path("counts").get(1).path("formatted").asText()).startsWith("≈ "); + assertThat(profile.toString()).doesNotContain("\"operator\":\"IN\""); + var response = invoker.invoke(new UiApiRequest().withMethod(HttpMethod.POST).withPath("/table/{tableName}/count") + .withPathParam("tableName", "orders").withJsonBody(Map.of("filter", rows.path("drill").path("filter"))) + .withSessionUUID(caller).withUserTimezone("America/Chicago")); + assertThat(response.getStatus()).isEqualTo(200); + assertThat(response.getJson().path("count").asLong()).isEqualTo(12); + checkDrills(profile); + JsonNode table = call(HttpMethod.POST, "/tables/{table}/table-profile", Map.of("fields", List.of("amount", "quantity")), 200); + assertThat(table.path("estimate").asBoolean()).isTrue(); + assertThat(table.path("rowCount").path("value").asLong()).isEqualTo(12); + checkDrills(table); + } + + /******************************************************************************* + ** Recursively post returned count drills to the core count endpoint. + *******************************************************************************/ + private void checkDrills(JsonNode node) throws Exception + { + if(node.path("estimate").asBoolean(false)) + { + return; + } + if(node.isObject() && node.hasNonNull("drill")) + { + JsonNode expected = node.has("count") ? node.get("count") : node.has("y") ? node.get("y") : "COUNT".equals(node.path("valueType").asText()) ? node.get("value") : null; + if(expected != null && expected.isNumber()) + { + JsonNode drill = node.get("drill"); + var response = invoker.invoke(new UiApiRequest().withMethod(HttpMethod.POST).withPath("/table/{tableName}/count") + .withPathParam("tableName", drill.path("tableName").asText()).withJsonBody(Map.of("filter", drill.get("filter"))).withSessionUUID(caller).withUserTimezone("America/Chicago")); + assertThat(response.getStatus()).as(response.getBody()).isEqualTo(200); + assertThat(response.getJson().path("count").asLong()).as(drill.toString()).isEqualTo(expected.asLong()); + } + } + if(node.isContainerNode()) + { + for(JsonNode child : node) + { + checkDrills(child); + } + } + } + + /******************************************************************************* + ** Denials, locked fields, missing tables and malformed request bodies. + *******************************************************************************/ + @Test + void errors() throws Exception + { + start(); + call(HttpMethod.POST, "/tables/{table}/column-profile", Map.of("fieldName", "secret"), 400); + call(HttpMethod.POST, "/tables/{table}/column-profile", Map.of("fieldName", "missing"), 400); + call(HttpMethod.POST, "/tables/{table}/column-profile", Map.of(), 400); + call(HttpMethod.POST, "/tables/{table}/column-profile", Map.of("fieldName", 1), 400); + call(HttpMethod.POST, "/tables/{table}/column-profile", Map.of("fieldName", "amount", "exact", "yes"), 400); + call(HttpMethod.POST, "/tables/{table}/distinct-values", Map.of("fieldName", "note", "pageSize", 0), 400); + call(HttpMethod.POST, "/tables/{table}/duplicates", Map.of("fields", List.of()), 400); + call(HttpMethod.POST, "/tables/{table}/table-profile", Map.of("fields", "amount"), 400); + call(HttpMethod.POST, "/runs", Map.of("operation", "PIVOT"), 400); + call(HttpMethod.POST, "/runs", com.fasterxml.jackson.databind.node.NullNode.getInstance(), 400); + call(HttpMethod.POST, "/tables/{table}/column-profile", Map.of("fieldName", "amount", "filter", + Map.of("criteria", List.of(Map.of("fieldName", "secret", "operator", "EQUALS", "values", List.of("hidden"))))), 400); + var absent = invoker.invoke(new UiApiRequest().withMethod(HttpMethod.GET).withPath(DataAnalysisApi.PREFIX + "/tables/{table}/capabilities") + .withPathParam("table", "missing").withSessionUUID(caller).withUserTimezone("America/Chicago")); + assertThat(absent.getStatus()).isEqualTo(404); + caller = "denied"; + assertThat(call(HttpMethod.GET, "/tables/{table}/capabilities", null, 200).path("canAnalyze").asBoolean()).isFalse(); + call(HttpMethod.POST, "/tables/{table}/column-profile", Map.of("fieldName", "amount"), 403); + call(HttpMethod.POST, "/tables/{table}/distinct-values", Map.of("fieldName", "amount"), 403); + call(HttpMethod.POST, "/tables/{table}/table-profile", Map.of(), 403); + call(HttpMethod.POST, "/tables/{table}/duplicates", Map.of("fields", List.of("amount")), 403); + call(HttpMethod.POST, "/runs", Map.of("operation", "COLUMN_PROFILE", "tableName", "orders", "request", Map.of("fieldName", "amount")), 403); + } + + /******************************************************************************* + ** Generated fragment is committed verbatim and has only the seven operations. + *******************************************************************************/ + @Test + void contract() throws Exception + { + MiddlewareVersionV1 version = new MiddlewareVersionV1(); + version.setQInstance(instance); + String yaml = UiApiContract.generateExtensionYaml(version, DataAnalysisApi.extension()); + Path path = Path.of("src/main/resources/openapi/qqq-v1/data-analysis.yaml"); + if(Boolean.getBoolean("openapi.write")) + { + Files.createDirectories(path.getParent()); + Files.writeString(path, yaml); + } + assertThat(Files.exists(path)).as("Regenerate with ./mvnw -o test -Dtest=DataAnalysisApiTest#contract -Dopenapi.write=true -Dqqq.snapshot.version=4.1.0-DATAANALYSIS-SNAPSHOT").isTrue(); + assertThat(Files.readString(path).replace("\r\n", "\n")).as("OpenAPI changed: regenerate with -Dopenapi.write=true").isEqualTo(yaml.replace("\r\n", "\n")); + } +} + diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisSecurityTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisSecurityTest.java index 84ded3a..57013c9 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisSecurityTest.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisSecurityTest.java @@ -91,7 +91,7 @@ class DataAnalysisSecurityTest instance.withEnforcePermissionsInCoreActions(false); assertThat(DataAnalysisPermissions.canAnalyze(table)).isFalse(); QContext.init(instance, TestSupport.user("alice").withPermission("sales.orders.analyze")); - assertThat(DataAnalysisPermissions.canAnalyze(table)).isTrue(); + assertThat(DataAnalysisPermissions.canAnalyze(table)).isFalse(); table.withPermissionRules(null); instance.withDefaultPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_INSERT_EDIT_DELETE_PERMISSIONS).withPermissionBaseName("defaultBase")); assertThat(DataAnalysisPermissions.analyzePermissionName(table)).isEqualTo("defaultBase.analyze"); @@ -101,6 +101,29 @@ class DataAnalysisSecurityTest + /******************************************************************************* + ** The core permission catalog honors configured inclusion and exclusion. + *******************************************************************************/ + @Test + void analyzeCatalog() throws Exception + { + QInstance instance = new QInstance().withEnforcePermissionsInCoreActions(true); + instance.addBackend(new com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData().withName("memory").withBackendType(com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule.class)); + for(String name : List.of("orders", "excluded", "outside")) + { + instance.addTable(new QTableMetaData().withName(name).withLabel(name).withPermissionRules( + new QPermissionRules().withLevel(PermissionLevel.READ_INSERT_EDIT_DELETE_PERMISSIONS))); + } + new DataAnalysisQBitProducer().withQBitConfig(new DataAnalysisQBitConfig().withBackendName("memory") + .withIncludeTables(java.util.Set.of("orders", "excluded")).withExcludeTables(java.util.Set.of("excluded"))).produce(instance); + assertThat(com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper.getAllAvailablePermissions(instance)).anySatisfy(permission -> + { + assertThat(permission.getName()).isEqualTo("orders.analyze"); + assertThat(permission.getPermissionType()).isEqualTo("Analyze"); + }).noneMatch(permission -> permission.getName().equals("excluded.analyze") || permission.getName().equals("outside.analyze") + || permission.getName().startsWith("data_analysis") && permission.getName().endsWith(".analyze")); + } + /******************************************************************************* ** Saved, star and run locks hide others even when the shared flag is true. *******************************************************************************/ diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java index a2421cf..c5343e5 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java @@ -279,12 +279,26 @@ abstract class EngineContractTest extends EngineFixture void estimatesRunsAndCache() throws Exception { config.withDefaultSampleThresholdRows(5).withDefaultSampleRows(4); - service = new DataAnalysisService(config); + service = newService(config, java.util.concurrent.ForkJoinPool.commonPool()); ColumnProfile estimate = service.columnProfile(column("amount")); assertThat(estimate.getEstimate()).isTrue(); - assertThat(estimate.getSampleRows()).isEqualTo(4); - assertMetric(estimate.getCounts(), "rows", "4"); - assertMetric(estimate.getNumeric().getMetrics(), "sum", "10"); + assertMetric(estimate.getCounts(), "rows", "12"); + assertThat(metric(estimate.getCounts(), "rows").getEstimate()).isFalse(); + assertThat(metric(estimate.getCounts(), "nonNull").getEstimate()).isTrue(); + assertThat(metric(estimate.getCounts(), "nonNull").getFormatted()).startsWith("≈ "); + assertDrills(estimate); + assertThat(count(metric(estimate.getCounts(), "rows").getDrill())).isEqualTo(12); + if(service.backend("orders") instanceof com.kingsrook.qbits.dataanalysis.engine.CoreAnalysisBackend) + { + assertThat(estimate.getWarnings()).anyMatch(w -> w.contains("random sample") && w.contains("scaled to the full 12 rows")); + } + else + { + assertThat(estimate.getSampleRows()).isEqualTo(4); + assertMetric(estimate.getCounts(), "nonNull", "4"); + assertMetric(estimate.getNumeric().getMetrics(), "sum", "10"); + assertThat(estimate.getWarnings()).contains("Estimate from the first 4 rows by id"); + } String id = service.startExactRun(column("amount")); RunStatus run = awaitRun(id); assertThat(run.getStatus()).isEqualTo("COMPLETE"); @@ -444,6 +458,10 @@ abstract class EngineContractTest extends EngineFixture *******************************************************************************/ private void walk(JsonNode node) throws Exception { + if(node.path("estimate").asBoolean(false)) + { + return; + } if(node.isObject()) { if(node.hasNonNull("drill")) @@ -551,7 +569,7 @@ abstract class EngineContractTest extends EngineFixture int previous = runs().size(); service.columnProfile(column("amount")); assertThat(runs()).hasSize(previous + 1); - service = new DataAnalysisService(config, task -> + service = newService(config, task -> { throw new java.util.concurrent.RejectedExecutionException("test"); }); @@ -632,7 +650,7 @@ abstract class EngineContractTest extends EngineFixture void executionFailureAndGuardedBooleans() throws Exception { List jobs = new java.util.ArrayList<>(); - service = new DataAnalysisService(config, jobs::add); + service = newService(config, jobs::add); String id = service.startExactRun(column("amount")); var table = instance.getTables().remove("orders"); jobs.getFirst().run(); @@ -641,7 +659,7 @@ abstract class EngineContractTest extends EngineFixture assertThat(service.getRun(id).getError()).contains("could not be completed"); assertThat(runs().getFirst().getValue("result_json")).isNull(); config.withDefaultMinGroupSize(3); - service = new DataAnalysisService(config, Runnable::run); + service = newService(config, Runnable::run); var paid = service.columnProfile(column("paid")); assertThat(paid.getBooleans()).anySatisfy(m -> { diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/EngineFixture.java b/src/test/java/com/kingsrook/qbits/dataanalysis/EngineFixture.java index cb3a382..8642f4f 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/EngineFixture.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/EngineFixture.java @@ -118,7 +118,15 @@ abstract class EngineFixture .withValue("note", notes[i]).withValue("paid", i == 11 ? null : i % 2 == 0).withValue("modifyDate", Instant.parse("2026-10-13T12:00:00Z"))); } config = new DataAnalysisQBitConfig().withBackendName("memory"); - service = new DataAnalysisService(config, Runnable::run); + service = newService(config, Runnable::run); + } + + /******************************************************************************* + ** Allow the identical contract to exercise both primitive implementations. + *******************************************************************************/ + protected DataAnalysisService newService(DataAnalysisQBitConfig configuration, java.util.concurrent.Executor executor) + { + return (new DataAnalysisService(configuration, executor)); } /******************************************************************************* diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/H2CompatibilityEngineTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/H2CompatibilityEngineTest.java new file mode 100644 index 0000000..afedf5b --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/H2CompatibilityEngineTest.java @@ -0,0 +1,39 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis; + +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; + +/******************************************************************************* + ** Verify native bucket SQL in H2's ordinal-grouping compatibility mode. + *******************************************************************************/ +class H2CompatibilityEngineTest extends H2EngineTest +{ + /******************************************************************************* + ** H2 supports core's GROUP BY positions in PostgreSQL compatibility mode. + *******************************************************************************/ + @Override + protected RDBMSBackendMetaData database() + { + RDBMSBackendMetaData database = super.database(); + return (database.withJdbcUrl(database.getJdbcUrl() + ";MODE=PostgreSQL")); + } +} diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/H2SamplingTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/H2SamplingTest.java new file mode 100644 index 0000000..2570efe --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/H2SamplingTest.java @@ -0,0 +1,82 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis; + +import java.util.List; +import com.kingsrook.qbits.dataanalysis.engine.DrillFilters; +import com.kingsrook.qbits.dataanalysis.engine.StandInAnalysisBackend; +import com.kingsrook.qbits.dataanalysis.result.ColumnProfileRequest; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.within; + +/******************************************************************************* + ** RAND samples fluctuate around population counts without changing drill scope. + *******************************************************************************/ +class H2SamplingTest extends EngineFixture +{ + /******************************************************************************* + ** Ordinal grouping is supported by H2's PostgreSQL compatibility mode. + *******************************************************************************/ + @Override + protected RDBMSBackendMetaData database() + { + return (new RDBMSBackendMetaData().withVendor("h2").withUsername("sa").withPassword("") + .withJdbcUrl("jdbc:h2:mem:sampling_" + java.util.UUID.randomUUID() + ";DB_CLOSE_DELAY=-1;DATABASE_TO_UPPER=FALSE;MODE=PostgreSQL")); + } + + /******************************************************************************* + ** A large sample gives stable tolerances and includes filtered-out records. + *******************************************************************************/ + @Test + void randomCountsAndHistograms() throws Exception + { + try(var statement = connection.createStatement()) + { + statement.execute("INSERT INTO orders (id, customer, amount) SELECT x, 1, CASE WHEN MOD(x, 4) = 0 THEN NULL ELSE MOD(x, 10) END FROM SYSTEM_RANGE(1001, 11000) AS source(x)"); + } + config.withDefaultSampleThresholdRows(100).withDefaultSampleRows(2000); + service.getConfigResolver().invalidate(); + var filter = DrillFilters.condition("id", QCriteriaOperator.GREATER_THAN, 1000); + var profile = service.columnProfile(new ColumnProfileRequest().withTableName("orders").withFieldName("amount").withFilter(filter)); + assertThat(profile.getEstimate()).isTrue(); + var metrics = profile.getCounts().stream().collect(java.util.stream.Collectors.toMap(m -> m.getKey(), m -> m)); + assertThat(metrics.get("rows").getValue()).isEqualTo(10000L); + assertThat(metrics.get("rows").getEstimate()).isFalse(); + assertThat(((Number) metrics.get("nulls").getValue()).longValue()).isCloseTo(2500L, within(500L)); + assertThat(((Number) metrics.get("nonNull").getValue()).longValue()).isCloseTo(7500L, within(500L)); + assertThat(metrics.get("nonNull").getFormatted()).startsWith("≈ "); + assertThat(profile.getNumeric().getHistogram().stream().mapToLong(v -> v.getCount()).sum()).isCloseTo(7500L, within(700L)); + var exact = new StandInAnalysisBackend(service.getConfigResolver().resolve("orders")); + assertThat(exact.count("orders", metrics.get("rows").getDrill().getFilter())).isEqualTo(10000); + for(var value : profile.getTopValues()) + { + long population = exact.count("orders", value.getDrill().getFilter()); + assertThat(value.getCount()).isCloseTo(population, within(500L)); + } + assertThat(com.kingsrook.qqq.backend.core.utils.JsonUtils.toJson(profile)).doesNotContain("\"operator\":\"IN\""); + var table = service.tableProfile(new com.kingsrook.qbits.dataanalysis.result.TableProfileRequest().withTableName("orders").withFilter(filter).withFields(List.of("amount"))); + assertThat(table.getRowCount().getValue()).isEqualTo(10000L); + assertThat(table.getEstimate()).isTrue(); + } +} diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/H2StreamedEngineTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/H2StreamedEngineTest.java new file mode 100644 index 0000000..ecc0423 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/H2StreamedEngineTest.java @@ -0,0 +1,48 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis; + +import java.util.concurrent.Executor; +import com.kingsrook.qbits.dataanalysis.engine.AnalysisBackend; +import com.kingsrook.qbits.dataanalysis.engine.StandInAnalysisBackend; +import com.kingsrook.qqq.backend.core.exceptions.QException; + +/******************************************************************************* + ** Run every engine contract assertion with the portable streamed implementation. + *******************************************************************************/ +class H2StreamedEngineTest extends H2EngineTest +{ + /******************************************************************************* + ** Change only the backend under test, preserving the entire shared contract. + *******************************************************************************/ + @Override + protected DataAnalysisService newService(DataAnalysisQBitConfig configuration, Executor executor) + { + return (new DataAnalysisService(configuration, executor) + { + @Override + public AnalysisBackend backend(String table) throws QException + { + return (new StandInAnalysisBackend(getConfigResolver().resolve(table))); + } + }); + } +} diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/OptionalMiddlewareTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/OptionalMiddlewareTest.java new file mode 100644 index 0000000..94bb62e --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/OptionalMiddlewareTest.java @@ -0,0 +1,77 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis; + +import java.nio.file.Path; +import java.util.Arrays; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + +/******************************************************************************* + ** A separate JVM proves the optional middleware is not needed by the engine. + *******************************************************************************/ +class OptionalMiddlewareTest +{ + /******************************************************************************* + ** Remove Javalin and middleware artifacts from the actual test classpath. + *******************************************************************************/ + @Test + void engineWithoutMiddleware() throws Exception + { + String classpath = Arrays.stream(System.getProperty("surefire.test.class.path", System.getProperty("java.class.path")).split(java.io.File.pathSeparator)) + .filter(entry -> !entry.contains("qqq-middleware-javalin") && !Path.of(entry).getFileName().toString().startsWith("javalin-")) + .collect(Collectors.joining(java.io.File.pathSeparator)); + Path log = Path.of("target", "core-only-smoke.log"); + Process process = new ProcessBuilder(Path.of(System.getProperty("java.home"), "bin", "java").toString(), "-cp", classpath, CoreOnlySmoke.class.getName()) + .redirectErrorStream(true).redirectOutput(log.toFile()).start(); + try + { + assertThat(process.waitFor(30, TimeUnit.SECONDS)).as("Core-only JVM timed out").isTrue(); + assertThat(process.exitValue()).as(java.nio.file.Files.readString(log)).isZero(); + } + finally + { + if(process.isAlive()) + { + process.destroyForcibly(); + } + } + } + + /******************************************************************************* + ** Entry point in the classpath with middleware removed. + *******************************************************************************/ + public static class CoreOnlySmoke + { + /******************************************************************************* + ** Metadata and runtime must initialize without loading any API classes. + *******************************************************************************/ + public static void main(String[] args) throws Exception + { + var instance = TestSupport.instance(); + assertThat(instance.getSupplementalMetaData("dataAnalysisRuntime")).isNotNull(); + assertThat(instance.getSupplementalMetaData("middlewareSpecExtensions")).isNull(); + assertThat(instance.getTable("data_analysis_run")).isNotNull(); + } + } +} diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/PostgresEngineTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/PostgresEngineTest.java index 9a23267..42121cd 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/PostgresEngineTest.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/PostgresEngineTest.java @@ -60,6 +60,29 @@ class PostgresEngineTest extends EngineContractTest } } + /******************************************************************************* + ** Large populations never become one bind per sampled primary key. + *******************************************************************************/ + @org.junit.jupiter.api.Test + void largeEstimate() throws Exception + { + try(var statement = connection.createStatement()) + { + statement.execute("TRUNCATE TABLE orders"); + statement.execute("INSERT INTO orders (id, customer, paid) SELECT id, 1, id % 2 = 0 FROM generate_series(1, 50000) AS id"); + } + config.withDefaultSampleThresholdRows(100).withDefaultSampleRows(1000); + service.getConfigResolver().invalidate(); + var profile = service.columnProfile(new com.kingsrook.qbits.dataanalysis.result.ColumnProfileRequest().withTableName("orders").withFieldName("paid")); + org.assertj.core.api.Assertions.assertThat(profile.getEstimate()).isTrue(); + assertMetric(profile.getCounts(), "rows", "50000"); + org.assertj.core.api.Assertions.assertThat(com.kingsrook.qqq.backend.core.utils.JsonUtils.toJson(profile)).doesNotContain("\"operator\":\"IN\""); + // Also exceed the old parameter ceiling with the requested sample itself. + config.withDefaultSampleRows(40000); + service.getConfigResolver().invalidate(); + assertMetric(service.columnProfile(new com.kingsrook.qbits.dataanalysis.result.ColumnProfileRequest().withTableName("orders").withFieldName("paid")).getCounts(), "rows", "50000"); + } + /******************************************************************************* ** Credentials belong only to the ephemeral test container. *******************************************************************************/ -- 2.54.0 From c90d6c98664de13a0d42e3fce33218dfe979e69b Mon Sep 17 00:00:00 2001 From: James Maes Date: Tue, 13 Oct 2026 17:44:00 -0500 Subject: [PATCH 02/10] chore: drop the agent handoff note Co-Authored-By: Claude Opus 5.5 --- docs/TODO.md | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 docs/TODO.md diff --git a/docs/TODO.md b/docs/TODO.md deleted file mode 100644 index e718b61..0000000 --- a/docs/TODO.md +++ /dev/null @@ -1,7 +0,0 @@ -# Phase b part 2 remaining work - -- Run the 21 PostgreSQL contract tests with accessible Docker, including the 50,000-row estimate regression at sample sizes 1,000 and 40,000. -- Review the existing endpoint work's core READ and H2 ordinal-grouping compatibility choices; this sampling task preserves them. -- Append the durable project/daily handoff in [SESSION-STATE.md](SESSION-STATE.md) to Second Brain from a workspace with vault write access. - -Completed: replace sampled-key IN scopes, scale native estimate counts, preserve raw ratios and full-population drills, update the OpenAPI contract and tests, and pass the requested offline Maven verification. -- 2.54.0 From d8b52113caed47ca5af6977ce0863b5004178fde Mon Sep 17 00:00:00 2001 From: James Maes Date: Tue, 13 Oct 2026 18:26:45 -0500 Subject: [PATCH 03/10] feat(results): human labels, formatted values, nice bins, date labels, current home Every metric carries a plain-English label and a display-formatted value (grouping, percent, the field's display format), histogram bins use nice edges with formatted labels, date buckets are labelled by unit, warnings appear only when true, and the Analyses home shows saved analyses, watches and recent runs with honest empty states (and no mojibake). Co-Authored-By: Claude Opus 5.5 --- .../dataanalysis/api/DataAnalysisSchemas.java | 24 +- .../dataanalysis/app/DataAnalysisHome.java | 87 +++++-- .../dataanalysis/engine/AnalysisBackend.java | 13 +- .../engine/AnalysisFormatting.java | 225 ++++++++++++++++++ .../dataanalysis/engine/ColumnProfiler.java | 30 ++- .../engine/CoreAnalysisBackend.java | 20 +- .../dataanalysis/engine/MetricLabels.java | 65 +++++ .../engine/MinGroupSizeGuard.java | 4 +- .../qbits/dataanalysis/engine/NiceBins.java | 62 +++++ .../dataanalysis/engine/ProfileSupport.java | 11 +- .../engine/StandInAnalysisBackend.java | 46 +++- .../dataanalysis/engine/TableProfiler.java | 2 +- .../openapi/qqq-v1/data-analysis.yaml | 15 +- .../dataanalysis/AnalysisFormattingTest.java | 158 ++++++++++++ .../dataanalysis/CoreAnalysisBackendTest.java | 5 +- .../dataanalysis/DataAnalysisHomeTest.java | 45 +++- .../dataanalysis/EngineContractTest.java | 63 ++++- .../qbits/dataanalysis/EngineFixture.java | 3 +- .../PresentationSamplingTest.java | 115 +++++++++ 19 files changed, 901 insertions(+), 92 deletions(-) create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisFormatting.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/MetricLabels.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/NiceBins.java create mode 100644 src/test/java/com/kingsrook/qbits/dataanalysis/AnalysisFormattingTest.java create mode 100644 src/test/java/com/kingsrook/qbits/dataanalysis/PresentationSamplingTest.java diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java b/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java index 794dbd5..151c72e 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java @@ -603,13 +603,13 @@ public class DataAnalysisSchemas @OpenAPIDescription("Type.") public String type; - @OpenAPIDescription("Completeness.") + @OpenAPIDescription("Non-empty share as a 0..1 ratio rounded to six decimals.") public BigDecimal completeness; @OpenAPIDescription("Distinct.") public Long distinct; - @OpenAPIDescription("Uniqueness.") + @OpenAPIDescription("Distinct share as a 0..1 ratio rounded to six decimals.") public BigDecimal uniqueness; @OpenAPIDescription("Constant.") @@ -634,9 +634,9 @@ public class DataAnalysisSchemas properties.put("field", new Schema().withType(Type.STRING).withDescription("Field.").withNullable(true)); properties.put("label", new Schema().withType(Type.STRING).withDescription("Label.").withNullable(true)); properties.put("type", new Schema().withType(Type.STRING).withDescription("Type.").withNullable(true)); - properties.put("completeness", new Schema().withType(Type.NUMBER).withDescription("Completeness.").withNullable(true)); + properties.put("completeness", new Schema().withType(Type.NUMBER).withDescription("Non-empty share as a 0..1 ratio rounded to six decimals.").withNullable(true)); properties.put("distinct", new Schema().withType(Type.INTEGER).withDescription("Distinct.").withNullable(true)); - properties.put("uniqueness", new Schema().withType(Type.NUMBER).withDescription("Uniqueness.").withNullable(true)); + properties.put("uniqueness", new Schema().withType(Type.NUMBER).withDescription("Distinct share as a 0..1 ratio rounded to six decimals.").withNullable(true)); properties.put("constant", new Schema().withType(Type.BOOLEAN).withDescription("Constant.").withNullable(true)); properties.put("empty", new Schema().withType(Type.BOOLEAN).withDescription("Empty.").withNullable(true)); properties.put("nulls", ref("DataAnalysisMetricV1").withDescription("Nulls.").withNullable(true)); @@ -654,16 +654,16 @@ public class DataAnalysisSchemas public Boolean estimate; public String key; - @OpenAPIDescription("Label.") + @OpenAPIDescription("Plain-English metric label, ready to display.") public String label; - @OpenAPIDescription("Value.") + @OpenAPIDescription("Raw numeric or temporal value. PERCENT is a 0..1 ratio rounded to six decimals; DECIMAL field statistics have ten significant digits. DATE_TIME is an ISO instant.") public Serializable value; @OpenAPIDescription("Value type.") public ValueType valueType; - @OpenAPIDescription("Formatted.") + @OpenAPIDescription("Display-ready value, including grouping, field display formats, percent signs and session-zone dates; null when absent or suppressed.") public String formatted; @OpenAPIDescription("Open the matching records using this table and QueryFilter; null when suppressed.") @@ -683,11 +683,11 @@ public class DataAnalysisSchemas { Map properties = new LinkedHashMap<>(); properties.put("key", new Schema().withType(Type.STRING).withDescription("Key.").withNullable(true)); - properties.put("label", new Schema().withType(Type.STRING).withDescription("Label.").withNullable(true)); - properties.put("value", new Schema().withOneOf(List.of(new Schema().withType(Type.STRING), new Schema().withType(Type.NUMBER), new Schema().withType(Type.BOOLEAN), new Schema().withType(Type.OBJECT).withNullable(true))).withDescription("Value.").withNullable(true)); + properties.put("label", new Schema().withType(Type.STRING).withDescription("Plain-English metric label, ready to display.").withNullable(true)); + properties.put("value", new Schema().withOneOf(List.of(new Schema().withType(Type.STRING), new Schema().withType(Type.NUMBER), new Schema().withType(Type.BOOLEAN), new Schema().withType(Type.OBJECT).withNullable(true))).withDescription("Raw numeric or temporal value. PERCENT is a 0..1 ratio rounded to six decimals; DECIMAL field statistics have ten significant digits. DATE_TIME is an ISO instant.").withNullable(true)); properties.put("valueType", new Schema().withType(Type.STRING).withEnumValues(java.util.Arrays.stream(ValueType.values()).map(Enum::name).toList()).withDescription("Value type.").withNullable(true)); properties.put("estimate", new Schema().withType(Type.BOOLEAN).withDescription("Whether this metric is estimated; the population rows metric remains exact.")); - properties.put("formatted", new Schema().withType(Type.STRING).withDescription("Formatted.").withNullable(true)); + properties.put("formatted", new Schema().withType(Type.STRING).withDescription("Display-ready value, including grouping, field display formats, percent signs and session-zone dates; null when absent or suppressed.").withNullable(true)); properties.put("drill", ref("DataAnalysisDrillV1").withDescription("Open the matching records using this table and QueryFilter; null when suppressed.").withNullable(true)); properties.put("suppressed", new Schema().withType(Type.BOOLEAN).withDescription("Suppressed.").withNullable(true)); properties.put("suppressedReason", new Schema().withType(Type.STRING).withDescription("Suppressed reason.").withNullable(true)); @@ -961,7 +961,7 @@ public class DataAnalysisSchemas @OpenAPIDescription("Count.") public Long count; - @OpenAPIDescription("Share.") + @OpenAPIDescription("Share as a 0..1 ratio rounded to six decimals.") public BigDecimal share; @OpenAPIDescription("Open the matching records using this table and QueryFilter; null when suppressed.") @@ -977,7 +977,7 @@ public class DataAnalysisSchemas properties.put("value", new Schema().withOneOf(List.of(new Schema().withType(Type.STRING), new Schema().withType(Type.NUMBER), new Schema().withType(Type.BOOLEAN), new Schema().withType(Type.OBJECT).withNullable(true))).withDescription("Value.").withNullable(true)); properties.put("label", new Schema().withType(Type.STRING).withDescription("Label.").withNullable(true)); properties.put("count", new Schema().withType(Type.INTEGER).withDescription("Count.").withNullable(true)); - properties.put("share", new Schema().withType(Type.NUMBER).withDescription("Share.").withNullable(true)); + properties.put("share", new Schema().withType(Type.NUMBER).withDescription("Share as a 0..1 ratio rounded to six decimals.").withNullable(true)); properties.put("drill", ref("DataAnalysisDrillV1").withDescription("Open the matching records using this table and QueryFilter; null when suppressed.").withNullable(true)); return (new Schema().withType(Type.OBJECT).withNullable(true).withProperties(properties)); } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/app/DataAnalysisHome.java b/src/main/java/com/kingsrook/qbits/dataanalysis/app/DataAnalysisHome.java index b8d0bb8..fa345a9 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/app/DataAnalysisHome.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/app/DataAnalysisHome.java @@ -22,26 +22,33 @@ package com.kingsrook.qbits.dataanalysis.app; import java.time.Instant; -import java.time.temporal.ChronoUnit; +import java.util.Arrays; import java.util.List; +import java.util.Locale; +import java.util.stream.Collectors; import com.kingsrook.qbits.dataanalysis.DataAnalysisQBitConfig; +import com.kingsrook.qbits.dataanalysis.engine.AnalysisFormatting; import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables; import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType; import com.kingsrook.qqq.backend.core.actions.tables.CountAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; import com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML; import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; @@ -93,7 +100,8 @@ public class DataAnalysisHome extends AbstractWidgetRenderer /******************************************************************************* - ** Standard QQQ widget renderer entry point. + ** RawHTML is carried as a JSON string by the HTML widget, not a text/html response. + ** Entities keep static separators independent of any host charset handling. *******************************************************************************/ @Override public RenderWidgetOutput render(RenderWidgetInput input) throws QException @@ -110,7 +118,7 @@ public class DataAnalysisHome extends AbstractWidgetRenderer { StringBuilder html = new StringBuilder("
"); String userId = QContext.getQSession() == null || QContext.getQSession().getUser() == null ? null : QContext.getQSession().getUser().getIdReference(); - html.append("

Saved analyses

"); + html.append("

Saved analyses

"); if(userId == null) { html.append("

Sign in to see your saved analyses and stars.

"); @@ -119,7 +127,7 @@ public class DataAnalysisHome extends AbstractWidgetRenderer { int mine = count(DataAnalysisTables.SAVED, equal("owner_user_id", userId)); int shared = count(DataAnalysisTables.SAVED, equal("shared", true)); - html.append("

Mine: ").append(mine).append(" · Shared (visible to you): ").append(shared).append("

"); + html.append("

Mine: ").append(AnalysisFormatting.number(mine, 0)).append(" · Shared (visible to you): ").append(AnalysisFormatting.number(shared, 0)).append("

"); if(canRead(DataAnalysisTables.SAVED_STAR)) { CountInput stars = new CountInput(DataAnalysisTables.SAVED).withIncludeDistinctCount(true) @@ -127,28 +135,28 @@ public class DataAnalysisHome extends AbstractWidgetRenderer .withFilter(equal(DataAnalysisTables.SAVED_STAR + ".user_id", userId)); stars.setPermissionEnforcement(PermissionEnforcement.ENFORCE); int starred = new CountAction().execute(stars).getDistinctCount(); - html.append("

Starred by me: ").append(starred).append("

"); + html.append("

Starred by me: ").append(AnalysisFormatting.number(starred, 0)).append("

"); } if(mine == 0 && shared == 0) { - html.append("

No saved analyses yet. Saving analyses from the Analyze action is planned for a later release.

"); + html.append("

No saved analyses yet. Open any table and choose Analyze on a column header to explore its data.

"); } - html.append("

The Analyze action will be available in a later UI release.

"); link(html, DataAnalysisTables.SAVED); } else { html.append("

Saved analyses are unavailable: you need data_analysis.saved.read.

"); } - html.append("

Watches

"); + html.append("

Watches

"); if(canRead(DataAnalysisTables.WATCH)) { + int total = count(DataAnalysisTables.WATCH, new QQueryFilter()); int enabled = count(DataAnalysisTables.WATCH, equal("enabled", true)); int firing = count(DataAnalysisTables.WATCH, equal("state", "FIRING")); - html.append("

Enabled: ").append(enabled).append(" · Firing: ").append(firing).append("

"); - if(enabled == 0 && firing == 0) + html.append("

Total: ").append(AnalysisFormatting.number(total, 0)).append(" · Enabled: ").append(AnalysisFormatting.number(enabled, 0)).append(" · Firing: ").append(AnalysisFormatting.number(firing, 0)).append("

"); + if(total == 0) { - html.append("

No active watches: watch evaluation is planned for phase e.

"); + html.append("

No watches yet.

"); } link(html, DataAnalysisTables.WATCH); } @@ -156,15 +164,37 @@ public class DataAnalysisHome extends AbstractWidgetRenderer { html.append("

Watches are unavailable: you need data_analysis.watch.read.

"); } - html.append("

Runs in the last 7 days

"); - if(canRead(DataAnalysisTables.RUN)) + html.append("

Recent analysis runs

"); + if(userId == null) { - int complete = count(DataAnalysisTables.RUN, recentRuns("COMPLETE", now)); - int failed = count(DataAnalysisTables.RUN, recentRuns("FAILED", now)); - html.append("

Complete: ").append(complete).append(" · Failed: ").append(failed).append("

"); - if(complete == 0 && failed == 0) + html.append("

Sign in to see your recent analysis runs.

"); + } + else if(canRead(DataAnalysisTables.RUN)) + { + QueryInput query = new QueryInput(DataAnalysisTables.RUN).withFilter(equal("user_id", userId) + .withCriteria(new QFilterCriteria("started_at", QCriteriaOperator.LESS_THAN_OR_EQUALS, now)) + .withOrderBy(new QFilterOrderBy("started_at", false)).withOrderBy(new QFilterOrderBy("id", false)).withLimit(5)); + query.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + List runs = new QueryAction().execute(query).getRecords(); + if(runs.isEmpty()) { - html.append("

No completed or failed runs in this period.

"); + html.append("

No analysis runs yet.

"); + } + else + { + html.append("
    "); + for(QRecord run : runs) + { + QTableMetaData table = QContext.getQInstance().getTable(run.getValueString("table_name")); + String tableLabel = table == null ? run.getValueString("table_name") : table.getLabel(); + String path = QContext.getQInstance().getTablePath(DataAnalysisTables.RUN) + "/" + run.getValueInteger("id"); + html.append("
  • ").append(escape(tableLabel)).append(" · ") + .append(escape(words(run.getValueString("operation")))).append("
    Status
    ") + .append(escape(words(run.getValueString("status")))).append("
    Finished
    ") + .append(run.getValue("finished_at") == null ? "Not finished" : escape(AnalysisFormatting.instant(run.getValueInstant("finished_at")))) + .append("
    • "); + } + html.append("
"); } link(html, DataAnalysisTables.RUN); } @@ -198,13 +228,22 @@ public class DataAnalysisHome extends AbstractWidgetRenderer /******************************************************************************* - ** Runs started within the inclusive seven-day interval, excluding future rows. + ** Stored operation and status keys are displayed as ordinary words. *******************************************************************************/ - private static QQueryFilter recentRuns(String status, Instant now) + private static String words(String value) { - return (equal("status", status) - .withCriteria(new QFilterCriteria("started_at", QCriteriaOperator.GREATER_THAN_OR_EQUALS, now.minus(7, ChronoUnit.DAYS))) - .withCriteria(new QFilterCriteria("started_at", QCriteriaOperator.LESS_THAN_OR_EQUALS, now))); + return (value == null ? "Unknown" : Arrays.stream(value.toLowerCase(Locale.ROOT).split("_")) + .filter(part -> !part.isEmpty()).map(part -> Character.toUpperCase(part.charAt(0)) + part.substring(1)).collect(Collectors.joining(" "))); + } + + + + /******************************************************************************* + ** Metadata and persisted values must never become executable widget markup. + *******************************************************************************/ + private static String escape(String value) + { + return (value == null ? "Unknown" : value.replace("&", "&").replace("\"", """).replace("<", "<").replace(">", ">").replace("'", "'")); } @@ -227,7 +266,7 @@ public class DataAnalysisHome extends AbstractWidgetRenderer private static void link(StringBuilder html, String table) throws QException { String path = QContext.getQInstance().getTablePath(table); - html.append("", ">")) + html.append("View all"); } } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisBackend.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisBackend.java index 14ebe27..77f122c 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisBackend.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisBackend.java @@ -38,7 +38,7 @@ public interface AnalysisBackend *******************************************************************************/ enum DateUnit { - DAY, WEEK, MONTH, YEAR, DAY_OF_WEEK, HOUR_OF_DAY + DAY, WEEK, MONTH, QUARTER, YEAR, HOUR, DAY_OF_WEEK, HOUR_OF_DAY } @@ -120,6 +120,15 @@ public interface AnalysisBackend + /******************************************************************************* + ** Actual inspected non-null values, independent of estimated population counts. + *******************************************************************************/ + record PatternSummary(List groups, long values, boolean truncated) + { + } + + + /******************************************************************************* ** Filtered count; a native estimate scales conditional counts to its population. *******************************************************************************/ @@ -168,7 +177,7 @@ public interface AnalysisBackend /******************************************************************************* ** Top shapes over a bounded primary-key prefix. *******************************************************************************/ - List patterns(String table, QQueryFilter filter, String field, int sampleRows) throws QException; + PatternSummary patterns(String table, QQueryFilter filter, String field, int sampleRows) throws QException; /******************************************************************************* ** Top groups with count greater than one. diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisFormatting.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisFormatting.java new file mode 100644 index 0000000..badae57 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisFormatting.java @@ -0,0 +1,225 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.io.Serializable; +import java.math.BigDecimal; +import java.math.MathContext; +import java.math.RoundingMode; +import java.time.DayOfWeek; +import java.time.Instant; +import java.time.LocalDate; +import java.time.Period; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.format.TextStyle; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import com.kingsrook.qbits.dataanalysis.result.Metric; +import com.kingsrook.qbits.dataanalysis.result.ValueType; +import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + +/******************************************************************************* + ** Display-ready values shared by metrics, chart labels and the home widget. + *******************************************************************************/ +public final class AnalysisFormatting +{ + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private AnalysisFormatting() + { + } + + + + /******************************************************************************* + ** Ratios remain numeric and bounded, with stable JSON precision. + *******************************************************************************/ + public static BigDecimal ratio(BigDecimal value) + { + return (value.max(BigDecimal.ZERO).min(BigDecimal.ONE).setScale(6, RoundingMode.HALF_UP).stripTrailingZeros()); + } + + + + /******************************************************************************* + ** Plain numeric defaults are independent of the server locale. + *******************************************************************************/ + public static String number(Number value, int decimals) + { + return (String.format(Locale.US, "%,." + decimals + "f", new BigDecimal(value.toString()))); + } + + + + /******************************************************************************* + ** Types and length keys determine display without changing numeric raw values. + *******************************************************************************/ + public static String scalar(String key, Serializable value, ValueType type) + { + if(value == null) + { + return (null); + } + if(type == ValueType.PERCENT) + { + BigDecimal percent = new BigDecimal(value.toString()).multiply(BigDecimal.valueOf(100)).setScale(1, RoundingMode.HALF_UP); + return (percent.signum() == 0 || percent.compareTo(BigDecimal.valueOf(100)) == 0 ? percent.toBigInteger() + "%" : percent.toPlainString() + "%"); + } + if(type == ValueType.DATE_TIME) + { + return (instant((Instant) value)); + } + if(value instanceof Number n) + { + return (number(n, "meanLength".equals(key) ? 1 : type == ValueType.COUNT || key.endsWith("Length") ? 0 : 2)); + } + return (value.toString()); + } + + + + /******************************************************************************* + ** Display formats belong to the host field; fallback defaults preserve fractions. + *******************************************************************************/ + public static String numeric(QFieldMetaData field, String key, BigDecimal value) + { + if(value == null) + { + return (null); + } + String format = field.getDisplayFormat(); + if(format != null && !format.isBlank() && !DisplayFormat.DEFAULT.equals(format)) + { + Serializable displayValue = format.endsWith("d") ? value.toBigInteger() : value; + return (QValueFormatter.formatValue(format, displayValue)); + } + return (number(value, field.getType() == QFieldType.DECIMAL || "mean".equals(key) || "stddev".equals(key) ? 2 : 0)); + } + + + + /******************************************************************************* + ** Decimal statistics retain ten significant digits; formatting uses the original. + *******************************************************************************/ + public static Metric numericMetric(QFieldMetaData field, String key, BigDecimal value) + { + BigDecimal raw = value == null || field.getType() != QFieldType.DECIMAL ? value : value.round(new MathContext(10)); + return (ProfileSupport.metric(key, raw, ValueType.DECIMAL).withFormatted(numeric(field, key, value))); + } + + + + /******************************************************************************* + ** Use the same field formatter on both ends of every histogram interval. + *******************************************************************************/ + public static String interval(QFieldMetaData field, BigDecimal low, BigDecimal high) + { + return (low == null ? "Below " + edge(field, high) + : high == null ? edge(field, low) + " and above" + : edge(field, low) + " – " + edge(field, high)); + } + + + + /******************************************************************************* + ** Whole edges omit redundant cents; unformatted fractional edges stay distinct. + *******************************************************************************/ + private static String edge(QFieldMetaData field, BigDecimal value) + { + String format = field.getDisplayFormat(); + int scale = Math.max(0, value.stripTrailingZeros().scale()); + if(format == null || format.isBlank() || DisplayFormat.DEFAULT.equals(format)) + { + return (number(value, scale)); + } + String formatted = numeric(field, "min", value); + return (scale == 0 && DisplayFormat.CURRENCY.equals(format) && formatted.endsWith(".00") + ? formatted.substring(0, formatted.length() - 3) : formatted); + } + + + + /******************************************************************************* + ** Instants remain UTC in raw values and display in the current session zone. + *******************************************************************************/ + public static String instant(Instant value) + { + return (DateTimeFormatter.ofPattern("uuuu-MM-dd HH:mm", Locale.US).format(value.atZone(ValueUtils.getSessionOrInstanceZoneId()))); + } + + + + /******************************************************************************* + ** Calendar duration avoids misleading fixed-length month and year arithmetic. + *******************************************************************************/ + public static String duration(LocalDate first, LocalDate last) + { + Period period = Period.between(first, last); + List parts = new ArrayList<>(); + if(period.getYears() != 0) + { + parts.add(period.getYears() + (period.getYears() == 1 ? " year" : " years")); + } + if(period.getMonths() != 0) + { + parts.add(period.getMonths() + (period.getMonths() == 1 ? " month" : " months")); + } + if(period.getDays() != 0 || parts.isEmpty()) + { + parts.add(period.getDays() + (period.getDays() == 1 ? " day" : " days")); + } + return (String.join(" ", parts)); + } + + + + /******************************************************************************* + ** Calendar labels describe one bucket; the raw value remains its ISO start. + *******************************************************************************/ + public static String dateBucket(Serializable start, AnalysisBackend.DateUnit unit, ZoneId zone) + { + if(unit == AnalysisBackend.DateUnit.DAY_OF_WEEK) + { + return (DayOfWeek.of(((Number) start).intValue()).getDisplayName(TextStyle.FULL, Locale.US)); + } + if(unit == AnalysisBackend.DateUnit.HOUR_OF_DAY) + { + return (String.format(Locale.US, "%02d:00", ((Number) start).intValue())); + } + LocalDate date = ProfileSupport.day(start, zone); + return (switch(unit) + { + case YEAR -> Integer.toString(date.getYear()); + case QUARTER -> date.getYear() + " Q" + ((date.getMonthValue() - 1) / 3 + 1); + case MONTH -> DateTimeFormatter.ofPattern("MMM uuuu", Locale.US).format(date); + case WEEK -> "Week of " + date; + case HOUR -> DateTimeFormatter.ofPattern("uuuu-MM-dd HH:00", Locale.US).format(((Instant) start).atZone(zone)); + default -> date.toString(); + }); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ColumnProfiler.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ColumnProfiler.java index 55edeb3..27945b2 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ColumnProfiler.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ColumnProfiler.java @@ -86,7 +86,7 @@ public class ColumnProfiler QFieldType type = field.getType(); if(type == QFieldType.INTEGER || type == QFieldType.LONG || type == QFieldType.DECIMAL) { - numeric(result, filter, drillFilter, rows); + numeric(result, filter, drillFilter, rows, field); } else if(type == QFieldType.DATE || type == QFieldType.DATE_TIME) { @@ -95,13 +95,18 @@ public class ColumnProfiler else if(type.isStringLike()) { AnalysisBackend.LengthSummary summary = backend.lengthSummary(table, filter, name); + AnalysisBackend.PatternSummary patterns = backend.patterns(table, filter, name, 10_000); + long patternValues = patterns.values(); TextProfile text = new TextProfile().withHistogram(ProfileSupport.values(table, drillFilter, null, summary.histogram(), rows)) - .withPatterns(ProfileSupport.values(table, drillFilter, null, backend.patterns(table, filter, name, 10_000), Math.min(rows, 10_000))) - .withSampleRows(Math.min(scope.rowsScanned(), 10_000)); + .withPatterns(ProfileSupport.values(table, drillFilter, null, patterns.groups(), patternValues)) + .withSampleRows(patternValues); summary.metrics().forEach((key, value) -> text.getMetrics().add(ProfileSupport.metric(key, value, ValueType.DECIMAL))); result.withText(text); - result.getWarnings().add(scope.samplePercent() == null ? "Patterns use the first " + Math.min(rows, 10_000) + " rows by primary key." - : "Patterns use up to " + Math.min(scope.rowsScanned(), 10_000) + " sampled rows."); + if(patterns.truncated()) + { + result.getWarnings().add(scope.samplePercent() == null ? "Patterns are based on the first " + AnalysisFormatting.number(patternValues, 0) + " values." + : "Patterns are based on " + AnalysisFormatting.number(patternValues, 0) + " sampled values."); + } } else if(type == QFieldType.BOOLEAN) { @@ -142,14 +147,15 @@ public class ColumnProfiler else { result.withChart(ProfileSupport.chart(ChartKind.TIME_SERIES, ChartKind.BAR, name, result.getDates().getBuckets())); + result.getChart().getX().withLabel(field.getLabel() + (result.getDates().getUnit() == null ? "" : " (" + result.getDates().getUnit().toLowerCase(java.util.Locale.ROOT) + ")")); } - if(rows == 0) + if(scope.total() == 0) { result.getWarnings().add("No readable records match this analysis scope."); } - else if(counts.nonNull() == 0 && (!scope.estimate() || scope.rowsScanned() > 0)) + else if(counts.nonNull() == 0 && rows > 0 && (!scope.estimate() || scope.rowsScanned() > 0)) { - result.getWarnings().add("All values in this analysis scope are null."); + result.getWarnings().add(scope.estimate() ? "All observed values are null." : "All values in this analysis scope are null."); } if(scope.estimate()) { @@ -170,16 +176,16 @@ public class ColumnProfiler /******************************************************************************* - ** Numeric moments, twenty interior bins, exterior bins and Tukey outliers. + ** Numeric moments, nice interior bins, exterior bins and Tukey outliers. *******************************************************************************/ - private void numeric(ColumnProfile result, QQueryFilter filter, QQueryFilter drillFilter, long rows) throws QException + private void numeric(ColumnProfile result, QQueryFilter filter, QQueryFilter drillFilter, long rows, QFieldMetaData metadata) throws QException { String table = result.getTableName(); String field = result.getFieldName(); AnalysisBackend.NumericSummary summary = backend.numericSummary(table, filter, field, List.of("0.01", "0.05", "0.25", "0.50", "0.75", "0.95", "0.99").stream().map(BigDecimal::new).toList()); NumericProfile numeric = new NumericProfile(); - summary.metrics().forEach((key, value) -> numeric.getMetrics().add(ProfileSupport.metric(key, value, ValueType.DECIMAL))); + summary.metrics().forEach((key, value) -> numeric.getMetrics().add(AnalysisFormatting.numericMetric(metadata, key, value))); for(AnalysisBackend.GroupCount count : summary.counts()) { numeric.getMetrics().add(ProfileSupport.count(count.label(), count.count(), table, drillFilter, count.condition())); @@ -211,7 +217,7 @@ public class ColumnProfiler LocalDate last = ProfileSupport.day(extrema.get("max"), zone); AnalysisBackend.DateUnit unit = autoBucket(first, last); dates.withUnit(unit.name()); - dates.getMetrics().add(ProfileSupport.metric("range", ChronoUnit.DAYS.between(first, last), ValueType.DURATION)); + dates.getMetrics().add(ProfileSupport.metric("range", ChronoUnit.DAYS.between(first, last), ValueType.DURATION).withFormatted(AnalysisFormatting.duration(first, last))); List buckets = ProfileSupport.values(table, drillFilter, null, backend.dateBuckets(table, filter, field, unit, zone), rows); dates.withBuckets(buckets).withGaps(buckets.stream().filter(b -> b.getCount() == 0).toList()); if(type == QFieldType.DATE_TIME) diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/CoreAnalysisBackend.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/CoreAnalysisBackend.java index e5f9201..3520581 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/CoreAnalysisBackend.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/CoreAnalysisBackend.java @@ -85,7 +85,7 @@ public class CoreAnalysisBackend extends StandInAnalysisBackend { samplePercent = null; long total = super.count(table, filter); - if(exact || total <= config.getSampleThresholdRows() || !capabilities(table).contains(AggregateCapability.SAMPLING)) + if(exact || total <= config.getSampleThresholdRows() || total <= config.getSampleRows() || !capabilities(table).contains(AggregateCapability.SAMPLING)) { return (super.scope(table, filter, exact)); } @@ -375,21 +375,25 @@ public class CoreAnalysisBackend extends StandInAnalysisBackend { return (super.numericHistogram(table, filter, field, low, high, bins)); } + NiceBins nice = NiceBins.covering(low, high); + low = nice.low(); + high = nice.high(); + bins = nice.count(); GroupBy group = new GroupBy(field(table, field)).withBucket(new NumericBucket(low, high, bins)); Map counts = bucketCounts(table, filter, group); List values = !estimating && isMemory(table) ? rows(table, filter, Set.of(field), config.getMaxRows()).stream().map(r -> r.getValue(field)) .filter(Objects::nonNull).map(v -> new BigDecimal(v.toString())).toList() : List.of(); List result = new ArrayList<>(); - result.add(new GroupCount("Below " + low, "Below " + low, aggregateCount(counts.getOrDefault(0, 0L)), numericRange(table, field, values, null, low, false), aggregateShare(counts.getOrDefault(0, 0L)))); + result.add(new GroupCount(AnalysisFormatting.interval(field(table, field), null, low), AnalysisFormatting.interval(field(table, field), null, low), aggregateCount(counts.getOrDefault(0, 0L)), numericRange(table, field, values, null, low, false), aggregateShare(counts.getOrDefault(0, 0L)))); BigDecimal width = high.subtract(low).divide(BigDecimal.valueOf(bins), MathContext.DECIMAL128); for(int i = 0; i < bins; i++) { BigDecimal start = low.add(width.multiply(BigDecimal.valueOf(i))); BigDecimal end = i == bins - 1 ? high : low.add(width.multiply(BigDecimal.valueOf(i + 1))); - String label = "[" + start + ", " + end + (i == bins - 1 ? "]" : ")"); + String label = AnalysisFormatting.interval(field(table, field), start, end); result.add(new GroupCount(label, label, aggregateCount(counts.getOrDefault(i + 1, 0L)), numericRange(table, field, values, start, end, i == bins - 1), aggregateShare(counts.getOrDefault(i + 1, 0L)))); } - result.add(new GroupCount("Above " + high, "Above " + high, aggregateCount(counts.getOrDefault(bins + 1, 0L)), + result.add(new GroupCount(AnalysisFormatting.interval(field(table, field), high, null), AnalysisFormatting.interval(field(table, field), high, null), aggregateCount(counts.getOrDefault(bins + 1, 0L)), DrillFilters.and(DrillFilters.condition(field, QCriteriaOperator.IS_NOT_BLANK), DrillFilters.condition(field, QCriteriaOperator.GREATER_THAN, high)), aggregateShare(counts.getOrDefault(bins + 1, 0L)))); return (result); } @@ -400,6 +404,10 @@ public class CoreAnalysisBackend extends StandInAnalysisBackend @Override public List dateBuckets(String table, QQueryFilter filter, String field, DateUnit unit, ZoneId zone) throws QException { + if(unit == DateUnit.HOUR || unit == DateUnit.QUARTER) + { + return (super.dateBuckets(table, filter, field, unit, zone)); + } GroupBy group = new GroupBy(field(table, field)).withBucket(new DateBucket(DateBucketUnit.valueOf(unit.name()))); AggregateInput probe = new AggregateInput(table).withGroupBy(group); if(!new QBackendModuleDispatcher().getQBackendModule(probe.getBackend()).getAggregateInterface().getSupportedCapabilities(probe).contains(AggregateCapability.DATE_BUCKET)) @@ -423,8 +431,8 @@ public class CoreAnalysisBackend extends StandInAnalysisBackend while(!start.isAfter(dates.getLast())) { LocalDate end = next(start, unit); - String label = start + " / " + end; - result.add(new GroupCount(label, label, aggregateCount(counts.getOrDefault(start, 0L)), + String label = AnalysisFormatting.dateBucket(start, unit, zone); + result.add(new GroupCount(start, label, aggregateCount(counts.getOrDefault(start, 0L)), DrillFilters.dateRange(field, start, end, field(table, field).getType() == QFieldType.DATE_TIME, zone), aggregateShare(counts.getOrDefault(start, 0L)))); start = end; } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MetricLabels.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MetricLabels.java new file mode 100644 index 0000000..2b7ce09 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MetricLabels.java @@ -0,0 +1,65 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.util.Map; +import java.util.Objects; +import com.kingsrook.qbits.dataanalysis.result.ValueType; + +/******************************************************************************* + ** One vocabulary for every emitted metric, including date-specific extrema. + *******************************************************************************/ +public final class MetricLabels +{ + private static final Map LABELS = Map.ofEntries( + Map.entry("rows", "Rows"), Map.entry("nonNull", "Non-empty"), Map.entry("nulls", "Nulls"), + Map.entry("blanks", "Blanks"), Map.entry("distinct", "Distinct values"), Map.entry("uniqueness", "Uniqueness"), + Map.entry("completeness", "Completeness"), Map.entry("share", "Share"), + Map.entry("min", "Minimum"), Map.entry("max", "Maximum"), Map.entry("mean", "Mean"), + Map.entry("median", "Median"), Map.entry("stddev", "Standard deviation"), Map.entry("sum", "Sum"), + Map.entry("zeros", "Zeros"), Map.entry("negatives", "Negatives"), Map.entry("outliers", "Outliers"), + Map.entry("p1", "1st percentile"), Map.entry("p5", "5th percentile"), Map.entry("p25", "25th percentile (Q1)"), + Map.entry("p50", "Median"), Map.entry("p75", "75th percentile (Q3)"), Map.entry("p95", "95th percentile"), Map.entry("p99", "99th percentile"), + Map.entry("minLength", "Shortest"), Map.entry("maxLength", "Longest"), Map.entry("meanLength", "Average length"), + Map.entry("earliest", "Earliest"), Map.entry("latest", "Latest"), Map.entry("range", "Range"), + Map.entry("true", "True"), Map.entry("false", "False"), Map.entry("other", "Other")); + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private MetricLabels() + { + } + + + + /******************************************************************************* + ** Fail at construction when a new metric lacks a human label. + *******************************************************************************/ + public static String label(String key, ValueType type) + { + if(type == ValueType.DATE || type == ValueType.DATE_TIME) + { + key = "min".equals(key) ? "earliest" : "max".equals(key) ? "latest" : key; + } + return (Objects.requireNonNull(LABELS.get(key), "Missing metric label: " + key)); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MinGroupSizeGuard.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MinGroupSizeGuard.java index c9a82a2..3331d96 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MinGroupSizeGuard.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MinGroupSizeGuard.java @@ -71,7 +71,7 @@ public class MinGroupSizeGuard } if(hidden > 0) { - result.add(new ValueCount().withLabel("Other (" + hidden + " groups below " + minimum + ")").withCount(count).withShare(share)); + result.add(new ValueCount().withLabel("Other (" + hidden + " groups below " + minimum + ")").withCount(count).withShare(AnalysisFormatting.ratio(share))); } return (result); } @@ -114,7 +114,7 @@ public class MinGroupSizeGuard List booleanGroups = profile.getBooleans().stream().map(m -> new ValueCount().withValue(m.getKey()).withLabel(m.getLabel()) .withCount(((Number) m.getValue()).longValue()).withShare(ProfileSupport.share(((Number) m.getValue()).longValue(), rows)).withDrill(m.getDrill())).toList(); profile.withBooleans(groups(booleanGroups).stream().map(v -> ProfileSupport.metric(v.getValue() == null ? "other" : v.getValue().toString(), v.getCount(), - com.kingsrook.qbits.dataanalysis.result.ValueType.COUNT).withLabel(v.getLabel()).withDrill(v.getDrill())).toList()); + com.kingsrook.qbits.dataanalysis.result.ValueType.COUNT).withDrill(v.getDrill())).toList()); } } if(profile.getNumeric() != null) diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/NiceBins.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/NiceBins.java new file mode 100644 index 0000000..76807f3 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/NiceBins.java @@ -0,0 +1,62 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.List; + +/******************************************************************************* + ** Exact decimal boundaries with a readable step and approximately twenty bins. + *******************************************************************************/ +public record NiceBins(BigDecimal low, BigDecimal high, int count) +{ + /******************************************************************************* + ** Select the closest 10..25-bin enclosure from 1, 2, 2.5 and 5 times powers of ten. + *******************************************************************************/ + public static NiceBins covering(BigDecimal low, BigDecimal high) + { + BigDecimal span = high.subtract(low); + if(span.signum() == 0) + { + span = low.abs().max(BigDecimal.ONE); + low = low.subtract(span.divide(BigDecimal.valueOf(2))); + high = high.add(span.divide(BigDecimal.valueOf(2))); + } + int exponent = span.precision() - span.scale() - 2; + NiceBins best = null; + for(int power = exponent - 2; power <= exponent + 2; power++) + { + for(String factor : List.of("1", "2", "2.5", "5")) + { + BigDecimal step = new BigDecimal(factor).scaleByPowerOfTen(power); + BigDecimal first = low.divide(step, 0, RoundingMode.FLOOR).multiply(step); + BigDecimal last = high.divide(step, 0, RoundingMode.CEILING).multiply(step); + int count = last.subtract(first).divide(step).intValueExact(); + if(count >= 10 && count <= 25 && (best == null || Math.abs(count - 20) < Math.abs(best.count() - 20))) + { + best = new NiceBins(first, last, count); + } + } + } + return (java.util.Objects.requireNonNull(best)); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ProfileSupport.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ProfileSupport.java index 83c8b59..716de31 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ProfileSupport.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ProfileSupport.java @@ -56,7 +56,7 @@ public final class ProfileSupport *******************************************************************************/ public static BigDecimal share(long count, long rows) { - return (rows == 0 ? BigDecimal.ZERO : BigDecimal.valueOf(count).divide(BigDecimal.valueOf(rows), MathContext.DECIMAL128)); + return (rows == 0 ? BigDecimal.ZERO : BigDecimal.valueOf(count).divide(BigDecimal.valueOf(rows), 6, java.math.RoundingMode.HALF_UP).stripTrailingZeros()); } @@ -76,7 +76,12 @@ public final class ProfileSupport *******************************************************************************/ public static Metric metric(String key, Serializable value, ValueType type) { - return (new Metric().withKey(key).withLabel(key).withValue(value).withValueType(type).withFormatted(value == null ? null : value.toString())); + if(type == ValueType.PERCENT && value != null) + { + value = AnalysisFormatting.ratio(new BigDecimal(value.toString())); + } + return (new Metric().withKey(key).withLabel(MetricLabels.label(key, type)).withValue(value).withValueType(type) + .withFormatted(AnalysisFormatting.scalar(key, value, type))); } @@ -116,7 +121,7 @@ public final class ProfileSupport label = translated; } } - result.add(new ValueCount().withValue(group.value()).withLabel(label).withCount(group.count()).withShare(group.share() == null ? share(group.count(), total) : group.share()) + result.add(new ValueCount().withValue(group.value()).withLabel(label).withCount(group.count()).withShare(group.share() == null ? share(group.count(), total) : AnalysisFormatting.ratio(group.share())) .withDrill(DrillFilters.drill(table, filter, group.condition()))); } return (result); diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/StandInAnalysisBackend.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/StandInAnalysisBackend.java index a8ee186..48b342f 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/StandInAnalysisBackend.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/StandInAnalysisBackend.java @@ -101,7 +101,7 @@ public class StandInAnalysisBackend implements AnalysisBackend public Scope scope(String table, QQueryFilter filter, boolean exact) throws QException { long total = count(table, filter); - boolean estimate = !exact && total > config.getSampleThresholdRows(); + boolean estimate = !exact && total > config.getSampleThresholdRows() && total > Math.min(config.getSampleRows(), config.getMaxRows()); estimating = estimate; int limit = estimate ? Math.min(config.getSampleRows(), config.getMaxRows()) : config.getMaxRows(); QQueryFilter scoped = DrillFilters.copy(filter); @@ -455,17 +455,21 @@ public class StandInAnalysisBackend implements AnalysisBackend { return (result); } + NiceBins nice = NiceBins.covering(low, high); + low = nice.low(); + high = nice.high(); + bins = nice.count(); List values = rows(table, filter, Set.of(field), config.getMaxRows()).stream().map(r -> r.getValue(field)) .filter(Objects::nonNull).map(v -> new BigDecimal(v.toString())).toList(); - result.add(bucket(table, filter, "Below " + low, numericRange(table, field, values, null, low, false))); + result.add(bucket(table, filter, AnalysisFormatting.interval(field(table, field), null, low), numericRange(table, field, values, null, low, false))); BigDecimal width = high.subtract(low).divide(BigDecimal.valueOf(bins), MATH); for(int i = 0; i < bins; i++) { BigDecimal start = low.add(width.multiply(BigDecimal.valueOf(i))); BigDecimal end = i == bins - 1 ? high : low.add(width.multiply(BigDecimal.valueOf(i + 1))); - result.add(bucket(table, filter, "[" + start + ", " + end + (i == bins - 1 ? "]" : ")"), numericRange(table, field, values, start, end, i == bins - 1))); + result.add(bucket(table, filter, AnalysisFormatting.interval(field(table, field), start, end), numericRange(table, field, values, start, end, i == bins - 1))); } - result.add(bucket(table, filter, "Above " + high, DrillFilters.and(DrillFilters.condition(field, QCriteriaOperator.IS_NOT_BLANK), DrillFilters.condition(field, QCriteriaOperator.GREATER_THAN, high)))); + result.add(bucket(table, filter, AnalysisFormatting.interval(field(table, field), high, null), DrillFilters.and(DrillFilters.condition(field, QCriteriaOperator.IS_NOT_BLANK), DrillFilters.condition(field, QCriteriaOperator.GREATER_THAN, high)))); return (result); } @@ -491,7 +495,26 @@ public class StandInAnalysisBackend implements AnalysisBackend if(unit == DateUnit.DAY_OF_WEEK || unit == DateUnit.HOUR_OF_DAY) { return (mappedGroups(table, filter, field, records, v -> unit == DateUnit.DAY_OF_WEEK ? day(v, zone).getDayOfWeek().getValue() - : ((Instant) v).atZone(zone).getHour(), 24)); + : ((Instant) v).atZone(zone).getHour(), 24).stream() + .map(g -> new GroupCount(g.value(), AnalysisFormatting.dateBucket(g.value(), unit, zone), g.count(), g.condition(), g.share())).toList()); + } + if(unit == DateUnit.HOUR) + { + List instants = records.stream().map(r -> (Instant) r.getValue(field)).filter(Objects::nonNull).sorted().toList(); + List hours = new ArrayList<>(); + if(!instants.isEmpty()) + { + Instant start = instants.getFirst().atZone(zone).truncatedTo(java.time.temporal.ChronoUnit.HOURS).toInstant(); + while(!start.isAfter(instants.getLast())) + { + Instant end = start.plus(1, java.time.temporal.ChronoUnit.HOURS); + QQueryFilter condition = DrillFilters.and(DrillFilters.condition(field, QCriteriaOperator.GREATER_THAN_OR_EQUALS, start), + DrillFilters.condition(field, QCriteriaOperator.LESS_THAN, end)); + hours.add(new GroupCount(start, AnalysisFormatting.dateBucket(start, unit, zone), count(table, DrillFilters.and(filter, condition)), condition, countShare())); + start = end; + } + } + return (hours); } List dates = records.stream().map(r -> r.getValue(field)).filter(Objects::nonNull).map(v -> day(v, zone)).sorted().toList(); List result = new ArrayList<>(); @@ -504,7 +527,8 @@ public class StandInAnalysisBackend implements AnalysisBackend while(!start.isAfter(last)) { LocalDate end = next(start, unit); - result.add(bucket(table, filter, start + " / " + end, DrillFilters.dateRange(field, start, end, field(table, field).getType() == QFieldType.DATE_TIME, zone))); + QQueryFilter condition = DrillFilters.dateRange(field, start, end, field(table, field).getType() == QFieldType.DATE_TIME, zone); + result.add(new GroupCount(start, AnalysisFormatting.dateBucket(start, unit, zone), count(table, DrillFilters.and(filter, condition)), condition, countShare())); start = end; } return (result); @@ -531,6 +555,7 @@ public class StandInAnalysisBackend implements AnalysisBackend { case WEEK -> date.with(TemporalAdjusters.previousOrSame(DayOfWeek.MONDAY)); case MONTH -> date.withDayOfMonth(1); + case QUARTER -> date.withMonth((date.getMonthValue() - 1) / 3 * 3 + 1).withDayOfMonth(1); case YEAR -> date.withDayOfYear(1); default -> date; }); @@ -547,6 +572,7 @@ public class StandInAnalysisBackend implements AnalysisBackend { case WEEK -> date.plusWeeks(1); case MONTH -> date.plusMonths(1); + case QUARTER -> date.plusMonths(3); case YEAR -> date.plusYears(1); default -> date.plusDays(1); }); @@ -605,12 +631,14 @@ public class StandInAnalysisBackend implements AnalysisBackend ** Shapes use a primary-key prefix of at most the requested sample size. *******************************************************************************/ @Override - public List patterns(String table, QQueryFilter filter, String field, int sampleRows) throws QException + public PatternSummary patterns(String table, QQueryFilter filter, String field, int sampleRows) throws QException { - List records = rows(table, filter, Set.of(field, primaryKey(table)), sampleRows); + List records = rows(table, DrillFilters.and(filter, DrillFilters.nonNull(field, true)), Set.of(field, primaryKey(table)), sampleRows + 1); + boolean truncated = records.size() > sampleRows; + records = records.subList(0, Math.min(records.size(), sampleRows)); QQueryFilter sampled = records.isEmpty() ? DrillFilters.in(primaryKey(table), List.of()) : DrillFilters.condition(primaryKey(table), QCriteriaOperator.LESS_THAN_OR_EQUALS, records.getLast().getValue(primaryKey(table))); - return (mappedGroups(table, estimating ? filter : DrillFilters.and(filter, sampled), field, records, v -> shape(v.toString()), 10)); + return (new PatternSummary(mappedGroups(table, estimating ? filter : DrillFilters.and(filter, sampled), field, records, v -> shape(v.toString()), 10), records.size(), truncated)); } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/TableProfiler.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/TableProfiler.java index 2560174..14a8a1d 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/TableProfiler.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/TableProfiler.java @@ -70,7 +70,7 @@ public class TableProfiler String name = field.getName(); AnalysisBackend.FieldCounts c = counts.get(name); result.getFields().add(new FieldSummary().withField(name).withLabel(field.getLabel()).withType(field.getType().name()) - .withCompleteness(c.completeness()).withDistinct(c.distinct()).withUniqueness(c.uniqueness()) + .withCompleteness(AnalysisFormatting.ratio(c.completeness())).withDistinct(c.distinct()).withUniqueness(AnalysisFormatting.ratio(c.uniqueness())) .withConstant(c.constant()).withEmpty(c.blanks() == rows) .withNulls(ProfileSupport.count("nulls", c.nulls(), table, drillFilter, DrillFilters.nulls(name, field.getType().isStringLike()))) .withBlanks(ProfileSupport.count("blanks", c.blanks(), table, drillFilter, c.blankFilter()))); diff --git a/src/main/resources/openapi/qqq-v1/data-analysis.yaml b/src/main/resources/openapi/qqq-v1/data-analysis.yaml index 7f0b3df..de5cdc6 100644 --- a/src/main/resources/openapi/qqq-v1/data-analysis.yaml +++ b/src/main/resources/openapi/qqq-v1/data-analysis.yaml @@ -511,7 +511,7 @@ components: nullable: true type: "string" completeness: - description: "Completeness." + description: "Non-empty share as a 0..1 ratio rounded to six decimals." nullable: true type: "number" distinct: @@ -519,7 +519,7 @@ components: nullable: true type: "integer" uniqueness: - description: "Uniqueness." + description: "Distinct share as a 0..1 ratio rounded to six decimals." nullable: true type: "number" constant: @@ -567,11 +567,13 @@ components: nullable: true type: "string" label: - description: "Label." + description: "Plain-English metric label, ready to display." nullable: true type: "string" value: - description: "Value." + description: "Raw numeric or temporal value. PERCENT is a 0..1 ratio rounded\ + \ to six decimals; DECIMAL field statistics have ten significant digits.\ + \ DATE_TIME is an ISO instant." nullable: true oneOf: - type: "string" @@ -596,7 +598,8 @@ components: \ remains exact." type: "boolean" formatted: - description: "Formatted." + description: "Display-ready value, including grouping, field display formats,\ + \ percent signs and session-zone dates; null when absent or suppressed." nullable: true type: "string" drill: @@ -872,7 +875,7 @@ components: nullable: true type: "integer" share: - description: "Share." + description: "Share as a 0..1 ratio rounded to six decimals." nullable: true type: "number" drill: diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/AnalysisFormattingTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/AnalysisFormattingTest.java new file mode 100644 index 0000000..f454bee --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/AnalysisFormattingTest.java @@ -0,0 +1,158 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis; + +import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.dataanalysis.engine.AnalysisBackend; +import com.kingsrook.qbits.dataanalysis.engine.AnalysisFormatting; +import com.kingsrook.qbits.dataanalysis.engine.MetricLabels; +import com.kingsrook.qbits.dataanalysis.engine.NiceBins; +import com.kingsrook.qbits.dataanalysis.engine.ProfileSupport; +import com.kingsrook.qbits.dataanalysis.result.ValueType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/******************************************************************************* + ** Public presentation contracts, including precision and calendar edge cases. + *******************************************************************************/ +class AnalysisFormattingTest +{ + /******************************************************************************* + ** Raw ratios remain numeric while display strings are ready for a CLI. + *******************************************************************************/ + @Test + void numbersPercentagesAndLengths() + { + assertThat(ProfileSupport.metric("rows", 12498L, ValueType.COUNT).getFormatted()).isEqualTo("12,498"); + for(String key : List.of("uniqueness", "completeness", "share")) + { + var metric = ProfileSupport.metric(key, new BigDecimal("0.987654321"), ValueType.PERCENT); + assertThat(metric.getValue()).isEqualTo(new BigDecimal("0.987654")); + assertThat(metric.getFormatted()).isEqualTo("98.8%"); + assertThat(metric.getValueType()).isEqualTo(ValueType.PERCENT); + } + assertThat(ProfileSupport.metric("share", BigDecimal.ONE, ValueType.PERCENT).getFormatted()).isEqualTo("100%"); + assertThat(ProfileSupport.metric("share", BigDecimal.ZERO, ValueType.PERCENT).getFormatted()).isEqualTo("0%"); + assertThat(ProfileSupport.share(1, 3)).isEqualByComparingTo("0.333333"); + assertThat(ProfileSupport.share(0, 0)).isZero(); + assertThat(ProfileSupport.metric("meanLength", new BigDecimal("12.345"), ValueType.DECIMAL).getFormatted()).isEqualTo("12.3"); + assertThat(ProfileSupport.metric("minLength", BigDecimal.TEN, ValueType.DECIMAL).getFormatted()).isEqualTo("10"); + assertThat(ProfileSupport.metric("max", null, ValueType.DECIMAL).getFormatted()).isNull(); + assertThatThrownBy(() -> MetricLabels.label("unknown", ValueType.COUNT)).isInstanceOf(NullPointerException.class).hasMessageContaining("Missing metric label"); + } + + + + /******************************************************************************* + ** Host currency and custom display formats take precedence over numeric defaults. + *******************************************************************************/ + @Test + void fieldFormatsAndRawPrecision() + { + QFieldMetaData decimal = new QFieldMetaData("amount", QFieldType.DECIMAL); + assertThat(AnalysisFormatting.numeric(decimal, "min", new BigDecimal("81753.32"))).isEqualTo("81,753.32"); + decimal.withDisplayFormat(DisplayFormat.CURRENCY); + assertThat(AnalysisFormatting.numeric(decimal, "min", new BigDecimal("81753.32"))).isEqualTo("$81,753.32"); + var metric = AnalysisFormatting.numericMetric(decimal, "mean", new BigDecimal("125916.45167830173")); + assertThat(metric.getValue()).isEqualTo(new BigDecimal("125916.4517")); + assertThat(metric.getFormatted()).isEqualTo("$125,916.45"); + assertThat(AnalysisFormatting.numericMetric(decimal, "max", null).getValue()).isNull(); + QFieldMetaData integer = new QFieldMetaData("quantity", QFieldType.INTEGER); + assertThat(AnalysisFormatting.numeric(integer, "max", new BigDecimal("12498"))).isEqualTo("12,498"); + assertThat(AnalysisFormatting.numeric(integer, "mean", new BigDecimal("12.345"))).isEqualTo("12.35"); + assertThat(AnalysisFormatting.numeric(integer, "stddev", new BigDecimal("12.345"))).isEqualTo("12.35"); + assertThat(AnalysisFormatting.numeric(integer.withDisplayFormat(DisplayFormat.DECIMAL1_COMMAS), "mean", new BigDecimal("12.345"))).isEqualTo("12.3"); + QFieldMetaData large = new QFieldMetaData("large", QFieldType.LONG).withDisplayFormat(DisplayFormat.COMMAS); + assertThat(AnalysisFormatting.numeric(large, "sum", new BigDecimal("9000000000"))).isEqualTo("9,000,000,000"); + } + + + + /******************************************************************************* + ** Nice boundaries remain exact at small scales, negative values and constants. + *******************************************************************************/ + @Test + void niceEdgesAndLabels() + { + for(List range : List.of(List.of("25317.13", "65781.79"), List.of("-0.0093", "0.0357"), List.of("42", "42"), List.of("0", "0"), List.of("-9", "905"))) + { + BigDecimal low = new BigDecimal(range.getFirst()); + BigDecimal high = new BigDecimal(range.getLast()); + NiceBins bins = NiceBins.covering(low, high); + assertThat(bins.count()).isBetween(10, 25); + assertThat(bins.low()).isLessThanOrEqualTo(low); + assertThat(bins.high()).isGreaterThanOrEqualTo(high); + BigDecimal step = bins.high().subtract(bins.low()).divide(BigDecimal.valueOf(bins.count())).stripTrailingZeros(); + assertThat(step.scaleByPowerOfTen(step.scale() - step.precision() + 1).toPlainString()).isIn("1", "2", "2.5", "5"); + assertThat(bins.low().remainder(step)).isZero(); + assertThat(bins.high().remainder(step)).isZero(); + } + QFieldMetaData field = new QFieldMetaData("amount", QFieldType.INTEGER); + assertThat(AnalysisFormatting.interval(field, new BigDecimal("60000"), new BigDecimal("65000"))).isEqualTo("60,000 – 65,000"); + assertThat(AnalysisFormatting.interval(field, null, new BigDecimal("25000"))).isEqualTo("Below 25,000"); + assertThat(AnalysisFormatting.interval(field, new BigDecimal("65000"), null)).isEqualTo("65,000 and above"); + assertThat(AnalysisFormatting.interval(field, new BigDecimal("1.25"), new BigDecimal("1.5"))).isEqualTo("1.25 – 1.5"); + field.withDisplayFormat(DisplayFormat.CURRENCY); + assertThat(AnalysisFormatting.interval(field, new BigDecimal("60000"), new BigDecimal("65000"))).isEqualTo("$60,000 – $65,000"); + } + + + + /******************************************************************************* + ** Every supported unit has a human label without replacing its ISO coordinate. + *******************************************************************************/ + @Test + void dateUnitsAndDurations() + { + ZoneId zone = ZoneId.of("America/Chicago"); + LocalDate date = LocalDate.of(2024, 3, 4); + Map labels = Map.of( + AnalysisBackend.DateUnit.YEAR, "2024", AnalysisBackend.DateUnit.QUARTER, "2024 Q1", AnalysisBackend.DateUnit.MONTH, "Mar 2024", + AnalysisBackend.DateUnit.WEEK, "Week of 2024-03-04", AnalysisBackend.DateUnit.DAY, "2024-03-04"); + labels.forEach((unit, label) -> assertThat(AnalysisFormatting.dateBucket(date, unit, zone)).isEqualTo(label)); + assertThat(AnalysisFormatting.dateBucket(Instant.parse("2024-03-04T19:00:00Z"), AnalysisBackend.DateUnit.HOUR, zone)).isEqualTo("2024-03-04 13:00"); + for(int i = 1; i <= 7; i++) + { + assertThat(AnalysisFormatting.dateBucket(i, AnalysisBackend.DateUnit.DAY_OF_WEEK, zone)) + .isEqualTo(List.of("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday").get(i - 1)); + } + for(int i = 0; i < 24; i++) + { + assertThat(AnalysisFormatting.dateBucket(i, AnalysisBackend.DateUnit.HOUR_OF_DAY, zone)).isEqualTo(String.format(java.util.Locale.US, "%02d:00", i)); + } + assertThat(AnalysisFormatting.duration(date, date.plusYears(48).plusMonths(2))).isEqualTo("48 years 2 months"); + assertThat(AnalysisFormatting.duration(date, date.plusDays(12))).isEqualTo("12 days"); + assertThat(AnalysisFormatting.duration(date, date.plusYears(1).plusMonths(1).plusDays(1))).isEqualTo("1 year 1 month 1 day"); + assertThat(AnalysisFormatting.duration(date, date)).isEqualTo("0 days"); + assertThat(ProfileSupport.metric("min", date, ValueType.DATE).getLabel()).isEqualTo("Earliest"); + assertThat(ProfileSupport.metric("max", date, ValueType.DATE).getLabel()).isEqualTo("Latest"); + assertThat(ProfileSupport.metric("min", date, ValueType.DATE).getFormatted()).isEqualTo("2024-03-04"); + } +} diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/CoreAnalysisBackendTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/CoreAnalysisBackendTest.java index 324d20e..ed2a248 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/CoreAnalysisBackendTest.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/CoreAnalysisBackendTest.java @@ -110,7 +110,7 @@ class CoreAnalysisBackendTest extends EngineFixture } /******************************************************************************* - ** Zero and capped percentages never divide by zero or restrict the row drill. + ** Empty samples keep the population drill; inspecting every row is exact. *******************************************************************************/ @Test void samplingBoundaries() throws Exception @@ -123,7 +123,8 @@ class CoreAnalysisBackendTest extends EngineFixture config.withDefaultSampleRows(100); service.getConfigResolver().invalidate(); var scope = service.backend("orders").scope("orders", null, false); - assertThat(scope.samplePercent()).isEqualByComparingTo("100"); + assertThat(scope.samplePercent()).isNull(); + assertThat(scope.estimate()).isFalse(); assertThat(scope.rowsScanned()).isEqualTo(12); } diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisHomeTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisHomeTest.java index 68c7fd9..f8439b7 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisHomeTest.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisHomeTest.java @@ -33,7 +33,6 @@ import com.kingsrook.qqq.backend.core.model.session.QSession; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; /******************************************************************************* @@ -62,8 +61,12 @@ class DataAnalysisHomeTest QContext.init(instance, TestSupport.user("alice").withPermissions("data_analysis.saved.read", "data_analysis.watch.read", "data_analysis.run.read")); String html = DataAnalysisHome.render(Instant.now()); assertThat(html).contains("Mine: 0", "Starred by me: 0", "No saved analyses yet", - "No active watches", "No completed or failed runs", "View all"); - assertThat(new DataAnalysisHome().render(new RenderWidgetInput())).isNotNull(); + "No watches yet.", "No analysis runs yet.", "View all", "Open any table and choose Analyze on a column header to explore its data.", "

"); + var widget = new DataAnalysisHome().render(new RenderWidgetInput()).getWidgetData(); + assertThat(widget).isInstanceOf(com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML.class); + assertThat(((com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML) widget).getType()).isEqualTo("html"); + assertThat(html).contains("·").doesNotContain("Â", "later", "phase", " c < 128)).isTrue(); QContext.init(instance, TestSupport.user("alice")); html = DataAnalysisHome.render(Instant.now()); assertThat(html).contains("you need data_analysis.saved.read", "you need data_analysis.watch.read").doesNotContain("Mine: 0", "Complete: 0"); @@ -99,11 +102,39 @@ class DataAnalysisHomeTest run("bob", "FAILED", now); QContext.init(instance, TestSupport.user("alice").withPermissions("data_analysis.saved.read", "data_analysis.watch.read", "data_analysis.run.read")); String html = DataAnalysisHome.render(now); - assertThat(html).contains("Mine: 2", "Shared (visible to you): 1", "Starred by me: 1", "Enabled: 2", "Firing: 1", "Complete: 2", "Failed: 1") - .doesNotContain("No saved analyses", "No active watches", "No completed or failed runs"); + assertThat(html).contains("Mine: 2", "Shared (visible to you): 1", "Starred by me: 1", "Enabled: 2", "Firing: 1", "Recent analysis runs", "Complete", "Failed", "Running", "Not finished") + .doesNotContain("No saved analyses", "No watches yet", "No analysis runs yet"); QContext.init(instance, TestSupport.user("bob").withPermissions("data_analysis.saved.read", "data_analysis.run.read")); html = DataAnalysisHome.render(now); - assertThat(html).contains("Mine: 1", "Shared (visible to you): 1", "Starred by me: 0", "Complete: 0", "Failed: 1", "you need data_analysis.watch.read"); + assertThat(html).contains("Mine: 1", "Shared (visible to you): 1", "Starred by me: 0", "Failed", "you need data_analysis.watch.read"); + } + + + + /******************************************************************************* + ** Latest five remain owner-scoped even with administrator bypass and escape HTML. + *******************************************************************************/ + @Test + void recentRunDetailsAndLimit() throws Exception + { + QInstance instance = TestSupport.instance(); + instance.getTable(DataAnalysisTables.SAVED).withLabel("Saved & review"); + Instant now = Instant.parse("2026-10-13T23:10:00Z"); + for(int i = 0; i < 7; i++) + { + TestSupport.insert(DataAnalysisTables.RUN, new QRecord().withValue("table_name", DataAnalysisTables.SAVED) + .withValue("operation", "COLUMN_PROFILE").withValue("user_id", "admin").withValue("status", "COMPLETE") + .withValue("started_at", now.minusSeconds(i * 60L)).withValue("finished_at", now.minusSeconds(i * 60L))); + } + TestSupport.insert(DataAnalysisTables.RUN, new QRecord().withValue("table_name", "private_bob_table") + .withValue("operation", "TABLE_PROFILE").withValue("user_id", "bob").withValue("status", "FAILED").withValue("started_at", now)); + QContext.init(instance, TestSupport.admin().withValue(QSession.VALUE_KEY_USER_TIMEZONE, "America/Chicago")); + String html = DataAnalysisHome.render(now); + assertThat(html).contains("Saved <analysis> & review", "Column Profile", "2026-10-13 18:10", "2026-10-13 18:06", ""); + assertThat(html.split("
  • ", -1)).hasSize(6); + assertThat(html).contains("href=\"" + instance.getTablePath(DataAnalysisTables.RUN) + "/1\""); + assertThat(html).contains("href=\"" + instance.getTablePath(DataAnalysisTables.RUN) + "\">View all"); } @@ -113,7 +144,7 @@ class DataAnalysisHomeTest *******************************************************************************/ private static void run(String user, String status, Instant started) throws Exception { - TestSupport.insert(DataAnalysisTables.RUN, new QRecord().withValue("table_name", "orders").withValue("operation", "profile") + TestSupport.insert(DataAnalysisTables.RUN, new QRecord().withValue("table_name", "orders").withValue("operation", "COLUMN_PROFILE") .withValue("user_id", user).withValue("status", status).withValue("started_at", started)); } } diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java index c5343e5..3275715 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java @@ -64,7 +64,60 @@ import static org.assertj.core.api.Assertions.within; abstract class EngineContractTest extends EngineFixture { /******************************************************************************* - ** All numeric moments and percentiles agree to 1e-9. + ** Every emitted key across all profile kinds has a label, on every backend. + *******************************************************************************/ + @Test + void presentationAcrossProfileKinds() throws Exception + { + java.util.Set keys = new java.util.HashSet<>(); + for(String field : List.of("amount", "quantity", "note", "paid", "order_date", "created")) + { + ColumnProfile profile = service.columnProfile(column(field)); + java.util.List metrics = new java.util.ArrayList<>(profile.getCounts()); + if(profile.getNumeric() != null) + { + metrics.addAll(profile.getNumeric().getMetrics()); + } + if(profile.getText() != null) + { + metrics.addAll(profile.getText().getMetrics()); + } + if(profile.getDates() != null) + { + metrics.addAll(profile.getDates().getMetrics()); + assertThat(profile.getChart().getX().getLabel()).endsWith(" (day)"); + assertThat(profile.getChart().getSeries().getFirst().getPoints().getFirst().getX().toString()).isEqualTo("2026-01-01"); + } + if(profile.getBooleans() != null) + { + metrics.addAll(profile.getBooleans()); + } + for(Metric metric : metrics) + { + keys.add(metric.getKey()); + assertThat(metric.getLabel()).isEqualTo(com.kingsrook.qbits.dataanalysis.engine.MetricLabels.label(metric.getKey(), metric.getValueType())); + assertThat(metric.getFormatted()).isNotBlank(); + } + } + assertThat(keys).containsExactlyInAnyOrder("rows", "nonNull", "nulls", "blanks", "distinct", "uniqueness", "min", "max", "mean", "sum", "stddev", + "p1", "p5", "p25", "p50", "p75", "p95", "p99", "zeros", "negatives", "outliers", "minLength", "maxLength", "meanLength", "true", "false", "range"); + ColumnProfile amount = service.columnProfile(column("amount")); + assertThat(metric(amount.getNumeric().getMetrics(), "sum").getFormatted()).isEqualTo("$1,170.00"); + assertThat(amount.getNumeric().getHistogram()).allSatisfy(v -> assertThat(v.getLabel()).contains("$")); + Metric uniqueness = metric(amount.getCounts(), "uniqueness"); + assertThat(uniqueness.getValueType().name()).isEqualTo("PERCENT"); + assertThat(uniqueness.getValue().toString()).isEqualTo("0.727273"); + assertThat(uniqueness.getFormatted()).isEqualTo("72.7%"); + assertThat(metric(service.columnProfile(column("created")).getDates().getMetrics(), "min").getFormatted()).isEqualTo("2026-01-01 06:00"); + assertThat(service.columnProfile(column("note")).getWarnings()).isEmpty(); + var table = service.tableProfile(new TableProfileRequest().withTableName("orders").withFields(List.of("note"))); + assertThat(table.getRowCount().getLabel()).isEqualTo("Rows"); + assertThat(table.getFields().getFirst().getNulls().getLabel()).isEqualTo("Nulls"); + assertThat(table.getFields().getFirst().getBlanks().getLabel()).isEqualTo("Blanks"); + } + + /******************************************************************************* + ** Decimal moments retain ten significant digits; drills retain exact membership. *******************************************************************************/ @Test void numericProfile() throws Exception @@ -80,12 +133,12 @@ abstract class EngineContractTest extends EngineFixture Map.entry("zeros", "2"), Map.entry("negatives", "1"), Map.entry("outliers", "1")); for(var entry : expected.entrySet()) { - assertMetric(result.getNumeric().getMetrics(), entry.getKey(), entry.getValue()); + assertMetric(result.getNumeric().getMetrics(), entry.getKey(), new BigDecimal(entry.getValue()).round(new java.math.MathContext(10)).toPlainString()); } double[] values = {-10, 0, 0, 10, 10, 20, 20, 30, 40, 50, 1000}; double mean = 1170.0 / 11; double variance = java.util.Arrays.stream(values).map(v -> Math.pow(v - mean, 2)).sum() / 10; - assertMetric(result.getNumeric().getMetrics(), "stddev", Double.toString(Math.sqrt(variance))); + assertMetric(result.getNumeric().getMetrics(), "stddev", BigDecimal.valueOf(Math.sqrt(variance)).round(new java.math.MathContext(10)).toPlainString()); assertThat(result.getNumeric().getHistogram()).hasSize(22); assertThat(result.getNumeric().getHistogram().stream().mapToLong(v -> v.getCount()).sum()).isEqualTo(11); assertThat(result.getNumeric().getOutliers()).singleElement().satisfies(v -> assertThat(new BigDecimal(v.getValue().toString())).isEqualByComparingTo("1000")); @@ -156,7 +209,7 @@ abstract class EngineContractTest extends EngineFixture assertThat(table.getFields()).anySatisfy(f -> { assertThat(f.getField()).isEqualTo("note"); - assertThat(f.getCompleteness().doubleValue()).isCloseTo(8.0 / 12, within(1e-9)); + assertThat(f.getCompleteness().doubleValue()).isEqualTo(0.666667); }); assertThat(table.getCandidateKeys()).anyMatch(k -> k.getFields().equals(List.of("id"))); var pairs = service.tableProfile(new TableProfileRequest().withTableName("orders").withFields(List.of("status", "quantity"))); @@ -409,7 +462,7 @@ abstract class EngineContractTest extends EngineFixture assertThat(ColumnProfiler.autoBucket(first, first.plusYears(2).plusDays(1))).isEqualTo(AnalysisBackend.DateUnit.MONTH); assertThat(ColumnProfiler.autoBucket(first, first.plusYears(10))).isEqualTo(AnalysisBackend.DateUnit.MONTH); assertThat(ColumnProfiler.autoBucket(first, first.plusYears(10).plusDays(1))).isEqualTo(AnalysisBackend.DateUnit.YEAR); - for(AnalysisBackend.DateUnit unit : List.of(AnalysisBackend.DateUnit.WEEK, AnalysisBackend.DateUnit.MONTH, AnalysisBackend.DateUnit.YEAR)) + for(AnalysisBackend.DateUnit unit : List.of(AnalysisBackend.DateUnit.WEEK, AnalysisBackend.DateUnit.MONTH, AnalysisBackend.DateUnit.QUARTER, AnalysisBackend.DateUnit.YEAR)) { var buckets = service.backend("orders").dateBuckets("orders", null, "order_date", unit, java.time.ZoneId.of("America/Chicago")); assertThat(buckets.stream().mapToLong(AnalysisBackend.GroupCount::count).sum()).isEqualTo(12); diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/EngineFixture.java b/src/test/java/com/kingsrook/qbits/dataanalysis/EngineFixture.java index 8642f4f..8c4105a 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/EngineFixture.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/EngineFixture.java @@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; @@ -96,7 +97,7 @@ abstract class EngineFixture .withPermissionRules(DataAnalysisTables.permissionRules("orders")) .withField(new QFieldMetaData("id", QFieldType.INTEGER)).withField(new QFieldMetaData("customer", QFieldType.INTEGER).withPossibleValueSourceName("customer")) .withField(new QFieldMetaData("status", QFieldType.STRING).withPossibleValueSourceName("statusValues")) - .withField(new QFieldMetaData("amount", QFieldType.DECIMAL)).withField(new QFieldMetaData("quantity", QFieldType.INTEGER)) + .withField(new QFieldMetaData("amount", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.CURRENCY)).withField(new QFieldMetaData("quantity", QFieldType.INTEGER)) .withField(new QFieldMetaData("order_date", QFieldType.DATE)).withField(new QFieldMetaData("created", QFieldType.DATE_TIME)) .withField(new QFieldMetaData("note", QFieldType.STRING)).withField(new QFieldMetaData("paid", QFieldType.BOOLEAN)) .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date")) diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/PresentationSamplingTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/PresentationSamplingTest.java new file mode 100644 index 0000000..b4a7ec4 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/PresentationSamplingTest.java @@ -0,0 +1,115 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis; + +import java.time.Instant; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qbits.dataanalysis.engine.AnalysisBackend; +import com.kingsrook.qbits.dataanalysis.engine.DrillFilters; +import com.kingsrook.qbits.dataanalysis.result.ColumnProfileRequest; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + +/******************************************************************************* + ** Real rows prove warning boundaries and date coordinates without Docker. + *******************************************************************************/ +class PresentationSamplingTest extends EngineFixture +{ + /******************************************************************************* + ** Null prefixes do not consume the budget; warn only after 10,000 values. + *******************************************************************************/ + @Test + void patternWarningOnlyWhenTruncated() throws Exception + { + List records = new ArrayList<>(); + for(int i = 100; i <= 10101; i++) + { + records.add(new QRecord().withValue("id", i).withValue("customer", 1).withValue("note", i == 100 ? null : "AA-11")); + } + assertThat(new InsertAction().execute(new InsertInput("orders").withRecords(records)).getRecords()).allSatisfy(r -> assertThat(r.getErrors()).isNullOrEmpty()); + var filter = DrillFilters.condition("id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, 100); + var request = new ColumnProfileRequest().withTableName("orders").withFieldName("note").withFilter(filter).withExact(true); + var truncated = service.columnProfile(request); + assertThat(truncated.getText().getSampleRows()).isEqualTo(10000); + assertThat(truncated.getWarnings()).containsExactly("Patterns are based on the first 10,000 values."); + request.withFilter(DrillFilters.and(filter, DrillFilters.condition("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, 10100))); + var all = service.columnProfile(request); + assertThat(all.getText().getSampleRows()).isEqualTo(10000); + assertThat(all.getWarnings()).isEmpty(); + } + + + + /******************************************************************************* + ** Empty samples are not empty populations; full scans are not estimates. + *******************************************************************************/ + @Test + void warningsDescribeObservedScope() throws Exception + { + config.withDefaultSampleThresholdRows(1).withDefaultSampleRows(100); + var request = new ColumnProfileRequest().withTableName("orders").withFieldName("note"); + var complete = service.columnProfile(request); + assertThat(complete.getEstimate()).isFalse(); + assertThat(complete.getWarnings()).isEmpty(); + config.withDefaultSampleRows(0); + service.getConfigResolver().invalidate(); + var emptySample = service.columnProfile(request); + assertThat(emptySample.getWarnings()).contains("No sample rows were observed; run exact for exact numbers.") + .doesNotContain("No readable records match this analysis scope.", "All values in this analysis scope are null."); + var empty = service.columnProfile(request.withExact(true).withFilter(DrillFilters.condition("id", QCriteriaOperator.EQUALS, -1))); + assertThat(empty.getWarnings()).containsExactly("No readable records match this analysis scope."); + var nulls = service.columnProfile(request.withFilter(DrillFilters.condition("id", QCriteriaOperator.EQUALS, 1))); + assertThat(nulls.getWarnings()).containsExactly("All values in this analysis scope are null."); + } + + + + /******************************************************************************* + ** Hour buckets cross DST in real elapsed hours and preserve ISO instants. + *******************************************************************************/ + @Test + void hourlyBucketCoordinatesAndLabels() throws Exception + { + var backend = service.backend("orders"); + ZoneId zone = ZoneId.of("America/Chicago"); + var buckets = backend.dateBuckets("orders", null, "created", AnalysisBackend.DateUnit.HOUR, zone); + assertThat(buckets.getFirst().value()).isEqualTo(Instant.parse("2026-01-01T12:00:00Z")); + assertThat(buckets.getFirst().label()).isEqualTo("2026-01-01 06:00"); + assertThat(buckets.stream().mapToLong(AnalysisBackend.GroupCount::count).sum()).isEqualTo(12); + for(var bucket : buckets) + { + assertThat(backend.count("orders", bucket.condition())).isEqualTo(bucket.count()); + } + TestSupport.insert("orders", new QRecord().withValue("id", 20).withValue("customer", 1).withValue("created", Instant.parse("2026-11-01T06:30:00Z"))); + TestSupport.insert("orders", new QRecord().withValue("id", 21).withValue("customer", 1).withValue("created", Instant.parse("2026-11-01T07:30:00Z"))); + var filter = DrillFilters.condition("id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, 20); + var dst = backend.dateBuckets("orders", filter, "created", AnalysisBackend.DateUnit.HOUR, zone); + assertThat(dst).hasSize(2).allSatisfy(b -> assertThat(b.label()).isEqualTo("2026-11-01 01:00")); + assertThat(dst).extracting(AnalysisBackend.GroupCount::value).doesNotHaveDuplicates(); + assertThat(dst).allSatisfy(b -> assertThat(b.count()).isEqualTo(1)); + } +} -- 2.54.0 From d591d3d7e84db3a4a920842b8c5908d47ff84654 Mon Sep 17 00:00:00 2001 From: James Maes Date: Tue, 13 Oct 2026 18:56:37 -0500 Subject: [PATCH 04/10] feat(pivot): pivot / group-by explorer and the field-access resolver seam POST /tables/{table}/pivot (extension 1.1.0): up to 3 row and 2 column dimensions with date buckets or nice numeric bins, fields from to-one exposed joins, count/sum/avg/min/max/distinct/percentile measures, totals, Other-merging by top-N, the privacy guard and a drill per cell. FieldAnalysisAccessResolver maps fields to FULL / EXACT_MATCH_ONLY / NONE (core secured-data tiers when present, FieldSecurityLock otherwise); only FULL fields are analyzable, drills never name others, and exact-run cache keys include the user's field-access fingerprint. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 +- README.md | 21 +- .../dataanalysis/DataAnalysisQBitConfig.java | 21 + .../dataanalysis/DataAnalysisService.java | 56 +- .../dataanalysis/api/DataAnalysisApi.java | 89 ++- .../dataanalysis/api/DataAnalysisSchemas.java | 285 ++++++++ .../engine/AnalysisDrillGuard.java | 168 +++++ .../dataanalysis/engine/AnalysisLabels.java | 26 +- .../dataanalysis/engine/AnalysisSecurity.java | 64 +- .../dataanalysis/engine/ColumnProfiler.java | 12 +- .../engine/CoreTierFieldAccessResolver.java | 118 ++++ .../dataanalysis/engine/DistinctValues.java | 12 +- .../engine/FieldAnalysisAccess.java | 29 + .../engine/FieldAnalysisAccessResolver.java | 43 ++ .../FieldSecurityLockFieldAccessResolver.java | 42 ++ .../engine/MinGroupSizeGuard.java | 12 + .../qbits/dataanalysis/engine/NiceBins.java | 12 +- .../dataanalysis/engine/PivotExplorer.java | 607 ++++++++++++++++++ .../dataanalysis/engine/ProfileSupport.java | 11 +- .../qbits/dataanalysis/result/PivotCell.java | 89 +++ .../dataanalysis/result/PivotDimension.java | 88 +++ .../dataanalysis/result/PivotHeader.java | 69 ++ .../dataanalysis/result/PivotMeasure.java | 95 +++ .../dataanalysis/result/PivotMeasureInfo.java | 87 +++ .../dataanalysis/result/PivotNumericBins.java | 88 +++ .../dataanalysis/result/PivotRequest.java | 182 ++++++ .../dataanalysis/result/PivotResult.java | 252 ++++++++ .../dataanalysis/run/AnalysisRunService.java | 3 +- .../openapi/qqq-v1/data-analysis.yaml | 235 ++++++- .../dataanalysis/CoreAnalysisBackendTest.java | 24 + .../dataanalysis/DataAnalysisApiTest.java | 41 +- .../dataanalysis/EngineContractTest.java | 6 +- .../dataanalysis/FieldAnalysisAccessTest.java | 244 +++++++ .../qbits/dataanalysis/PivotContractTest.java | 350 ++++++++++ 34 files changed, 3447 insertions(+), 42 deletions(-) create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisDrillGuard.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/CoreTierFieldAccessResolver.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/FieldAnalysisAccess.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/FieldAnalysisAccessResolver.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/FieldSecurityLockFieldAccessResolver.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/PivotExplorer.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotCell.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotDimension.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotHeader.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotMeasure.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotMeasureInfo.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotNumericBins.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotRequest.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotResult.java create mode 100644 src/test/java/com/kingsrook/qbits/dataanalysis/FieldAnalysisAccessTest.java create mode 100644 src/test/java/com/kingsrook/qbits/dataanalysis/PivotContractTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 3424b6f..66c54b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,13 @@ Following [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and [Semantic ### Added -- `data-analysis` UI API extension 1.0.0: seven session-bound profile, capability, distinct-value, duplicate and run endpoints, typed schemas and a checked OpenAPI fragment. +- Phase c pivot backend and `POST /qbit/data-analysis/tables/{table}/pivot`, extension version 1.1.0, typed schemas and regenerated OpenAPI fragment. +- Row/column dimensions, calendar/numeric buckets, seven measure operators, sparse cells, exact Other recomputation, totals, privacy suppression and count-equivalent drills. +- Exposed-join dimensions and measures with READ/FULL checks; filter-only drills for to-one paths and explicit null-drill warnings for to-many paths whose joined-row counts can exceed record counts. +- Configurable field-access resolver, cached reflective core tier bridge with legacy fallback, FULL-only capabilities/selections/labels, generated-drill protection and field-access cache fingerprints. +- Shared memory/H2/PostgreSQL pivot contracts, HTTP drills and errors, reflective resolver/fallback and tier-change cache tests. + +- Session-bound profile, capability, distinct-value, duplicate and run endpoints, typed schemas and a checked OpenAPI fragment (introduced in extension 1.0.0). - Core table-feature permission registration and capability-directed aggregate pushdown, with native sampling and portable streamed fallbacks. - HTTP authorization, drill-to-core-count round trips, async polling, served OpenAPI and assistant discovery tests; H2 streamed and native-compatibility contract variants. diff --git a/README.md b/README.md index 43e4329..48c0ff4 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ Count drills combine an untouched copy of the request filter with their producin ## HTTP endpoints -Extension `data-analysis` version **1.0.0**, prefix `/qbit/data-analysis` inside `/qqq/v1`: +Extension `data-analysis` version **1.1.0**, prefix `/qbit/data-analysis` inside `/qqq/v1`: | Method | Path after `/qqq/v1` | Body or purpose | | --- | --- | --- | @@ -147,12 +147,13 @@ Extension `data-analysis` version **1.0.0**, prefix `/qbit/data-analysis` inside | POST | `/qbit/data-analysis/tables/{table}/distinct-values` | `{fieldName, filter, page, pageSize, search}` | | POST | `/qbit/data-analysis/tables/{table}/table-profile` | `{filter, fields}` | | POST | `/qbit/data-analysis/tables/{table}/duplicates` | `{fields, filter, limit}`; default limit 50, maximum 500 | +| POST | `/qbit/data-analysis/tables/{table}/pivot` | `{rows, columns, values, filter, showTotals, limitGroups}`; 1–3 row dimensions, 0–2 column dimensions, 1–6 measures | | POST | `/qbit/data-analysis/runs` | `{operation: COLUMN_PROFILE or TABLE_PROFILE, tableName, request}` returns `{runId, status}` | | GET | `/qbit/data-analysis/runs/{runId}` | Status, timestamps, error, typed `columnProfile` or `tableProfile` | Every operation uses the request session through `DataAnalysisService`. Capabilities returns 200 with `canAnalyze: false` and no fields when denied. Other permission denials return 403, bad bodies or unavailable fields return 400, and unknown tables return 404. Core middleware supplies plain messages in its standard `{error: message}` JSON envelope. Run rows remain owner-locked. -Requests and drills reuse core's `QueryFilter` JSON shape and schema reference. Versioned `DataAnalysis...V1` schemas cover profiles, metrics, drills, charts, series, points, field summaries, distinct and duplicate results, capabilities and runs. The [OpenAPI fragment](src/main/resources/openapi/qqq-v1/data-analysis.yaml) is checked against regeneration. The deployed `/qqq/v1/openapi.json` and `UiApiInvoker.listOperations()` include all seven operations. +Requests and drills reuse core's `QueryFilter` JSON shape and schema reference. Versioned `DataAnalysis...V1` schemas cover profiles, metrics, drills, charts, series, points, field summaries, distinct and duplicate results, capabilities and runs. The [OpenAPI fragment](src/main/resources/openapi/qqq-v1/data-analysis.yaml) is checked against regeneration. The deployed `/qqq/v1/openapi.json` and `UiApiInvoker.listOperations()` include all eight operations. Regenerate deliberately after an API change: @@ -160,6 +161,20 @@ Regenerate deliberately after an API change: ./mvnw -o -q test -Dtest=DataAnalysisApiTest#contract -Dopenapi.write=true -Dqqq.snapshot.version=4.1.0-DATAANALYSIS-SNAPSHOT ``` +## Pivot backend and field access + +`DataAnalysisService.pivot(PivotRequest)` returns sparse cells indexed into row/column header tuples, aligned totals, typed/formatted metrics, warnings and `asOf`. `COUNT` counts records (including null measure values); other operators require a field. `PERCENTILE` uses continuous interpolation with a fraction from 0 to 1. Calendar buckets use the session timezone. Numeric bins reuse profile nice-edge selection when explicit bounds are omitted; the last interior bin includes its maximum. + +The planner first requests count-ordered top-N groups for every dimension (default 50, maximum 200), then dimension counts for privacy/cardinality, one grouped matrix and one aggregate per totals level. Additional aggregates recompute merged Other cells/totals: averages, distinct counts and percentiles cannot safely be combined from displayed cells. Ordinary Other predicates exclude kept values and handle null/empty strings separately. Boolean, temporal and bucketed Other groups use equivalent membership/range predicates for core HTTP compatibility. Cyclic date predicates enumerate observed values through core grouping; memory numeric boundaries reuse the profiles' decimal-scale compatibility helper. Native pivots are exact; bounded core streamed fallback marks partial results as estimates and disables their drills. + +`joinTable.field` resolves only through an unambiguous exposed join path. Every table in that path requires READ and the selected field must be FULL; core enforces joins and record locks. The installed data-analysis snapshot does not contain the related-record picker path API, so this implementation uses `getExposedJoins()` and `QueryJoin`. Paths whose every hop is MANY_TO_ONE or ONE_TO_ONE in the analyzed direction support ordinary filter-only drills: core infers the joins from qualified criteria. To-many paths can multiply a base record, so their pivots retain null drills and warn that counts include one row per related table record. + +`min_group_size` suppresses rare dimension groups and rare row/column tuples; every measure of a small cell or total is suppressed, including COUNT, with no drill. Totals include the full authorized population. As with existing profiles, separate aggregate calls are not a transactional snapshot. + +Hosts can supply `withFieldAnalysisAccessResolver(() -> resolver)`. FULL fields alone appear in capabilities and may be selected. EXACT_MATCH_ONLY and NONE get the same 400 as an unknown field; user-authored filters are passed through for core enforcement. Generated drills using a restricted field (including representative primary keys) are disabled. Related PVS display labels also require FULL access. Exact-run cache identities hash the sorted FULL-field list, the optional core access fingerprint, and related label access. + +The default resolver discovers the optional core `fieldaccess.FieldAccessHelper` class and its static methods once. FULL maps to FULL, REVEALABLE to EXACT_MATCH_ONLY, MASKED/HIDDEN to NONE. Without that class, legacy field locks supply FULL/NONE. Replace the reflective calls when the core package ships. Reflection tests use a local stand-in helper rather than shadowing a future real core class at its FQN; the installed snapshot exercises the actual fallback selection. Exact-run persistence continues to support profiles only; pivot results are synchronous. + ## Backend selection `DataAnalysisService` selects `CoreAnalysisBackend` when the source backend advertises aggregate capabilities, including memory. Supported standard deviation, percentiles, null/blank counts, numeric/date buckets and length statistics execute through core aggregates as the current session. Unsupported primitives delegate to `StandInAnalysisBackend`. Patterns and derived length histograms remain streamed; representative distinct drills and cyclic-date membership use bounded scans. @@ -180,7 +195,7 @@ Data versions use filtered COUNT plus MAX of a readable modification-date-like f 4. Review per-table include/exclude sets, limits and minimum group sizes before enabling analysis. Integrate Saved Analyses and Watches with host ESB/Quick Search when those optional host features are installed; this scaffold does not add those dependencies. 5. Use the Analyses home as the owner of analysis data. Other dashboards should link there until saved-analysis widgets are implemented. -Phase **b** profiles, drill-through, runs/cache and HTTP endpoints are implemented. Remaining work: **b** analysis UI; **c** pivots; **d** comparisons and relationships; **e** watch evaluation and alerts; **f** saved-analysis execution, sharing, stars integration, schedules, chart widgets, full home and exports. The current home summarizes stored rows; the host facade executes profiles. The home does not send deliveries. +Phase **b** profiles, drill-through, runs/cache and HTTP endpoints, and phase **c** pivot backend are implemented. Remaining work: **b** analysis UI; **c** pivot UI; **d** comparisons and relationships; **e** watch evaluation and alerts; **f** saved-analysis execution, sharing, stars integration, schedules, chart widgets, full home and exports. The current home summarizes stored rows; the host facade executes profiles. The home does not send deliveries. ## Build diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitConfig.java b/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitConfig.java index 384b1b3..6120fa9 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitConfig.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitConfig.java @@ -26,6 +26,9 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Supplier; +import com.kingsrook.qbits.dataanalysis.engine.CoreTierFieldAccessResolver; +import com.kingsrook.qbits.dataanalysis.engine.FieldAnalysisAccessResolver; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig; @@ -35,6 +38,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig; *******************************************************************************/ public class DataAnalysisQBitConfig implements QBitConfig { + private Supplier fieldAnalysisAccessResolver = CoreTierFieldAccessResolver::defaultResolver; private String backendName; private String schedulerName; private String appName = "dataAnalysis"; @@ -387,4 +391,21 @@ public class DataAnalysisQBitConfig implements QBitConfig this.tableDefaults = tableDefaults; return (this); } + + /******************************************************************************* + ** Host override for session-aware field analysis access. + *******************************************************************************/ + public Supplier getFieldAnalysisAccessResolver() + { + return (fieldAnalysisAccessResolver); + } + + /******************************************************************************* + ** Configure an instance supplier; each service resolves one policy instance. + *******************************************************************************/ + public DataAnalysisQBitConfig withFieldAnalysisAccessResolver(Supplier resolver) + { + this.fieldAnalysisAccessResolver = java.util.Objects.requireNonNull(resolver); + return (this); + } } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisService.java b/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisService.java index c6b106d..729b33a 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisService.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisService.java @@ -21,14 +21,21 @@ package com.kingsrook.qbits.dataanalysis; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.Executor; import com.kingsrook.qbits.dataanalysis.engine.AnalysisBackend; +import com.kingsrook.qbits.dataanalysis.engine.AnalysisDrillGuard; +import com.kingsrook.qbits.dataanalysis.engine.AnalysisLabels; import com.kingsrook.qbits.dataanalysis.engine.AnalysisSecurity; import com.kingsrook.qbits.dataanalysis.engine.ColumnProfiler; import com.kingsrook.qbits.dataanalysis.engine.CoreAnalysisBackend; import com.kingsrook.qbits.dataanalysis.engine.DistinctValues; +import com.kingsrook.qbits.dataanalysis.engine.DrillFilters; import com.kingsrook.qbits.dataanalysis.engine.DuplicateFinder; import com.kingsrook.qbits.dataanalysis.engine.EffectiveConfigResolver; +import com.kingsrook.qbits.dataanalysis.engine.FieldAnalysisAccessResolver; +import com.kingsrook.qbits.dataanalysis.engine.PivotExplorer; import com.kingsrook.qbits.dataanalysis.engine.StandInAnalysisBackend; import com.kingsrook.qbits.dataanalysis.engine.TableProfiler; import com.kingsrook.qbits.dataanalysis.result.AnalysisRequest; @@ -39,6 +46,8 @@ import com.kingsrook.qbits.dataanalysis.result.DistinctValuesPage; import com.kingsrook.qbits.dataanalysis.result.DistinctValuesRequest; import com.kingsrook.qbits.dataanalysis.result.DuplicatesRequest; import com.kingsrook.qbits.dataanalysis.result.DuplicatesResult; +import com.kingsrook.qbits.dataanalysis.result.PivotRequest; +import com.kingsrook.qbits.dataanalysis.result.PivotResult; import com.kingsrook.qbits.dataanalysis.result.RunStatus; import com.kingsrook.qbits.dataanalysis.result.TableProfile; import com.kingsrook.qbits.dataanalysis.result.TableProfileRequest; @@ -53,6 +62,7 @@ public class DataAnalysisService { private final EffectiveConfigResolver configs; private final AnalysisSecurity security; + private final FieldAnalysisAccessResolver fieldAccess; private final AnalysisRunService runs; /******************************************************************************* @@ -71,7 +81,8 @@ public class DataAnalysisService public DataAnalysisService(DataAnalysisQBitConfig config, Executor executor) { configs = new EffectiveConfigResolver(config); - security = new AnalysisSecurity(configs); + fieldAccess = config.getFieldAnalysisAccessResolver().get(); + security = new AnalysisSecurity(configs, fieldAccess); runs = new AnalysisRunService(this, executor); } @@ -116,7 +127,7 @@ public class DataAnalysisService public DistinctValuesPage distinctValues(DistinctValuesRequest request) throws QException { validate(request); - return (new DistinctValues(backend(request.getTableName())).page(request, security.field(request.getTableName(), request.getFieldName()), configs.resolve(request.getTableName()))); + return (protect(new DistinctValues(backend(request.getTableName()), fieldAccess).page(request, security.field(request.getTableName(), request.getFieldName()), configs.resolve(request.getTableName())), request)); } @@ -127,7 +138,7 @@ public class DataAnalysisService public DuplicatesResult duplicates(DuplicatesRequest request) throws QException { validate(request); - return (new DuplicateFinder(backend(request.getTableName())).find(request, configs.resolve(request.getTableName()))); + return (protect(new DuplicateFinder(backend(request.getTableName())).find(request, configs.resolve(request.getTableName())), request)); } @@ -219,7 +230,7 @@ public class DataAnalysisService public ColumnProfile compute(ColumnProfileRequest request, AnalysisBackend backend, AnalysisBackend.Scope scope) throws QException { validate(request); - return (new ColumnProfiler(backend).profile(request, security.field(request.getTableName(), request.getFieldName()), scope, configs.resolve(request.getTableName()))); + return (protect(new ColumnProfiler(backend, fieldAccess).profile(request, security.field(request.getTableName(), request.getFieldName()), scope, configs.resolve(request.getTableName())), request)); } @@ -232,6 +243,41 @@ public class DataAnalysisService validate(request); List fields = security.analyzableFields(request.getTableName()).stream() .filter(f -> request.getFields() == null || request.getFields().contains(f.getName())).toList(); - return (new TableProfiler(backend).profile(request, fields, scope)); + return (protect(new TableProfiler(backend).profile(request, fields, scope), request)); + } + + /******************************************************************************* + ** Field policy identity is re-evaluated before cache reads and writes. + *******************************************************************************/ + public String fieldAccessFingerprint(String table) throws QException + { + return (security.accessFingerprint(table)); + } + + /******************************************************************************* + ** Execute a pivot as the current user through core aggregates. + *******************************************************************************/ + public PivotResult pivot(PivotRequest request) throws QException + { + validate(request); + return (new PivotExplorer(security, configs.resolve(request.getTableName())).execute(request)); + } + + /******************************************************************************* + ** Generated primary-key drills also obey the configured field-access seam. + *******************************************************************************/ + private T protect(T result, AnalysisRequest request) throws QException + { + new AnalysisDrillGuard(Set.copyOf(capabilities(request.getTableName()).getFields()), + DrillFilters.restoreTypes(request.getTableName(), request.getFilter())).apply(result); + return (result); + } + + /******************************************************************************* + ** Include resolved tiers of related label fields in cache identity. + *******************************************************************************/ + public Map fieldAccessPolicy(String table) throws QException + { + return (AnalysisLabels.policy(table, fieldAccess)); } } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisApi.java b/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisApi.java index 2aeb0df..9951b86 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisApi.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisApi.java @@ -30,6 +30,7 @@ import com.kingsrook.qbits.dataanalysis.result.ChartKind; import com.kingsrook.qbits.dataanalysis.result.ColumnProfileRequest; import com.kingsrook.qbits.dataanalysis.result.DistinctValuesRequest; import com.kingsrook.qbits.dataanalysis.result.DuplicatesRequest; +import com.kingsrook.qbits.dataanalysis.result.PivotRequest; import com.kingsrook.qbits.dataanalysis.result.TableProfileRequest; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; @@ -70,10 +71,10 @@ public class DataAnalysisApi *******************************************************************************/ public static MiddlewareSpecExtension extension() { - return (new MiddlewareSpecExtension().withName("data-analysis").withSemanticVersion("1.0.0") - .withDescription("Profiles, distinct values, duplicate groups and exact analysis runs as the signed-in user.") + return (new MiddlewareSpecExtension().withName("data-analysis").withSemanticVersion("1.1.0") + .withDescription("Profiles, pivots, distinct values, duplicate groups and exact analysis runs as the signed-in user.") .withSpec(CapabilitiesSpec.class).withSpec(ColumnProfileSpec.class).withSpec(DistinctValuesSpec.class) - .withSpec(TableProfileSpec.class).withSpec(DuplicatesSpec.class).withSpec(StartRunSpec.class).withSpec(GetRunSpec.class)); + .withSpec(PivotSpec.class).withSpec(TableProfileSpec.class).withSpec(DuplicatesSpec.class).withSpec(StartRunSpec.class).withSpec(GetRunSpec.class)); } /******************************************************************************* @@ -184,7 +185,7 @@ public class DataAnalysisApi throw new IllegalArgumentException("exact must be a boolean."); } QQueryFilter filter = body.get("filter") == null ? null : JsonUtils.toObject(JsonUtils.toJson(body.get("filter")), QQueryFilter.class); - validateFilter(filter, readable); + validateFilter(filter); List fields = null; if(body.get("fields") != null) { @@ -224,7 +225,7 @@ public class DataAnalysisApi /******************************************************************************* ** Bad filter fields fail before any aggregate or cached result is consulted. *******************************************************************************/ - private static void validateFilter(QQueryFilter filter, List readable) + private static void validateFilter(QQueryFilter filter) { if(filter == null) { @@ -232,13 +233,12 @@ public class DataAnalysisApi } for(var criterion : filter.getCriteria()) { - if(criterion.getOperator() == null || !readable.contains(criterion.getFieldName()) - || criterion.getOtherFieldName() != null && !readable.contains(criterion.getOtherFieldName())) + if(criterion.getOperator() == null || criterion.getFieldName() == null) { throw new IllegalArgumentException("Choose readable filter fields and a valid operator."); } } - filter.getSubFilters().forEach(sub -> validateFilter(sub, readable)); + filter.getSubFilters().forEach(sub -> validateFilter(sub)); } /******************************************************************************* @@ -504,4 +504,77 @@ public class DataAnalysisApi }; return (result); } + + /******************************************************************************* + ** Group records by row and column dimensions. + *******************************************************************************/ + public static class PivotSpec extends ProfileSpec + { + /******************************************************************************* + ** Publish assistant-friendly grouping and measure capabilities. + *******************************************************************************/ + public PivotSpec() + { + super("/tables/{table}/pivot", HttpMethod.POST, "Explore a pivot table", + "Group records by up to 3 row and 2 column fields (with date or numeric buckets) and compute counts, sums, averages, min/max, distinct counts or percentiles per cell; every cell has a drill-through filter.", "PivotRequest", "Pivot"); + } + + /******************************************************************************* + ** Strict body validation precedes the session-bound engine. + *******************************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + String tableName = input.getPathParam("table"); + table(tableName); + DataAnalysisRuntime.service().capabilities(tableName); + PivotRequest pivot; + try + { + Map body = input.getBody(); + integer(body, "limitGroups", 50, 1, 200); + if(body.containsKey("showTotals") && !(body.get("showTotals") instanceof Boolean)) + { + throw new IllegalArgumentException("showTotals must be a boolean."); + } + for(String axis : List.of("rows", "columns")) + { + if(body.containsKey(axis)) + { + if(!(body.get(axis) instanceof List dimensions)) + { + throw new IllegalArgumentException("Dimensions must be arrays."); + } + for(Object dimension : dimensions) + { + if(!(dimension instanceof Map map) || !(map.get("field") instanceof String)) + { + throw new IllegalArgumentException("Each dimension requires a field name."); + } + if(map.get("numericBins") instanceof Map bins) + { + Object count = bins.get("count"); + if(count != null && (!(count instanceof Number number) || number.doubleValue() != number.intValue())) + { + throw new IllegalArgumentException("Bin count must be an integer."); + } + } + } + } + } + if(!(body.get("values") instanceof List values) || values.stream().anyMatch(v -> !(v instanceof Map m) + || !(m.get("operator") instanceof String) || m.get("field") != null && !(m.get("field") instanceof String))) + { + throw new IllegalArgumentException("Measures require an operator and optional field name."); + } + pivot = JsonUtils.toObject(JsonUtils.toJson(body), PivotRequest.class).withTableName(tableName); + validateFilter(pivot.getFilter()); + } + catch(Exception e) + { + throw new QBadRequestException("Invalid pivot request."); + } + return (transport(DataAnalysisRuntime.service().pivot(pivot), DataAnalysisSchemas.DataAnalysisPivotV1.class)); + } + } } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java b/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java index 151c72e..19faa6e 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java @@ -106,6 +106,14 @@ public class DataAnalysisSchemas result.put("DataAnalysisLimitsV1", new DataAnalysisLimitsV1().toSchema()); result.put("DataAnalysisStartRunRequestV1", new DataAnalysisStartRunRequestV1().toSchema()); result.put("DataAnalysisRunStartedV1", new DataAnalysisRunStartedV1().toSchema()); + result.put("DataAnalysisPivotRequestV1", new DataAnalysisPivotRequestV1().toSchema()); + result.put("DataAnalysisPivotDimensionV1", new DataAnalysisPivotDimensionV1().toSchema()); + result.put("DataAnalysisPivotNumericBinsV1", new DataAnalysisPivotNumericBinsV1().toSchema()); + result.put("DataAnalysisPivotMeasureV1", new DataAnalysisPivotMeasureV1().toSchema()); + result.put("DataAnalysisPivotHeaderV1", new DataAnalysisPivotHeaderV1().toSchema()); + result.put("DataAnalysisPivotMeasureInfoV1", new DataAnalysisPivotMeasureInfoV1().toSchema()); + result.put("DataAnalysisPivotCellV1", new DataAnalysisPivotCellV1().toSchema()); + result.put("DataAnalysisPivotV1", new DataAnalysisPivotV1().toSchema()); return (result); } @@ -1146,4 +1154,281 @@ public class DataAnalysisSchemas return (new Schema().withType(Type.OBJECT).withNullable(true).withProperties(properties)); } } + + /******************************************************************************* + ** Nested arrays share the same explicit schema construction. + *******************************************************************************/ + private static Schema array(Schema items) + { + return (new Schema().withType(Type.ARRAY).withItems(items)); + } + + /******************************************************************************* + ** PivotRequest version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisPivotRequestV1 implements ToSchema + { + @OpenAPIDescription("User query filter, passed through for core enforcement.") + public QQueryFilter filter; + + @OpenAPIDescription("One to three row dimensions.") + public List rows; + + @OpenAPIDescription("Zero to two column dimensions; defaults to empty.") + public List columns; + + @OpenAPIDescription("One to six measures.") + public List values; + + @OpenAPIDescription("Include row, column and grand totals; defaults to true.") + public Boolean showTotals; + + @OpenAPIDescription("Kept groups per dimension; default 50, maximum 200.") + public Integer limitGroups; + + /******************************************************************************* + ** Explicit references preserve the standalone extension contract. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("filter", ref("QueryFilter").withDescription("User query filter, passed through for core enforcement.")); + properties.put("rows", array(ref("DataAnalysisPivotDimensionV1")).withDescription("One to three row dimensions.")); + properties.put("columns", array(ref("DataAnalysisPivotDimensionV1")).withDescription("Zero to two column dimensions; defaults to empty.")); + properties.put("values", array(ref("DataAnalysisPivotMeasureV1")).withDescription("One to six measures.")); + properties.put("showTotals", new Schema().withType(Type.BOOLEAN).withDescription("Include row, column and grand totals; defaults to true.")); + properties.put("limitGroups", new Schema().withType(Type.INTEGER).withDescription("Kept groups per dimension; default 50, maximum 200.")); + return (new RequiredSchema(List.of("rows", "values")).withType(Type.OBJECT).withProperties(properties)); + } + } + + /******************************************************************************* + ** PivotDimension version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisPivotDimensionV1 implements ToSchema + { + @OpenAPIDescription("FULL field name, optionally joinTable.field through an exposed join.") + public String field; + + @OpenAPIDescription("Optional calendar or cyclic bucket in the session timezone.") + public String dateBucket; + + @OpenAPIDescription("Optional equal-width numeric bins; mutually exclusive with dateBucket.") + public DataAnalysisPivotNumericBinsV1 numericBins; + + /******************************************************************************* + ** Explicit references preserve the standalone extension contract. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("field", new Schema().withType(Type.STRING).withDescription("FULL field name, optionally joinTable.field through an exposed join.")); + properties.put("dateBucket", new Schema().withType(Type.STRING).withEnumValues(List.of("YEAR", "QUARTER", "MONTH", "WEEK", "DAY", "HOUR", "DAY_OF_WEEK", "HOUR_OF_DAY")).withDescription("Optional calendar or cyclic bucket in the session timezone.")); + properties.put("numericBins", ref("DataAnalysisPivotNumericBinsV1").withDescription("Optional equal-width numeric bins; mutually exclusive with dateBucket.")); + return (new RequiredSchema(List.of("field")).withType(Type.OBJECT).withProperties(properties)); + } + } + + /******************************************************************************* + ** PivotNumericBins version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisPivotNumericBinsV1 implements ToSchema + { + @OpenAPIDescription("Number of bins, 1–200; defaults to 20.") + public Integer count; + + @OpenAPIDescription("Optional explicit lower bound; otherwise a nice enclosing edge.") + public BigDecimal min; + + @OpenAPIDescription("Optional explicit upper bound; final interior bin includes this value.") + public BigDecimal max; + + /******************************************************************************* + ** Explicit references preserve the standalone extension contract. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("count", new Schema().withType(Type.INTEGER).withDescription("Number of bins, 1–200; defaults to 20.")); + properties.put("min", new Schema().withType(Type.NUMBER).withDescription("Optional explicit lower bound; otherwise a nice enclosing edge.")); + properties.put("max", new Schema().withType(Type.NUMBER).withDescription("Optional explicit upper bound; final interior bin includes this value.")); + return (new Schema().withType(Type.OBJECT).withProperties(properties)); + } + } + + /******************************************************************************* + ** PivotMeasure version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisPivotMeasureV1 implements ToSchema + { + @OpenAPIDescription("Aggregate operator. COUNT counts records, including null values.") + public String operator; + + @OpenAPIDescription("FULL field to aggregate; optional for COUNT.") + public String field; + + @OpenAPIDescription("PERCENTILE only: percentile_cont fraction from 0 to 1.") + public BigDecimal percentile; + + /******************************************************************************* + ** Explicit references preserve the standalone extension contract. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("operator", new Schema().withType(Type.STRING).withEnumValues(List.of("COUNT", "SUM", "AVG", "MIN", "MAX", "COUNT_DISTINCT", "PERCENTILE")).withDescription("Aggregate operator. COUNT counts records, including null values.")); + properties.put("field", new Schema().withType(Type.STRING).withDescription("FULL field to aggregate; optional for COUNT.")); + properties.put("percentile", new Schema().withType(Type.NUMBER).withDescription("PERCENTILE only: percentile_cont fraction from 0 to 1.")); + return (new RequiredSchema(List.of("operator")).withType(Type.OBJECT).withProperties(properties)); + } + } + + /******************************************************************************* + ** PivotHeader version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisPivotHeaderV1 implements ToSchema + { + @OpenAPIDescription("Typed group key; null for a blank group or Other.") + public Serializable value; + + @OpenAPIDescription("Display label, including Other (k groups).") + public String label; + + /******************************************************************************* + ** Explicit references preserve the standalone extension contract. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("value", new Schema().withNullable(true).withDescription("Typed group key; null for a blank group or Other.")); + properties.put("label", new Schema().withType(Type.STRING).withDescription("Display label, including Other (k groups).")); + return (new Schema().withType(Type.OBJECT).withProperties(properties)); + } + } + + /******************************************************************************* + ** PivotMeasureInfo version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisPivotMeasureInfoV1 implements ToSchema + { + @OpenAPIDescription("Stable measure key by request position.") + public String key; + + @OpenAPIDescription("Display label.") + public String label; + + @OpenAPIDescription("Semantic metric type.") + public ValueType valueType; + + /******************************************************************************* + ** Explicit references preserve the standalone extension contract. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("key", new Schema().withType(Type.STRING).withDescription("Stable measure key by request position.")); + properties.put("label", new Schema().withType(Type.STRING).withDescription("Display label.")); + properties.put("valueType", new Schema().withType(Type.STRING).withEnumValues(java.util.Arrays.stream(ValueType.values()).map(Enum::name).toList()).withDescription("Semantic metric type.")); + return (new Schema().withType(Type.OBJECT).withProperties(properties)); + } + } + + /******************************************************************************* + ** PivotCell version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisPivotCellV1 implements ToSchema + { + @OpenAPIDescription("Zero-based index in rowHeaders.") + public Integer rowIndex; + + @OpenAPIDescription("Zero-based index in columnHeaders.") + public Integer columnIndex; + + @OpenAPIDescription("Metrics in requested measure order; suppressed metrics have no value or drill.") + public List values; + + /******************************************************************************* + ** Explicit references preserve the standalone extension contract. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("rowIndex", new Schema().withType(Type.INTEGER).withDescription("Zero-based index in rowHeaders.")); + properties.put("columnIndex", new Schema().withType(Type.INTEGER).withDescription("Zero-based index in columnHeaders.")); + properties.put("values", array(ref("DataAnalysisMetricV1")).withDescription("Metrics in requested measure order; suppressed metrics have no value or drill.")); + return (new Schema().withType(Type.OBJECT).withProperties(properties)); + } + } + + /******************************************************************************* + ** Pivot version 1 transport contract. + *******************************************************************************/ + public static class DataAnalysisPivotV1 implements ToSchema + { + @OpenAPIDescription("Row header tuples in cell-index order.") + public List> rowHeaders; + + @OpenAPIDescription("Column header tuples; an empty tuple represents no column dimensions.") + public List> columnHeaders; + + @OpenAPIDescription("Measure metadata in request order.") + public List measures; + + @OpenAPIDescription("Sparse cells; absent combinations are not materialized.") + public List cells; + + @OpenAPIDescription("Metric tuples aligned with rowHeaders; empty when totals disabled.") + public List> rowTotals; + + @OpenAPIDescription("Metric tuples aligned with columnHeaders; empty when totals disabled.") + public List> columnTotals; + + @OpenAPIDescription("Grand-total metrics; empty when totals disabled.") + public List grandTotals; + + @OpenAPIDescription("Row groups were merged into Other.") + public Boolean truncatedRows; + + @OpenAPIDescription("Column groups were merged into Other.") + public Boolean truncatedColumns; + + @OpenAPIDescription("True when bounded streamed fallback produced a partial result.") + public Boolean estimate; + + @OpenAPIDescription("Privacy, partial-result and joined-drill limitations.") + public List warnings; + + @OpenAPIDescription("UTC timestamp of computation.") + public String asOf; + + /******************************************************************************* + ** Explicit references preserve the standalone extension contract. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("rowHeaders", array(array(ref("DataAnalysisPivotHeaderV1"))).withDescription("Row header tuples in cell-index order.")); + properties.put("columnHeaders", array(array(ref("DataAnalysisPivotHeaderV1"))).withDescription("Column header tuples; an empty tuple represents no column dimensions.")); + properties.put("measures", array(ref("DataAnalysisPivotMeasureInfoV1")).withDescription("Measure metadata in request order.")); + properties.put("cells", array(ref("DataAnalysisPivotCellV1")).withDescription("Sparse cells; absent combinations are not materialized.")); + properties.put("rowTotals", array(array(ref("DataAnalysisMetricV1"))).withDescription("Metric tuples aligned with rowHeaders; empty when totals disabled.")); + properties.put("columnTotals", array(array(ref("DataAnalysisMetricV1"))).withDescription("Metric tuples aligned with columnHeaders; empty when totals disabled.")); + properties.put("grandTotals", array(ref("DataAnalysisMetricV1")).withDescription("Grand-total metrics; empty when totals disabled.")); + properties.put("truncatedRows", new Schema().withType(Type.BOOLEAN).withDescription("Row groups were merged into Other.")); + properties.put("truncatedColumns", new Schema().withType(Type.BOOLEAN).withDescription("Column groups were merged into Other.")); + properties.put("estimate", new Schema().withType(Type.BOOLEAN).withDescription("True when bounded streamed fallback produced a partial result.")); + properties.put("warnings", array(new Schema().withType(Type.STRING)).withDescription("Privacy, partial-result and joined-drill limitations.")); + properties.put("asOf", new Schema().withType(Type.STRING).withDescription("UTC timestamp of computation.")); + return (new Schema().withType(Type.OBJECT).withProperties(properties)); + } + } + } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisDrillGuard.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisDrillGuard.java new file mode 100644 index 0000000..f0fe01c --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisDrillGuard.java @@ -0,0 +1,168 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.util.Set; +import com.kingsrook.qbits.dataanalysis.result.CandidateKey; +import com.kingsrook.qbits.dataanalysis.result.Chart; +import com.kingsrook.qbits.dataanalysis.result.ColumnProfile; +import com.kingsrook.qbits.dataanalysis.result.DateProfile; +import com.kingsrook.qbits.dataanalysis.result.DistinctValuesPage; +import com.kingsrook.qbits.dataanalysis.result.Drill; +import com.kingsrook.qbits.dataanalysis.result.DuplicatesResult; +import com.kingsrook.qbits.dataanalysis.result.FieldSummary; +import com.kingsrook.qbits.dataanalysis.result.Metric; +import com.kingsrook.qbits.dataanalysis.result.NumericProfile; +import com.kingsrook.qbits.dataanalysis.result.Point; +import com.kingsrook.qbits.dataanalysis.result.Series; +import com.kingsrook.qbits.dataanalysis.result.TableProfile; +import com.kingsrook.qbits.dataanalysis.result.TextProfile; +import com.kingsrook.qbits.dataanalysis.result.ValueCount; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; + +/******************************************************************************* + ** Final defense for profile-generated drills, including representative primary + ** keys and prefix bounds. User-authored predicates remain intact; unsafe generated + ** drills are disabled rather than broadened by removing their conditions. + *******************************************************************************/ +public class AnalysisDrillGuard +{ + private final Set fields; + private final String original; + + /******************************************************************************* + ** Resolve FULL fields once per completed result. + *******************************************************************************/ + public AnalysisDrillGuard(Set fields, QQueryFilter original) + { + this.fields = fields; + this.original = JsonUtils.toJson(DrillFilters.copy(original)); + } + + /******************************************************************************* + ** Visit only typed result containers, never the caller's query. + *******************************************************************************/ + public void apply(Object value) + { + if(value instanceof Iterable list) + { + list.forEach(this::apply); + } + else if(value instanceof Metric metric) + { + metric.withDrill(permitted(metric.getDrill())); + } + else if(value instanceof ValueCount count) + { + count.withDrill(permitted(count.getDrill())); + } + else if(value instanceof Point point) + { + point.withDrill(permitted(point.getDrill())); + } + else if(value instanceof CandidateKey key) + { + key.withDrill(permitted(key.getDrill())); + } + else if(value instanceof ColumnProfile profile) + { + apply(profile.getCounts()); + apply(profile.getTopValues()); + apply(profile.getBooleans()); + apply(profile.getNumeric()); + apply(profile.getDates()); + apply(profile.getText()); + apply(profile.getChart()); + } + else if(value instanceof NumericProfile profile) + { + apply(profile.getMetrics()); + apply(profile.getHistogram()); + apply(profile.getOutliers()); + } + else if(value instanceof DateProfile profile) + { + apply(profile.getMetrics()); + apply(profile.getBuckets()); + apply(profile.getGaps()); + apply(profile.getDayOfWeek()); + apply(profile.getHourOfDay()); + } + else if(value instanceof TextProfile profile) + { + apply(profile.getMetrics()); + apply(profile.getHistogram()); + apply(profile.getPatterns()); + } + else if(value instanceof TableProfile profile) + { + apply(profile.getRowCount()); + apply(profile.getFields()); + apply(profile.getCandidateKeys()); + apply(profile.getHeatmap()); + } + else if(value instanceof FieldSummary field) + { + apply(field.getNulls()); + apply(field.getBlanks()); + } + else if(value instanceof Chart chart) + { + apply(chart.getSeries()); + } + else if(value instanceof Series series) + { + apply(series.getPoints()); + } + else if(value instanceof DistinctValuesPage page) + { + apply(page.getValues()); + } + else if(value instanceof DuplicatesResult result) + { + apply(result.getGroups()); + } + } + + /******************************************************************************* + ** Preserve count equivalence by removing the entire unsafe drill. + *******************************************************************************/ + private Drill permitted(Drill drill) + { + return (drill == null || safe(drill.getFilter()) ? drill : null); + } + + /******************************************************************************* + ** An exact copy of a user subtree is exempt from analysis field restrictions. + *******************************************************************************/ + private boolean safe(QQueryFilter filter) + { + if(original.equals(JsonUtils.toJson(DrillFilters.copy(filter)))) + { + return (true); + } + return (filter.getCriteria().stream().allMatch(c -> fields.contains(c.getFieldName()) + && (c.getOtherFieldName() == null || fields.contains(c.getOtherFieldName())) + && (c.getFieldFunction() == null || fields.contains(c.getFieldFunction().getFieldName()))) + && filter.getSubFilters().stream().allMatch(this::safe)); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisLabels.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisLabels.java index 4372566..3cf2229 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisLabels.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisLabels.java @@ -25,7 +25,6 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Set; import com.kingsrook.qqq.backend.core.actions.metadata.personalization.TableMetaDataPersonalizerAction; -import com.kingsrook.qqq.backend.core.actions.permissions.FieldSecurityLockHelper; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator; @@ -64,7 +63,7 @@ public final class AnalysisLabels *******************************************************************************/ public static String translate(QFieldMetaData field, Serializable value) throws QException { - return (translate(field, value, new LinkedHashSet<>())); + return (translate(field, value, CoreTierFieldAccessResolver.defaultResolver())); } @@ -72,7 +71,15 @@ public final class AnalysisLabels /******************************************************************************* ** Resolve nested PVS label fields without invoking core's trusted lookup path. *******************************************************************************/ - private static String translate(QFieldMetaData field, Serializable value, Set visited) throws QException + public static String translate(QFieldMetaData field, Serializable value, FieldAnalysisAccessResolver access) throws QException + { + return (translate(field, value, new LinkedHashSet<>(), access)); + } + + /******************************************************************************* + ** Resolve nested labels only through FULL fields under the host policy. + *******************************************************************************/ + private static String translate(QFieldMetaData field, Serializable value, Set visited, FieldAnalysisAccessResolver access) throws QException { if(value == null) { @@ -98,7 +105,7 @@ public final class AnalysisLabels List labelFields = original.getRecordLabelFields() == null ? List.of() : original.getRecordLabelFields(); for(String labelField : labelFields) { - if(!table.getFields().containsKey(labelField) || !FieldSecurityLockHelper.isFieldAccessible(table.getField(labelField))) + if(!table.getFields().containsKey(labelField) || access.resolve(table, table.getField(labelField)) != FieldAnalysisAccess.FULL) { return (fallback); } @@ -119,7 +126,7 @@ public final class AnalysisLabels QFieldMetaData labelMetadata = table.getField(labelField); if(labelMetadata.getPossibleValueSourceName() != null) { - record.withDisplayValue(labelField, translate(labelMetadata, record.getValue(labelField), visited)); + record.withDisplayValue(labelField, translate(labelMetadata, record.getValue(labelField), visited, access)); } } String label = QValueFormatter.formatRecordLabel(table, record); @@ -144,6 +151,14 @@ public final class AnalysisLabels ** host changing field/record locks cannot release older translated labels. *******************************************************************************/ public static java.util.Map policy(String tableName) throws QException + { + return (policy(tableName, CoreTierFieldAccessResolver.defaultResolver())); + } + + /******************************************************************************* + ** Include host-resolved tiers of related labels in the exact-run identity. + *******************************************************************************/ + public static java.util.Map policy(String tableName, FieldAnalysisAccessResolver access) throws QException { java.util.Map result = new java.util.TreeMap<>(); java.util.ArrayDeque pending = new java.util.ArrayDeque<>(); @@ -160,6 +175,7 @@ public final class AnalysisLabels QTableMetaData table = TableMetaDataPersonalizerAction.execute(input, true); java.util.Map entry = new java.util.TreeMap<>(); entry.put("fields", table.getFields()); + entry.put("fullFields", table.getFields().values().stream().filter(f -> access.resolve(table, f) == FieldAnalysisAccess.FULL).map(QFieldMetaData::getName).sorted().toList()); entry.put("locks", table.getRecordSecurityLocks()); entry.put("labelFields", table.getRecordLabelFields()); entry.put("labelFormat", table.getRecordLabelFormat()); diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisSecurity.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisSecurity.java index dbf399a..f72f08a 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisSecurity.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisSecurity.java @@ -20,19 +20,21 @@ package com.kingsrook.qbits.dataanalysis.engine; +import java.io.Serializable; import java.util.List; import com.kingsrook.qbits.dataanalysis.permissions.DataAnalysisPermissions; import com.kingsrook.qqq.backend.core.actions.metadata.personalization.TableMetaDataPersonalizerAction; -import com.kingsrook.qqq.backend.core.actions.permissions.FieldSecurityLockHelper; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType; import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; /******************************************************************************* ** Fail closed before any source-table computation, including cache retrieval. @@ -40,17 +42,27 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; public class AnalysisSecurity { private final EffectiveConfigResolver configs; + private final FieldAnalysisAccessResolver fieldAccess; /******************************************************************************* ** Use the runtime's configuration policy. *******************************************************************************/ public AnalysisSecurity(EffectiveConfigResolver configs) { - this.configs = configs; + this(configs, CoreTierFieldAccessResolver.defaultResolver()); } + /******************************************************************************* + ** Bind the host field policy. + *******************************************************************************/ + public AnalysisSecurity(EffectiveConfigResolver configs, FieldAnalysisAccessResolver fieldAccess) + { + this.configs = configs; + this.fieldAccess = fieldAccess; + } + /******************************************************************************* ** Explicit enforcement is independent of the instance's default switch. *******************************************************************************/ @@ -76,7 +88,7 @@ public class AnalysisSecurity { QTableMetaData table = table(name); EffectiveConfig c = configs.resolve(name); - return (table.getFields().values().stream().filter(FieldSecurityLockHelper::isFieldAccessible) + return (table.getFields().values().stream().filter(f -> fieldAccess.resolve(table, f) == FieldAnalysisAccess.FULL) .filter(f -> c.getAnalyzableFields().isEmpty() || c.getAnalyzableFields().contains(f.getName())) .filter(f -> !c.getExcludedFields().contains(f.getName())).toList()); } @@ -89,6 +101,50 @@ public class AnalysisSecurity public QFieldMetaData field(String table, String field) throws QException { return (analyzableFields(table).stream().filter(f -> f.getName().equals(field)).findFirst() - .orElseThrow(() -> new QPermissionDeniedException("Analysis is unavailable for this field."))); + .orElseThrow(() -> new QBadRequestException("Unknown analysis field."))); + } + + /******************************************************************************* + ** Core fingerprint plus the sorted FULL-field list, hashed by the run identity. + *******************************************************************************/ + public String accessFingerprint(String table) throws QException + { + return (fieldAccess.getAccessFingerprint() + ":" + JsonUtils.toJson(analyzableFields(table).stream().map(QFieldMetaData::getName).sorted().toList())); + } + + /******************************************************************************* + ** Check a joined field without requiring the related table analyze permission. + *******************************************************************************/ + public QFieldMetaData joinedField(String tableName, String fieldName) throws QException + { + QTableMetaData table = joinedTable(tableName); + QFieldMetaData field = table.getFields().get(fieldName); + if(field == null || fieldAccess.resolve(table, field) != FieldAnalysisAccess.FULL) + { + throw new QBadRequestException("Unknown analysis field."); + } + return (field); + } + + /******************************************************************************* + ** Display labels use the same policy as the selected pivot field. + *******************************************************************************/ + public String label(QFieldMetaData field, Serializable value) throws QException + { + return (AnalysisLabels.translate(field, value, fieldAccess)); + } + + /******************************************************************************* + ** Each intermediate exposed-join table requires READ, without analyze rights. + *******************************************************************************/ + public QTableMetaData joinedTable(String tableName) throws QException + { + QueryInput input = new QueryInput(tableName); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + if(QContext.getQInstance().getTable(tableName) == null || !PermissionsHelper.hasTablePermission(input, tableName, TablePermissionSubType.READ)) + { + throw new QBadRequestException("Unknown analysis field."); + } + return (TableMetaDataPersonalizerAction.execute(input, true)); } } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ColumnProfiler.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ColumnProfiler.java index 27945b2..8c81601 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ColumnProfiler.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ColumnProfiler.java @@ -48,13 +48,23 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils; public class ColumnProfiler { private final AnalysisBackend backend; + private final FieldAnalysisAccessResolver fieldAccess; /******************************************************************************* ** The backend can be replaced when core's new operators ship. *******************************************************************************/ public ColumnProfiler(AnalysisBackend backend) + { + this(backend, CoreTierFieldAccessResolver.defaultResolver()); + } + + /******************************************************************************* + ** Bind the host field policy for possible-value labels. + *******************************************************************************/ + public ColumnProfiler(AnalysisBackend backend, FieldAnalysisAccessResolver fieldAccess) { this.backend = backend; + this.fieldAccess = fieldAccess; } @@ -82,7 +92,7 @@ public class ColumnProfiler } result.getCounts().add(ProfileSupport.count("distinct", counts.distinct(), table, drillFilter, counts.distinctFilter())); result.getCounts().add(ProfileSupport.metric("uniqueness", counts.uniqueness(), ValueType.PERCENT)); - result.withTopValues(ProfileSupport.values(table, drillFilter, field, backend.topValues(table, filter, name, 20, 0, null), rows)); + result.withTopValues(ProfileSupport.values(table, drillFilter, field, backend.topValues(table, filter, name, 20, 0, null), rows, fieldAccess)); QFieldType type = field.getType(); if(type == QFieldType.INTEGER || type == QFieldType.LONG || type == QFieldType.DECIMAL) { diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/CoreTierFieldAccessResolver.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/CoreTierFieldAccessResolver.java new file mode 100644 index 0000000..1f9c7c4 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/CoreTierFieldAccessResolver.java @@ -0,0 +1,118 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.lang.reflect.Method; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + +/******************************************************************************* + ** Optional core bridge: replace with direct calls once QQQ ships the fieldaccess package. + ** Class and method discovery happen once; invocation failures fail closed. + *******************************************************************************/ +public class CoreTierFieldAccessResolver implements FieldAnalysisAccessResolver +{ + private static final CoreTierFieldAccessResolver DEFAULT = discover(); + private final Method tier; + private final Method fingerprint; + + /******************************************************************************* + ** Bind discovered static methods once. + *******************************************************************************/ + public CoreTierFieldAccessResolver(Class helper) throws ReflectiveOperationException + { + tier = helper.getMethod("getTier", QTableMetaData.class, String.class); + Method optional; + try + { + optional = helper.getMethod("getAccessFingerprint"); + } + catch(NoSuchMethodException e) + { + optional = null; + } + fingerprint = optional; + } + + /******************************************************************************* + ** Resolve the optional class once per class loader. + *******************************************************************************/ + private static CoreTierFieldAccessResolver discover() + { + try + { + return (new CoreTierFieldAccessResolver(Class.forName("com.kingsrook.qqq.backend.core.actions.permissions.fieldaccess.FieldAccessHelper"))); + } + catch(ClassNotFoundException e) + { + return (null); + } + catch(ReflectiveOperationException e) + { + throw new ExceptionInInitializerError(e); + } + } + + /******************************************************************************* + ** Use legacy locks only when the core helper class is absent. + *******************************************************************************/ + public static FieldAnalysisAccessResolver defaultResolver() + { + return (DEFAULT == null ? new FieldSecurityLockFieldAccessResolver() : DEFAULT); + } + + /******************************************************************************* + ** Unknown tiers and invocation failures never grant analysis rights. + *******************************************************************************/ + @Override + public FieldAnalysisAccess resolve(QTableMetaData table, QFieldMetaData field) + { + try + { + return (switch(String.valueOf(tier.invoke(null, table, field.getName()))) + { + case "FULL" -> FieldAnalysisAccess.FULL; + case "REVEALABLE" -> FieldAnalysisAccess.EXACT_MATCH_ONLY; + default -> FieldAnalysisAccess.NONE; + }); + } + catch(ReflectiveOperationException e) + { + return (FieldAnalysisAccess.NONE); + } + } + + /******************************************************************************* + ** A broken fingerprint prevents cache reuse rather than returning stale data. + *******************************************************************************/ + @Override + public String getAccessFingerprint() + { + try + { + return (fingerprint == null ? "" : (String) fingerprint.invoke(null)); + } + catch(ReflectiveOperationException e) + { + throw new IllegalStateException("Could not resolve field access fingerprint.", e); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/DistinctValues.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/DistinctValues.java index 4ee6dce..d09b914 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/DistinctValues.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/DistinctValues.java @@ -34,13 +34,23 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; public class DistinctValues { private final AnalysisBackend backend; + private final FieldAnalysisAccessResolver fieldAccess; /******************************************************************************* ** Bind primitive operations. *******************************************************************************/ public DistinctValues(AnalysisBackend backend) + { + this(backend, CoreTierFieldAccessResolver.defaultResolver()); + } + + /******************************************************************************* + ** Bind the host field policy for possible-value labels. + *******************************************************************************/ + public DistinctValues(AnalysisBackend backend, FieldAnalysisAccessResolver fieldAccess) { this.backend = backend; + this.fieldAccess = fieldAccess; } @@ -60,7 +70,7 @@ public class DistinctValues boolean wholeListing = config.getMinGroupSize() > 0 || request.getSearch() != null && !request.getSearch().isBlank(); int fetch = wholeListing ? Math.max(1, config.getMaxRows()) : request.getPageSize() + 1; List values = ProfileSupport.values(request.getTableName(), scope.filter(), field, - backend.topValues(request.getTableName(), scope.filter(), field.getName(), fetch, wholeListing ? 0 : offset, request.getSearch()), scope.rowsScanned()); + backend.topValues(request.getTableName(), scope.filter(), field.getName(), fetch, wholeListing ? 0 : offset, request.getSearch()), scope.rowsScanned(), fieldAccess); long totalDistinct = wholeListing ? values.stream().filter(v -> v.getValue() != null).count() : backend.fieldCounts(request.getTableName(), scope.filter(), List.of(field.getName())).get(field.getName()).distinct(); values = new MinGroupSizeGuard(config.getMinGroupSize()).groups(values); diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/FieldAnalysisAccess.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/FieldAnalysisAccess.java new file mode 100644 index 0000000..dee6b1b --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/FieldAnalysisAccess.java @@ -0,0 +1,29 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +/******************************************************************************* + ** Analysis access is stricter than the ability to submit an exact query. + *******************************************************************************/ +public enum FieldAnalysisAccess +{ + FULL, EXACT_MATCH_ONLY, NONE +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/FieldAnalysisAccessResolver.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/FieldAnalysisAccessResolver.java new file mode 100644 index 0000000..91bb0d0 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/FieldAnalysisAccessResolver.java @@ -0,0 +1,43 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + +/******************************************************************************* + ** Resolve the current user's analysis rights without exposing protected values. + *******************************************************************************/ +public interface FieldAnalysisAccessResolver +{ + /******************************************************************************* + ** Only FULL fields may contribute generated analysis predicates. + *******************************************************************************/ + FieldAnalysisAccess resolve(QTableMetaData table, QFieldMetaData field); + + /******************************************************************************* + ** Optional core policy fingerprint supplements the FULL-field list hash. + *******************************************************************************/ + default String getAccessFingerprint() + { + return (""); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/FieldSecurityLockFieldAccessResolver.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/FieldSecurityLockFieldAccessResolver.java new file mode 100644 index 0000000..1175e23 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/FieldSecurityLockFieldAccessResolver.java @@ -0,0 +1,42 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import com.kingsrook.qqq.backend.core.actions.permissions.FieldSecurityLockHelper; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + +/******************************************************************************* + ** Legacy lock adapter. The core tier API (qbit-secured-data) will map REVEALABLE + ** to EXACT_MATCH_ONLY, MASKED/HIDDEN without filter rights to NONE, and + ** MASKED/HIDDEN with filter rights to FULL. + *******************************************************************************/ +public class FieldSecurityLockFieldAccessResolver implements FieldAnalysisAccessResolver +{ + /******************************************************************************* + ** Keep the legacy lock enforcement when core tiers are not installed. + *******************************************************************************/ + @Override + public FieldAnalysisAccess resolve(QTableMetaData table, QFieldMetaData field) + { + return (FieldSecurityLockHelper.isFieldAccessible(field) ? FieldAnalysisAccess.FULL : FieldAnalysisAccess.NONE); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MinGroupSizeGuard.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MinGroupSizeGuard.java index 3331d96..67f3559 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MinGroupSizeGuard.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MinGroupSizeGuard.java @@ -143,4 +143,16 @@ public class MinGroupSizeGuard profile.getText().withHistogram(groups(profile.getText().getHistogram())).withPatterns(groups(profile.getText().getPatterns())); } } + + /******************************************************************************* + ** Small pivot cells suppress every measure, including count, with no drill. + *******************************************************************************/ + public void pivotMetrics(List metrics, long rows) + { + if(rows < minimum) + { + metrics.forEach(m -> m.withValue(null).withFormatted(null).withDrill(null).withSuppressed(true) + .withSuppressedReason("Hidden: fewer than " + minimum + " records")); + } + } } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/NiceBins.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/NiceBins.java index 76807f3..6616b50 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/NiceBins.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/NiceBins.java @@ -33,6 +33,14 @@ public record NiceBins(BigDecimal low, BigDecimal high, int count) ** Select the closest 10..25-bin enclosure from 1, 2, 2.5 and 5 times powers of ten. *******************************************************************************/ public static NiceBins covering(BigDecimal low, BigDecimal high) + { + return (covering(low, high, 20)); + } + + /******************************************************************************* + ** Choose readable enclosing edges for a requested pivot bin count. + *******************************************************************************/ + public static NiceBins covering(BigDecimal low, BigDecimal high, int target) { BigDecimal span = high.subtract(low); if(span.signum() == 0) @@ -41,7 +49,7 @@ public record NiceBins(BigDecimal low, BigDecimal high, int count) low = low.subtract(span.divide(BigDecimal.valueOf(2))); high = high.add(span.divide(BigDecimal.valueOf(2))); } - int exponent = span.precision() - span.scale() - 2; + int exponent = span.precision() - span.scale() - Integer.toString(target).length(); NiceBins best = null; for(int power = exponent - 2; power <= exponent + 2; power++) { @@ -51,7 +59,7 @@ public record NiceBins(BigDecimal low, BigDecimal high, int count) BigDecimal first = low.divide(step, 0, RoundingMode.FLOOR).multiply(step); BigDecimal last = high.divide(step, 0, RoundingMode.CEILING).multiply(step); int count = last.subtract(first).divide(step).intValueExact(); - if(count >= 10 && count <= 25 && (best == null || Math.abs(count - 20) < Math.abs(best.count() - 20))) + if(count >= Math.max(1, target / 2) && count <= Math.max(2, target * 5 / 4) && (best == null || Math.abs(count - target) < Math.abs(best.count() - target))) { best = new NiceBins(first, last, count); } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/PivotExplorer.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/PivotExplorer.java new file mode 100644 index 0000000..b1c23f5 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/PivotExplorer.java @@ -0,0 +1,607 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.io.Serializable; +import java.math.BigDecimal; +import java.math.MathContext; +import java.time.Instant; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import com.kingsrook.qbits.dataanalysis.result.Metric; +import com.kingsrook.qbits.dataanalysis.result.PivotCell; +import com.kingsrook.qbits.dataanalysis.result.PivotDimension; +import com.kingsrook.qbits.dataanalysis.result.PivotHeader; +import com.kingsrook.qbits.dataanalysis.result.PivotMeasure; +import com.kingsrook.qbits.dataanalysis.result.PivotMeasureInfo; +import com.kingsrook.qbits.dataanalysis.result.PivotRequest; +import com.kingsrook.qbits.dataanalysis.result.PivotResult; +import com.kingsrook.qbits.dataanalysis.result.ValueType; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; +import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.DateBucket; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.DateBucketUnit; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.NumericBucket; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.core.utils.aggregates.AggregateAnalysis; + +/******************************************************************************* + ** Request-local pivot planner. Core executes every aggregate with ENFORCE. + ** Merged cells are re-aggregated so non-additive measures remain exact. + *******************************************************************************/ +public class PivotExplorer +{ + private final AnalysisSecurity security; + private final EffectiveConfig config; + private final List joins = new ArrayList<>(); + private final LinkedHashSet toManyTables = new LinkedHashSet<>(); + private final LinkedHashSet multiHopTables = new LinkedHashSet<>(); + private PivotRequest request; + private Aggregate count; + private final List measures = new ArrayList<>(); + private final List measureInfo = new ArrayList<>(); + private boolean partial; + + /******************************************************************************* + ** Bind the current user's policy and bounded streamed fallback. + *******************************************************************************/ + public PivotExplorer(AnalysisSecurity security, EffectiveConfig config) + { + this.security = security; + this.config = config; + } + + /******************************************************************************* + ** One slot is a visible group or the exact complement of the kept groups. + *******************************************************************************/ + private record Slot(Serializable value, String label, QQueryFilter condition, boolean other) + { + } + + /******************************************************************************* + ** Dimension state owns predicates, so no generated filter names hidden fields. + *******************************************************************************/ + private record Dimension(GroupBy group, QFieldMetaData field, Map kept, Map counts, Slot other) + { + /******************************************************************************* + ** Return the kept group, Other, or null for a privacy-suppressed group. + *******************************************************************************/ + Slot slot(Serializable value, int minimum) + { + return (counts.getOrDefault(value, 0L) < minimum ? null : kept.getOrDefault(value, other)); + } + } + + /******************************************************************************* + ** Validate the complete plan before any source computation. + *******************************************************************************/ + public PivotResult execute(PivotRequest request) throws QException + { + this.request = request; + validate(); + String primaryKey = QContext.getQInstance().getTable(request.getTableName()).getPrimaryKeyField(); + count = new Aggregate(primaryKey, AggregateOperator.COUNT).withFieldType(QFieldType.INTEGER); + List requested = new ArrayList<>(request.getRows()); + requested.addAll(request.getColumns()); + List fields = new ArrayList<>(); + for(PivotDimension dimension : requested) + { + fields.add(resolve(dimension.getField())); + } + for(int i = 0; i < request.getValues().size(); i++) + { + prepareMeasure(request.getValues().get(i), i); + } + List dimensions = new ArrayList<>(); + for(int i = 0; i < requested.size(); i++) + { + dimensions.add(dimension(requested.get(i), fields.get(i))); + } + int rowCount = request.getRows().size(); + List rows = dimensions.subList(0, rowCount); + List columns = dimensions.subList(rowCount, dimensions.size()); + PivotResult result = new PivotResult().withAsOf(Instant.now()).withMeasures(measureInfo) + .withTruncatedRows(rows.stream().anyMatch(d -> d.other() != null)) + .withTruncatedColumns(columns.stream().anyMatch(d -> d.other() != null)); + Map, List> matrix = level(dimensions); + Map, List> rowTotals = request.getShowTotals() || config.getMinGroupSize() > 0 ? level(rows) : Map.of(); + Map, List> columnTotals = request.getShowTotals() || config.getMinGroupSize() > 0 ? level(columns) : Map.of(); + List> rowKeys = new ArrayList<>(); + List> columnKeys = new ArrayList<>(); + for(var entry : matrix.entrySet()) + { + List row = List.copyOf(entry.getKey().subList(0, rowCount)); + List column = List.copyOf(entry.getKey().subList(rowCount, dimensions.size())); + if(config.getMinGroupSize() > 0 && (rowTotals.getOrDefault(row, List.of()).stream().allMatch(m -> m.getSuppressed()) + || columnTotals.getOrDefault(column, List.of()).stream().allMatch(m -> m.getSuppressed()))) + { + continue; + } + if(!rowKeys.contains(row)) + { + rowKeys.add(row); + } + if(!columnKeys.contains(column)) + { + columnKeys.add(column); + } + result.getCells().add(new PivotCell().withRowIndex(rowKeys.indexOf(row)).withColumnIndex(columnKeys.indexOf(column)).withValues(entry.getValue())); + } + result.withRowHeaders(headers(rowKeys)).withColumnHeaders(headers(columnKeys)); + if(request.getShowTotals()) + { + rowKeys.forEach(key -> result.getRowTotals().add(rowTotals.getOrDefault(key, List.of()))); + columnKeys.forEach(key -> result.getColumnTotals().add(columnTotals.getOrDefault(key, List.of()))); + result.withGrandTotals(level(List.of()).getOrDefault(List.of(), List.of())); + } + for(String tableName : toManyTables) + { + result.getWarnings().add("Counts include one row per related " + QContext.getQInstance().getTable(tableName).getLabel() + + "; drill-through is not available for this dimension."); + } + for(String tableName : multiHopTables) + { + result.getWarnings().add("Drill-through is not available for fields reached through more than one join (" + QContext.getQInstance().getTable(tableName).getLabel() + ")."); + } + if(config.getMinGroupSize() > 0) + { + result.getWarnings().add("Groups below " + config.getMinGroupSize() + " records are omitted; small cells and totals are suppressed."); + } + if(partial) + { + result.withEstimate(true); + result.getWarnings().add("Partial pivot: the streamed row limit was reached. Drills are disabled because aggregates may cover different prefixes."); + result.getCells().forEach(c -> disableDrills(c.getValues())); + result.getRowTotals().forEach(this::disableDrills); + result.getColumnTotals().forEach(this::disableDrills); + disableDrills(result.getGrandTotals()); + } + return (result); + } + + /******************************************************************************* + ** Reject invalid shapes and types before aggregate execution. + *******************************************************************************/ + private void validate() throws QException + { + if(request.getRows() == null || request.getRows().isEmpty() || request.getRows().size() > 3 + || request.getColumns() == null || request.getColumns().size() > 2 || request.getValues() == null + || request.getValues().isEmpty() || request.getValues().size() > 6 || request.getShowTotals() == null + || request.getLimitGroups() == null || request.getLimitGroups() < 1 || request.getLimitGroups() > 200) + { + throw new QBadRequestException("Choose 1�3 row dimensions, 0�2 column dimensions, 1�6 measures and a group limit from 1 to 200."); + } + List dimensions = new ArrayList<>(request.getRows()); + dimensions.addAll(request.getColumns()); + LinkedHashSet names = new LinkedHashSet<>(); + for(PivotDimension dimension : dimensions) + { + if(dimension == null || dimension.getField() == null || !names.add(dimension.getField())) + { + throw new QBadRequestException("Choose distinct dimension fields."); + } + QFieldMetaData field = resolve(dimension.getField()); + var bins = dimension.getNumericBins(); + if(dimension.getDateBucket() != null && (bins != null || field.getType() != QFieldType.DATE && field.getType() != QFieldType.DATE_TIME) + || bins != null && (!numeric(field) || bins.getCount() == null || bins.getCount() < 1 || bins.getCount() > 200 + || bins.getMin() != null && bins.getMax() != null && bins.getMin().compareTo(bins.getMax()) >= 0)) + { + throw new QBadRequestException("Buckets must match the field type; numeric bins require 1�200 bins and min < max."); + } + } + } + + /******************************************************************************* + ** Only exposed, unambiguous paths may introduce a joined field. + ** The installed snapshot has no related-record picker join-path helper. + *******************************************************************************/ + private QFieldMetaData resolve(String name) throws QException + { + if(name == null || !name.contains(".")) + { + return (security.field(request.getTableName(), name)); + } + String[] parts = name.split("\\.", -1); + var table = QContext.getQInstance().getTable(request.getTableName()); + List exposed = table.getExposedJoins() == null ? List.of() : table.getExposedJoins(); + List paths = exposed.stream().filter(j -> j.getJoinTable().equals(parts[0])).toList(); + if(parts.length != 2 || paths.size() != 1 || paths.getFirst().getJoinPath() == null || paths.getFirst().getJoinPath().isEmpty()) + { + throw new QBadRequestException("Unknown analysis field."); + } + String current = table.getName(); + boolean toOne = true; + for(String joinName : paths.getFirst().getJoinPath()) + { + var metadata = QContext.getQInstance().getJoin(joinName); + if(metadata == null || !current.equals(metadata.getLeftTable()) && !current.equals(metadata.getRightTable())) + { + throw new QBadRequestException("Unknown analysis field."); + } + String next = current.equals(metadata.getLeftTable()) ? metadata.getRightTable() : metadata.getLeftTable(); + JoinType type = metadata.getType(); + if(type != null && current.equals(metadata.getRightTable())) + { + type = type.flip(); + } + toOne &= type == JoinType.MANY_TO_ONE || type == JoinType.ONE_TO_ONE; + security.joinedTable(next); + QueryJoin existing = joins.stream().filter(j -> next.equals(j.getJoinTable())).findFirst().orElse(null); + if(existing == null) + { + joins.add(new QueryJoin(current, next).withJoinMetaData(metadata)); + } + else if(!existing.getJoinMetaData().getName().equals(joinName)) + { + throw new QBadRequestException("Ambiguous analysis join path."); + } + current = next; + } + if(!current.equals(parts[0])) + { + throw new QBadRequestException("Unknown analysis field."); + } + if(!toOne) + { + toManyTables.add(current); + } + else if(paths.getFirst().getJoinPath().size() > 1) + { + /////////////////////////////////////////////////////////////////////////////// + // counts are per record, but core's implicit joins for a multi-hop filter // + // do not reproduce them on every backend yet (memory counts differ), so no // + // drill until that is fixed in core // + /////////////////////////////////////////////////////////////////////////////// + multiHopTables.add(current); + } + return (security.joinedField(parts[0], parts[1])); + } + + /******************************************************************************* + ** Count records even when a supplied COUNT field is nullable. + *******************************************************************************/ + private void prepareMeasure(PivotMeasure measure, int index) throws QException + { + if(measure == null || measure.getOperator() == null) + { + throw new QBadRequestException("Choose a measure operator."); + } + QFieldMetaData field = measure.getField() == null && measure.getOperator() == PivotMeasure.Operator.COUNT ? null : resolve(measure.getField()); + PivotMeasure.Operator operator = measure.getOperator(); + if((operator == PivotMeasure.Operator.SUM || operator == PivotMeasure.Operator.AVG || operator == PivotMeasure.Operator.PERCENTILE) && !numeric(field) + || operator == PivotMeasure.Operator.PERCENTILE && (measure.getPercentile() == null || measure.getPercentile().signum() < 0 || measure.getPercentile().compareTo(BigDecimal.ONE) > 0) + || operator != PivotMeasure.Operator.PERCENTILE && measure.getPercentile() != null) + { + throw new QBadRequestException("Numeric measures require a numeric field; percentile must be between 0 and 1."); + } + Aggregate aggregate = operator == PivotMeasure.Operator.COUNT ? count : new Aggregate(measure.getField(), + operator == PivotMeasure.Operator.PERCENTILE ? AggregateOperator.PERCENTILE_CONT : AggregateOperator.valueOf(operator.name())); + if(operator == PivotMeasure.Operator.PERCENTILE) + { + aggregate.withPercentile(measure.getPercentile()); + } + if(operator == PivotMeasure.Operator.COUNT_DISTINCT) + { + aggregate.withFieldType(QFieldType.INTEGER); + } + measures.add(aggregate); + ValueType type = operator == PivotMeasure.Operator.COUNT || operator == PivotMeasure.Operator.COUNT_DISTINCT ? ValueType.COUNT + : numeric(field) ? ValueType.DECIMAL : field.getType() == QFieldType.DATE ? ValueType.DATE : field.getType() == QFieldType.DATE_TIME ? ValueType.DATE_TIME : ValueType.TEXT; + measureInfo.add(new PivotMeasureInfo().withKey("value" + index).withLabel(operator.name() + (field == null ? " records" : " " + field.getLabel())).withValueType(type)); + } + + /******************************************************************************* + ** Numeric buckets and moments accept the same numeric QQQ field types. + *******************************************************************************/ + private static boolean numeric(QFieldMetaData field) + { + return (field != null && (field.getType() == QFieldType.INTEGER || field.getType() == QFieldType.LONG || field.getType() == QFieldType.DECIMAL)); + } + + /******************************************************************************* + ** Select top groups first, then collect cardinality and privacy counts. + *******************************************************************************/ + private Dimension dimension(PivotDimension requested, QFieldMetaData field) throws QException + { + GroupBy group = new GroupBy(field.getType(), requested.getField()); + if(requested.getDateBucket() != null) + { + group.withBucket(new DateBucket(requested.getDateBucket())); + } + if(requested.getNumericBins() != null) + { + var bins = requested.getNumericBins(); + BigDecimal low = bins.getMin(); + BigDecimal high = bins.getMax(); + if(low == null || high == null) + { + Aggregate min = new Aggregate(requested.getField(), AggregateOperator.MIN); + Aggregate max = new Aggregate(requested.getField(), AggregateOperator.MAX); + var extrema = aggregate(List.of(), request.getFilter(), List.of(min, max), null); + Serializable first = extrema.isEmpty() ? null : extrema.getFirst().getAggregateValues().get(min); + Serializable last = extrema.isEmpty() ? null : extrema.getFirst().getAggregateValues().get(max); + NiceBins nice = NiceBins.covering(first == null ? BigDecimal.ZERO : new BigDecimal(first.toString()), last == null ? BigDecimal.ONE : new BigDecimal(last.toString()), bins.getCount()); + low = low == null ? nice.low() : low; + high = high == null ? nice.high() : high; + } + if(low.compareTo(high) >= 0) + { + throw new QBadRequestException("Numeric bin min must be below max."); + } + group.withBucket(new NumericBucket(low, high, bins.getCount())); + } + List top = aggregate(List.of(group), request.getFilter(), List.of(count), request.getLimitGroups()); + List all = aggregate(List.of(group), request.getFilter(), List.of(count), null); + Map counts = new LinkedHashMap<>(); + all.forEach(r -> counts.put(r.getGroupByValues().get(group), count(r))); + Map kept = new LinkedHashMap<>(); + for(AggregateResult row : top) + { + Serializable value = row.getGroupByValues().get(group); + if(count(row) >= config.getMinGroupSize()) + { + kept.put(value, new Slot(value, label(group, field, value), condition(group, field, value), false)); + } + } + List remaining = counts.keySet().stream().filter(v -> !kept.containsKey(v) && counts.get(v) >= config.getMinGroupSize()).toList(); + Slot other = null; + if(!remaining.isEmpty()) + { + QQueryFilter predicate; + if(group.getBucket() == null && field.getType() != QFieldType.BOOLEAN && config.getMinGroupSize() == 0 && !kept.containsKey("") && kept.keySet().stream().noneMatch(v -> v instanceof Instant || v instanceof LocalDate)) + { + List values = kept.keySet().stream().filter(Objects::nonNull).toList(); + predicate = values.isEmpty() ? new QQueryFilter() : new QQueryFilter().withCriteria(group.getFieldName(), QCriteriaOperator.NOT_IN, values); + predicate = DrillFilters.and(predicate, DrillFilters.nonNull(group.getFieldName(), field.getType().isStringLike())); + if(!kept.containsKey(null)) + { + predicate = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR).withSubFilter(predicate).withSubFilter(DrillFilters.nulls(group.getFieldName(), field.getType().isStringLike())); + } + if(field.getType().isStringLike()) + { + predicate = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR).withSubFilter(predicate).withSubFilter(DrillFilters.value(group.getFieldName(), "", true)); + } + } + else + { + predicate = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR); + for(Serializable value : remaining) + { + predicate.withSubFilter(condition(group, field, value)); + } + } + other = new Slot(null, "Other (" + remaining.size() + " groups)", predicate, true); + } + return (new Dimension(group, field, kept, counts, other)); + } + + /******************************************************************************* + ** Native group keys are displayed as labels without changing filter operands. + *******************************************************************************/ + private String label(GroupBy group, QFieldMetaData field, Serializable value) throws QException + { + if(value == null) + { + return ("(Blank)"); + } + if(group.getBucket() instanceof DateBucket date) + { + return (AnalysisFormatting.dateBucket(value, AnalysisBackend.DateUnit.valueOf(date.getUnit().name()), ValueUtils.getSessionOrInstanceZoneId())); + } + if(group.getBucket() instanceof NumericBucket bins) + { + BigDecimal[] edges = edges(bins, ((Number) value).intValue()); + return (AnalysisFormatting.interval(field, edges[0], edges[1])); + } + return (security.label(field, value)); + } + + /******************************************************************************* + ** Match core's final closed numeric bin and exterior bins. + *******************************************************************************/ + private static BigDecimal[] edges(NumericBucket bins, int index) + { + BigDecimal width = bins.getMax().subtract(bins.getMin()).divide(BigDecimal.valueOf(bins.getCount()), MathContext.DECIMAL128); + return (new BigDecimal[] { index == 0 ? null : index > bins.getCount() ? bins.getMax() : bins.getMin().add(width.multiply(BigDecimal.valueOf(index - 1))), + index == 0 ? bins.getMin() : index > bins.getCount() ? null : index == bins.getCount() ? bins.getMax() : bins.getMin().add(width.multiply(BigDecimal.valueOf(index))) }); + } + + /******************************************************************************* + ** Calendar and numeric predicates use core bucket boundaries in the session zone. + *******************************************************************************/ + private QQueryFilter condition(GroupBy group, QFieldMetaData field, Serializable value) throws QException + { + String name = group.getFieldName(); + if(value == null || group.getBucket() == null) + { + return (DrillFilters.value(name, value, field.getType().isStringLike())); + } + if(group.getBucket() instanceof NumericBucket bins) + { + int index = ((Number) value).intValue(); + BigDecimal[] edges = edges(bins, index); + if(StandInAnalysisBackend.isMemory(request.getTableName()) && index <= bins.getCount()) + { + GroupBy raw = new GroupBy(field.getType(), name); + List members = aggregate(List.of(raw), request.getFilter(), List.of(count), null).stream() + .map(r -> r.getGroupByValues().get(raw)).filter(Objects::nonNull).map(v -> new BigDecimal(v.toString())).toList(); + return (new StandInAnalysisBackend(config).numericRange(request.getTableName(), name, members, edges[0], edges[1], index == bins.getCount())); + } + return (index > bins.getCount() ? DrillFilters.condition(name, QCriteriaOperator.GREATER_THAN, bins.getMax()) + : DrillFilters.range(name, edges[0], edges[1], index == bins.getCount())); + } + DateBucketUnit unit = ((DateBucket) group.getBucket()).getUnit(); + if(unit == DateBucketUnit.HOUR) + { + Instant start = ValueUtils.getValueAsInstant(value); + return (DrillFilters.range(name, start, start.plusSeconds(3600), false)); + } + if(unit == DateBucketUnit.DAY_OF_WEEK || unit == DateBucketUnit.HOUR_OF_DAY) + { + GroupBy raw = new GroupBy(field.getType(), name); + List members = aggregate(List.of(raw), request.getFilter(), List.of(count), null).stream().map(r -> r.getGroupByValues().get(raw)) + .filter(v -> Objects.equals(AggregateAnalysis.bucket(v, group.getBucket(), ValueUtils.getSessionOrInstanceZoneId()), value)).toList(); + return (DrillFilters.in(name, members)); + } + LocalDate start = ValueUtils.getValueAsLocalDate(value); + LocalDate end = switch(unit) + { + case YEAR -> start.plusYears(1); + case QUARTER -> start.plusMonths(3); + case MONTH -> start.plusMonths(1); + case WEEK -> start.plusWeeks(1); + default -> start.plusDays(1); + }; + return (DrillFilters.dateRange(name, start, end, field.getType() == QFieldType.DATE_TIME, ValueUtils.getSessionOrInstanceZoneId())); + } + + /******************************************************************************* + ** A level is the full matrix, row totals, column totals or grand totals. + *******************************************************************************/ + private Map, List> level(List dimensions) throws QException + { + List requested = new ArrayList<>(measures); + if(!requested.contains(count)) + { + requested.add(count); + } + Map, List> grouped = new LinkedHashMap<>(); + List aggregates = aggregate(dimensions.stream().map(Dimension::group).toList(), request.getFilter(), requested, null); + if(dimensions.isEmpty() && aggregates.isEmpty()) + { + AggregateResult empty = new AggregateResult(); + requested.forEach(a -> empty.withAggregateValue(a, a.getOperator() == AggregateOperator.COUNT || a.getOperator() == AggregateOperator.COUNT_DISTINCT ? 0L : null)); + aggregates = List.of(empty); + } + for(AggregateResult row : aggregates) + { + List key = new ArrayList<>(); + for(Dimension dimension : dimensions) + { + key.add(dimension.slot(row.getGroupByValues().get(dimension.group()), config.getMinGroupSize())); + } + if(!key.contains(null)) + { + grouped.computeIfAbsent(key, ignored -> new ArrayList<>()).add(row); + } + } + Map, List> result = new LinkedHashMap<>(); + for(var entry : grouped.entrySet()) + { + QQueryFilter predicate = new QQueryFilter(); + entry.getKey().forEach(slot -> predicate.withSubFilter(slot.condition())); + AggregateResult row = entry.getValue().getFirst(); + if(entry.getKey().stream().anyMatch(Slot::other)) + { + var merged = aggregate(List.of(), DrillFilters.and(request.getFilter(), predicate), requested, null); + if(merged.isEmpty()) + { + continue; + } + row = merged.getFirst(); + } + List values = new ArrayList<>(); + for(int i = 0; i < measures.size(); i++) + { + PivotMeasureInfo info = measureInfo.get(i); + Serializable value = row.getAggregateValues().get(measures.get(i)); + Metric metric = new Metric().withKey(info.getKey()).withLabel(info.getLabel()).withValueType(info.getValueType()).withValue(value) + .withFormatted(AnalysisFormatting.scalar(info.getKey(), value, info.getValueType())); + if(toManyTables.isEmpty() && multiHopTables.isEmpty()) + { + metric.withDrill(DrillFilters.drill(request.getTableName(), request.getFilter(), predicate)); + } + values.add(metric); + } + new MinGroupSizeGuard(config.getMinGroupSize()).pivotMetrics(values, count(row)); + result.put(entry.getKey(), values); + } + return (result); + } + + /******************************************************************************* + ** Count is always present, even when the user selects only non-count measures. + *******************************************************************************/ + private long count(AggregateResult row) + { + Object value = row.getAggregateValues().get(count); + return (value == null ? 0 : ((Number) value).longValue()); + } + + /******************************************************************************* + ** Core owns SQL pushdown, joins, record locks, field locks and bounded fallback. + *******************************************************************************/ + private List aggregate(List groups, QQueryFilter filter, List aggregates, Integer limit) throws QException + { + QQueryFilter query = DrillFilters.restoreTypes(request.getTableName(), filter); + if(limit != null) + { + query.withOrderBy(new QFilterOrderByAggregate(count, false)); + } + AggregateInput input = new AggregateInput(request.getTableName()).withFilter(query).withQueryJoins(joins.stream().map(QueryJoin::clone).toList()) + .withAllowStreamedFallback(true).withStreamedRowLimit(config.getMaxRows()).withLimit(limit); + groups.forEach(input::withGroupBy); + new LinkedHashSet<>(aggregates).forEach(input::withAggregate); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + var output = new AggregateAction().execute(input); + partial |= output.getTruncated(); + return (limit == null ? output.getResults() : output.getResults().stream().sorted(java.util.Comparator.comparingLong(this::count).reversed() + .thenComparing(r -> String.valueOf(r.getGroupByValues().values()))).limit(limit).toList()); + } + + /******************************************************************************* + ** Header tuples retain typed values; Other is distinguished by its label. + *******************************************************************************/ + private static List> headers(List> keys) + { + return (keys.stream().map(key -> key.stream().map(slot -> new PivotHeader().withValue(slot.value()).withLabel(slot.label())).toList()).toList()); + } + + /******************************************************************************* + ** Partial numbers must never claim an exact drill equivalence. + *******************************************************************************/ + private void disableDrills(List metrics) + { + metrics.forEach(m -> m.withDrill(null).withEstimate(true)); + } +} + diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ProfileSupport.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ProfileSupport.java index 716de31..4039225 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ProfileSupport.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ProfileSupport.java @@ -108,6 +108,15 @@ public final class ProfileSupport ** PVS translations run as the current user; drill values always remain ids. *******************************************************************************/ public static List values(String table, QQueryFilter filter, QFieldMetaData field, List groups, long total) throws QException + { + return (values(table, filter, field, groups, total, CoreTierFieldAccessResolver.defaultResolver())); + } + + /******************************************************************************* + ** Host field policy applies to related display labels as well as source values. + *******************************************************************************/ + public static List values(String table, QQueryFilter filter, QFieldMetaData field, List groups, long total, + FieldAnalysisAccessResolver access) throws QException { List result = new ArrayList<>(); for(AnalysisBackend.GroupCount group : groups) @@ -115,7 +124,7 @@ public final class ProfileSupport String label = group.label(); if(field != null && field.getPossibleValueSourceName() != null && group.value() != null) { - String translated = AnalysisLabels.translate(field, group.value()); + String translated = AnalysisLabels.translate(field, group.value(), access); if(translated != null) { label = translated; diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotCell.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotCell.java new file mode 100644 index 0000000..8ecdccb --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotCell.java @@ -0,0 +1,89 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +/******************************************************************************* + ** Typed PivotCell contract. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class PivotCell implements Serializable +{ + private Integer rowIndex; + private Integer columnIndex; + private List values = new ArrayList<>(); + + + /******************************************************************************* + ** Getter for rowIndex. + *******************************************************************************/ + public Integer getRowIndex() + { + return (rowIndex); + } + + /******************************************************************************* + ** Fluent setter for rowIndex. + *******************************************************************************/ + public PivotCell withRowIndex(Integer rowIndex) + { + this.rowIndex = rowIndex; + return (this); + } + + /******************************************************************************* + ** Getter for columnIndex. + *******************************************************************************/ + public Integer getColumnIndex() + { + return (columnIndex); + } + + /******************************************************************************* + ** Fluent setter for columnIndex. + *******************************************************************************/ + public PivotCell withColumnIndex(Integer columnIndex) + { + this.columnIndex = columnIndex; + return (this); + } + + /******************************************************************************* + ** Getter for values. + *******************************************************************************/ + public List getValues() + { + return (values); + } + + /******************************************************************************* + ** Fluent setter for values. + *******************************************************************************/ + public PivotCell withValues(List values) + { + this.values = values; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotDimension.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotDimension.java new file mode 100644 index 0000000..ffe9a13 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotDimension.java @@ -0,0 +1,88 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + +import java.io.Serializable; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.DateBucketUnit; + +/******************************************************************************* + ** Typed PivotDimension contract. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class PivotDimension implements Serializable +{ + private String field; + private DateBucketUnit dateBucket; + private PivotNumericBins numericBins; + + + /******************************************************************************* + ** Getter for field. + *******************************************************************************/ + public String getField() + { + return (field); + } + + /******************************************************************************* + ** Fluent setter for field. + *******************************************************************************/ + public PivotDimension withField(String field) + { + this.field = field; + return (this); + } + + /******************************************************************************* + ** Getter for dateBucket. + *******************************************************************************/ + public DateBucketUnit getDateBucket() + { + return (dateBucket); + } + + /******************************************************************************* + ** Fluent setter for dateBucket. + *******************************************************************************/ + public PivotDimension withDateBucket(DateBucketUnit dateBucket) + { + this.dateBucket = dateBucket; + return (this); + } + + /******************************************************************************* + ** Getter for numericBins. + *******************************************************************************/ + public PivotNumericBins getNumericBins() + { + return (numericBins); + } + + /******************************************************************************* + ** Fluent setter for numericBins. + *******************************************************************************/ + public PivotDimension withNumericBins(PivotNumericBins numericBins) + { + this.numericBins = numericBins; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotHeader.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotHeader.java new file mode 100644 index 0000000..22f9b8d --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotHeader.java @@ -0,0 +1,69 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + +import java.io.Serializable; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +/******************************************************************************* + ** Typed PivotHeader contract. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class PivotHeader implements Serializable +{ + private Serializable value; + private String label; + + + /******************************************************************************* + ** Getter for value. + *******************************************************************************/ + public Serializable getValue() + { + return (value); + } + + /******************************************************************************* + ** Fluent setter for value. + *******************************************************************************/ + public PivotHeader withValue(Serializable value) + { + this.value = value; + return (this); + } + + /******************************************************************************* + ** Getter for label. + *******************************************************************************/ + public String getLabel() + { + return (label); + } + + /******************************************************************************* + ** Fluent setter for label. + *******************************************************************************/ + public PivotHeader withLabel(String label) + { + this.label = label; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotMeasure.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotMeasure.java new file mode 100644 index 0000000..186cae2 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotMeasure.java @@ -0,0 +1,95 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + +import java.io.Serializable; +import java.math.BigDecimal; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +/******************************************************************************* + ** Typed PivotMeasure contract. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class PivotMeasure implements Serializable +{ + private Operator operator; + private String field; + private BigDecimal percentile; + + /******************************************************************************* + ** Supported pivot measures. COUNT counts records, regardless of field nulls. + *******************************************************************************/ + public enum Operator + { + COUNT, SUM, AVG, MIN, MAX, COUNT_DISTINCT, PERCENTILE + } + + /******************************************************************************* + ** Getter for operator. + *******************************************************************************/ + public Operator getOperator() + { + return (operator); + } + + /******************************************************************************* + ** Fluent setter for operator. + *******************************************************************************/ + public PivotMeasure withOperator(Operator operator) + { + this.operator = operator; + return (this); + } + + /******************************************************************************* + ** Getter for field. + *******************************************************************************/ + public String getField() + { + return (field); + } + + /******************************************************************************* + ** Fluent setter for field. + *******************************************************************************/ + public PivotMeasure withField(String field) + { + this.field = field; + return (this); + } + + /******************************************************************************* + ** Getter for percentile. + *******************************************************************************/ + public BigDecimal getPercentile() + { + return (percentile); + } + + /******************************************************************************* + ** Fluent setter for percentile. + *******************************************************************************/ + public PivotMeasure withPercentile(BigDecimal percentile) + { + this.percentile = percentile; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotMeasureInfo.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotMeasureInfo.java new file mode 100644 index 0000000..b3abf97 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotMeasureInfo.java @@ -0,0 +1,87 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + +import java.io.Serializable; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +/******************************************************************************* + ** Typed PivotMeasureInfo contract. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class PivotMeasureInfo implements Serializable +{ + private String key; + private String label; + private ValueType valueType; + + + /******************************************************************************* + ** Getter for key. + *******************************************************************************/ + public String getKey() + { + return (key); + } + + /******************************************************************************* + ** Fluent setter for key. + *******************************************************************************/ + public PivotMeasureInfo withKey(String key) + { + this.key = key; + return (this); + } + + /******************************************************************************* + ** Getter for label. + *******************************************************************************/ + public String getLabel() + { + return (label); + } + + /******************************************************************************* + ** Fluent setter for label. + *******************************************************************************/ + public PivotMeasureInfo withLabel(String label) + { + this.label = label; + return (this); + } + + /******************************************************************************* + ** Getter for valueType. + *******************************************************************************/ + public ValueType getValueType() + { + return (valueType); + } + + /******************************************************************************* + ** Fluent setter for valueType. + *******************************************************************************/ + public PivotMeasureInfo withValueType(ValueType valueType) + { + this.valueType = valueType; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotNumericBins.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotNumericBins.java new file mode 100644 index 0000000..3d634ec --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotNumericBins.java @@ -0,0 +1,88 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + +import java.io.Serializable; +import java.math.BigDecimal; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +/******************************************************************************* + ** Typed PivotNumericBins contract. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class PivotNumericBins implements Serializable +{ + private Integer count = 20; + private BigDecimal min; + private BigDecimal max; + + + /******************************************************************************* + ** Getter for count. + *******************************************************************************/ + public Integer getCount() + { + return (count); + } + + /******************************************************************************* + ** Fluent setter for count. + *******************************************************************************/ + public PivotNumericBins withCount(Integer count) + { + this.count = count; + return (this); + } + + /******************************************************************************* + ** Getter for min. + *******************************************************************************/ + public BigDecimal getMin() + { + return (min); + } + + /******************************************************************************* + ** Fluent setter for min. + *******************************************************************************/ + public PivotNumericBins withMin(BigDecimal min) + { + this.min = min; + return (this); + } + + /******************************************************************************* + ** Getter for max. + *******************************************************************************/ + public BigDecimal getMax() + { + return (max); + } + + /******************************************************************************* + ** Fluent setter for max. + *******************************************************************************/ + public PivotNumericBins withMax(BigDecimal max) + { + this.max = max; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotRequest.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotRequest.java new file mode 100644 index 0000000..db9683a --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotRequest.java @@ -0,0 +1,182 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + +import java.util.ArrayList; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; + +/******************************************************************************* + ** Typed PivotRequest contract. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class PivotRequest implements AnalysisRequest +{ + private String tableName; + private QQueryFilter filter; + private Boolean exact = true; + private List rows = new ArrayList<>(); + private List columns = new ArrayList<>(); + private List values = new ArrayList<>(); + private Boolean showTotals = true; + private Integer limitGroups = 50; + + + /******************************************************************************* + ** Getter for tableName. + *******************************************************************************/ + @Override + public String getTableName() + { + return (tableName); + } + + /******************************************************************************* + ** Fluent setter for tableName. + *******************************************************************************/ + public PivotRequest withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + /******************************************************************************* + ** Getter for filter. + *******************************************************************************/ + @Override + public QQueryFilter getFilter() + { + return (filter); + } + + /******************************************************************************* + ** Fluent setter for filter. + *******************************************************************************/ + public PivotRequest withFilter(QQueryFilter filter) + { + this.filter = filter; + return (this); + } + + /******************************************************************************* + ** Getter for exact. + *******************************************************************************/ + @Override + public Boolean getExact() + { + return (exact); + } + + /******************************************************************************* + ** Fluent setter for exact. + *******************************************************************************/ + public PivotRequest withExact(Boolean exact) + { + this.exact = exact; + return (this); + } + + /******************************************************************************* + ** Getter for rows. + *******************************************************************************/ + public List getRows() + { + return (rows); + } + + /******************************************************************************* + ** Fluent setter for rows. + *******************************************************************************/ + public PivotRequest withRows(List rows) + { + this.rows = rows; + return (this); + } + + /******************************************************************************* + ** Getter for columns. + *******************************************************************************/ + public List getColumns() + { + return (columns); + } + + /******************************************************************************* + ** Fluent setter for columns. + *******************************************************************************/ + public PivotRequest withColumns(List columns) + { + this.columns = columns; + return (this); + } + + /******************************************************************************* + ** Getter for values. + *******************************************************************************/ + public List getValues() + { + return (values); + } + + /******************************************************************************* + ** Fluent setter for values. + *******************************************************************************/ + public PivotRequest withValues(List values) + { + this.values = values; + return (this); + } + + /******************************************************************************* + ** Getter for showTotals. + *******************************************************************************/ + public Boolean getShowTotals() + { + return (showTotals); + } + + /******************************************************************************* + ** Fluent setter for showTotals. + *******************************************************************************/ + public PivotRequest withShowTotals(Boolean showTotals) + { + this.showTotals = showTotals; + return (this); + } + + /******************************************************************************* + ** Getter for limitGroups. + *******************************************************************************/ + public Integer getLimitGroups() + { + return (limitGroups); + } + + /******************************************************************************* + ** Fluent setter for limitGroups. + *******************************************************************************/ + public PivotRequest withLimitGroups(Integer limitGroups) + { + this.limitGroups = limitGroups; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotResult.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotResult.java new file mode 100644 index 0000000..4e33020 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/PivotResult.java @@ -0,0 +1,252 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + +import java.io.Serializable; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +/******************************************************************************* + ** Typed PivotResult contract. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class PivotResult implements Serializable +{ + private List> rowHeaders = new ArrayList<>(); + private List> columnHeaders = new ArrayList<>(); + private List measures = new ArrayList<>(); + private List cells = new ArrayList<>(); + private List> rowTotals = new ArrayList<>(); + private List> columnTotals = new ArrayList<>(); + private List grandTotals = new ArrayList<>(); + private Boolean truncatedRows = false; + private Boolean truncatedColumns = false; + private Boolean estimate = false; + private List warnings = new ArrayList<>(); + private Instant asOf; + + + /******************************************************************************* + ** Getter for rowHeaders. + *******************************************************************************/ + public List> getRowHeaders() + { + return (rowHeaders); + } + + /******************************************************************************* + ** Fluent setter for rowHeaders. + *******************************************************************************/ + public PivotResult withRowHeaders(List> rowHeaders) + { + this.rowHeaders = rowHeaders; + return (this); + } + + /******************************************************************************* + ** Getter for columnHeaders. + *******************************************************************************/ + public List> getColumnHeaders() + { + return (columnHeaders); + } + + /******************************************************************************* + ** Fluent setter for columnHeaders. + *******************************************************************************/ + public PivotResult withColumnHeaders(List> columnHeaders) + { + this.columnHeaders = columnHeaders; + return (this); + } + + /******************************************************************************* + ** Getter for measures. + *******************************************************************************/ + public List getMeasures() + { + return (measures); + } + + /******************************************************************************* + ** Fluent setter for measures. + *******************************************************************************/ + public PivotResult withMeasures(List measures) + { + this.measures = measures; + return (this); + } + + /******************************************************************************* + ** Getter for cells. + *******************************************************************************/ + public List getCells() + { + return (cells); + } + + /******************************************************************************* + ** Fluent setter for cells. + *******************************************************************************/ + public PivotResult withCells(List cells) + { + this.cells = cells; + return (this); + } + + /******************************************************************************* + ** Getter for rowTotals. + *******************************************************************************/ + public List> getRowTotals() + { + return (rowTotals); + } + + /******************************************************************************* + ** Fluent setter for rowTotals. + *******************************************************************************/ + public PivotResult withRowTotals(List> rowTotals) + { + this.rowTotals = rowTotals; + return (this); + } + + /******************************************************************************* + ** Getter for columnTotals. + *******************************************************************************/ + public List> getColumnTotals() + { + return (columnTotals); + } + + /******************************************************************************* + ** Fluent setter for columnTotals. + *******************************************************************************/ + public PivotResult withColumnTotals(List> columnTotals) + { + this.columnTotals = columnTotals; + return (this); + } + + /******************************************************************************* + ** Getter for grandTotals. + *******************************************************************************/ + public List getGrandTotals() + { + return (grandTotals); + } + + /******************************************************************************* + ** Fluent setter for grandTotals. + *******************************************************************************/ + public PivotResult withGrandTotals(List grandTotals) + { + this.grandTotals = grandTotals; + return (this); + } + + /******************************************************************************* + ** Getter for truncatedRows. + *******************************************************************************/ + public Boolean getTruncatedRows() + { + return (truncatedRows); + } + + /******************************************************************************* + ** Fluent setter for truncatedRows. + *******************************************************************************/ + public PivotResult withTruncatedRows(Boolean truncatedRows) + { + this.truncatedRows = truncatedRows; + return (this); + } + + /******************************************************************************* + ** Getter for truncatedColumns. + *******************************************************************************/ + public Boolean getTruncatedColumns() + { + return (truncatedColumns); + } + + /******************************************************************************* + ** Fluent setter for truncatedColumns. + *******************************************************************************/ + public PivotResult withTruncatedColumns(Boolean truncatedColumns) + { + this.truncatedColumns = truncatedColumns; + return (this); + } + + /******************************************************************************* + ** Getter for estimate. + *******************************************************************************/ + public Boolean getEstimate() + { + return (estimate); + } + + /******************************************************************************* + ** Fluent setter for estimate. + *******************************************************************************/ + public PivotResult withEstimate(Boolean estimate) + { + this.estimate = estimate; + return (this); + } + + /******************************************************************************* + ** Getter for warnings. + *******************************************************************************/ + public List getWarnings() + { + return (warnings); + } + + /******************************************************************************* + ** Fluent setter for warnings. + *******************************************************************************/ + public PivotResult withWarnings(List warnings) + { + this.warnings = warnings; + return (this); + } + + /******************************************************************************* + ** Getter for asOf. + *******************************************************************************/ + public Instant getAsOf() + { + return (asOf); + } + + /******************************************************************************* + ** Fluent setter for asOf. + *******************************************************************************/ + public PivotResult withAsOf(Instant asOf) + { + this.asOf = asOf; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/run/AnalysisRunService.java b/src/main/java/com/kingsrook/qbits/dataanalysis/run/AnalysisRunService.java index f9abd27..b519af8 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/run/AnalysisRunService.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/run/AnalysisRunService.java @@ -441,7 +441,8 @@ public class AnalysisRunService identity.put("zone", ValueUtils.getSessionOrInstanceZoneId().getId()); identity.put("config", service.getConfigResolver().resolve(request.getTableName())); identity.put("fields", service.capabilities(request.getTableName()).getFields().stream().sorted().toList()); - identity.put("policy", com.kingsrook.qbits.dataanalysis.engine.AnalysisLabels.policy(request.getTableName())); + identity.put("fieldAccess", digest(service.fieldAccessFingerprint(request.getTableName()))); + identity.put("policy", service.fieldAccessPolicy(request.getTableName())); identity.put("backendVariants", QContext.getQSession().getBackendVariants()); try { diff --git a/src/main/resources/openapi/qqq-v1/data-analysis.yaml b/src/main/resources/openapi/qqq-v1/data-analysis.yaml index de5cdc6..5d53e6d 100644 --- a/src/main/resources/openapi/qqq-v1/data-analysis.yaml +++ b/src/main/resources/openapi/qqq-v1/data-analysis.yaml @@ -638,6 +638,206 @@ components: nullable: true type: "array" type: "object" + DataAnalysisPivotCellV1: + properties: + rowIndex: + description: "Zero-based index in rowHeaders." + type: "integer" + columnIndex: + description: "Zero-based index in columnHeaders." + type: "integer" + values: + description: "Metrics in requested measure order; suppressed metrics have\ + \ no value or drill." + items: + $ref: "#/components/schemas/DataAnalysisMetricV1" + type: "array" + type: "object" + DataAnalysisPivotDimensionV1: + properties: + field: + description: "FULL field name, optionally joinTable.field through an exposed\ + \ join." + type: "string" + dateBucket: + description: "Optional calendar or cyclic bucket in the session timezone." + enum: + - "YEAR" + - "QUARTER" + - "MONTH" + - "WEEK" + - "DAY" + - "HOUR" + - "DAY_OF_WEEK" + - "HOUR_OF_DAY" + type: "string" + numericBins: + $ref: "#/components/schemas/DataAnalysisPivotNumericBinsV1" + description: "Optional equal-width numeric bins; mutually exclusive with\ + \ dateBucket." + required: + - "field" + type: "object" + DataAnalysisPivotHeaderV1: + properties: + value: + description: "Typed group key; null for a blank group or Other." + nullable: true + label: + description: "Display label, including Other (k groups)." + type: "string" + type: "object" + DataAnalysisPivotMeasureInfoV1: + properties: + key: + description: "Stable measure key by request position." + type: "string" + label: + description: "Display label." + type: "string" + valueType: + description: "Semantic metric type." + enum: + - "COUNT" + - "DECIMAL" + - "PERCENT" + - "DATE" + - "DATE_TIME" + - "DURATION" + - "TEXT" + type: "string" + type: "object" + DataAnalysisPivotMeasureV1: + properties: + operator: + description: "Aggregate operator. COUNT counts records, including null values." + enum: + - "COUNT" + - "SUM" + - "AVG" + - "MIN" + - "MAX" + - "COUNT_DISTINCT" + - "PERCENTILE" + type: "string" + field: + description: "FULL field to aggregate; optional for COUNT." + type: "string" + percentile: + description: "PERCENTILE only: percentile_cont fraction from 0 to 1." + type: "number" + required: + - "operator" + type: "object" + DataAnalysisPivotNumericBinsV1: + properties: + count: + description: "Number of bins, 1–200; defaults to 20." + type: "integer" + min: + description: "Optional explicit lower bound; otherwise a nice enclosing\ + \ edge." + type: "number" + max: + description: "Optional explicit upper bound; final interior bin includes\ + \ this value." + type: "number" + type: "object" + DataAnalysisPivotRequestV1: + properties: + filter: + $ref: "#/components/schemas/QueryFilter" + description: "User query filter, passed through for core enforcement." + rows: + description: "One to three row dimensions." + items: + $ref: "#/components/schemas/DataAnalysisPivotDimensionV1" + type: "array" + columns: + description: "Zero to two column dimensions; defaults to empty." + items: + $ref: "#/components/schemas/DataAnalysisPivotDimensionV1" + type: "array" + values: + description: "One to six measures." + items: + $ref: "#/components/schemas/DataAnalysisPivotMeasureV1" + type: "array" + showTotals: + description: "Include row, column and grand totals; defaults to true." + type: "boolean" + limitGroups: + description: "Kept groups per dimension; default 50, maximum 200." + type: "integer" + required: + - "rows" + - "values" + type: "object" + DataAnalysisPivotV1: + properties: + rowHeaders: + description: "Row header tuples in cell-index order." + items: + items: + $ref: "#/components/schemas/DataAnalysisPivotHeaderV1" + type: "array" + type: "array" + columnHeaders: + description: "Column header tuples; an empty tuple represents no column\ + \ dimensions." + items: + items: + $ref: "#/components/schemas/DataAnalysisPivotHeaderV1" + type: "array" + type: "array" + measures: + description: "Measure metadata in request order." + items: + $ref: "#/components/schemas/DataAnalysisPivotMeasureInfoV1" + type: "array" + cells: + description: "Sparse cells; absent combinations are not materialized." + items: + $ref: "#/components/schemas/DataAnalysisPivotCellV1" + type: "array" + rowTotals: + description: "Metric tuples aligned with rowHeaders; empty when totals disabled." + items: + items: + $ref: "#/components/schemas/DataAnalysisMetricV1" + type: "array" + type: "array" + columnTotals: + description: "Metric tuples aligned with columnHeaders; empty when totals\ + \ disabled." + items: + items: + $ref: "#/components/schemas/DataAnalysisMetricV1" + type: "array" + type: "array" + grandTotals: + description: "Grand-total metrics; empty when totals disabled." + items: + $ref: "#/components/schemas/DataAnalysisMetricV1" + type: "array" + truncatedRows: + description: "Row groups were merged into Other." + type: "boolean" + truncatedColumns: + description: "Column groups were merged into Other." + type: "boolean" + estimate: + description: "True when bounded streamed fallback produced a partial result." + type: "boolean" + warnings: + description: "Privacy, partial-result and joined-drill limitations." + items: + type: "string" + type: "array" + asOf: + description: "UTC timestamp of computation." + type: "string" + type: "object" DataAnalysisPointV1: nullable: true properties: @@ -965,10 +1165,10 @@ components: info: contact: email: "contact@kingsrook.com" - description: "Profiles, distinct values, duplicate groups and exact analysis runs\ - \ as the signed-in user." + description: "Profiles, pivots, distinct values, duplicate groups and exact analysis\ + \ runs as the signed-in user." title: "QQQ Middleware API extension: data-analysis" - version: "1.0.0" + version: "1.1.0" openapi: "3.0.3" paths: /qqq/v1/qbit/data-analysis/tables/{table}/capabilities: @@ -1050,6 +1250,35 @@ paths: - sessionUuidCookie: - "N/A" summary: "List distinct column values" + /qqq/v1/qbit/data-analysis/tables/{table}/pivot: + post: + description: "Group records by up to 3 row and 2 column fields (with date or\ + \ numeric buckets) and compute counts, sums, averages, min/max, distinct counts\ + \ or percentiles per cell; every cell has a drill-through filter." + parameters: + - description: "QQQ table name to analyze." + in: "path" + name: "table" + required: true + schema: + type: "string" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisPivotRequestV1" + required: true + responses: + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisPivotV1" + description: "Successful Response" + security: + - sessionUuidCookie: + - "N/A" + summary: "Explore a pivot table" /qqq/v1/qbit/data-analysis/tables/{table}/table-profile: post: description: "Profile readable fields under a filter for completeness, uniqueness,\ diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/CoreAnalysisBackendTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/CoreAnalysisBackendTest.java index ed2a248..d07a423 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/CoreAnalysisBackendTest.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/CoreAnalysisBackendTest.java @@ -202,4 +202,28 @@ class CoreAnalysisBackendTest extends EngineFixture }); } } + + /******************************************************************************* + ** Core fallback limits never produce a falsely exact pivot drill. + *******************************************************************************/ + @Test + void partialPivotDisablesDrills() throws Exception + { + capabilities(Set.of()); + config.withDefaultMaxRows(3); + service.getConfigResolver().invalidate(); + var request = new com.kingsrook.qbits.dataanalysis.result.PivotRequest().withTableName("orders") + .withRows(java.util.List.of(new com.kingsrook.qbits.dataanalysis.result.PivotDimension().withField("created") + .withDateBucket(com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.DateBucketUnit.DAY))) + .withValues(java.util.List.of(new com.kingsrook.qbits.dataanalysis.result.PivotMeasure().withField("amount") + .withOperator(com.kingsrook.qbits.dataanalysis.result.PivotMeasure.Operator.PERCENTILE).withPercentile(new java.math.BigDecimal("0.5")))); + var result = service.pivot(request); + assertThat(result.getEstimate()).isTrue(); + assertThat(result.getWarnings()).anyMatch(w -> w.startsWith("Partial pivot")); + assertThat(result.getCells()).isNotEmpty().allSatisfy(c -> assertThat(c.getValues()).allSatisfy(m -> + { + assertThat(m.getDrill()).isNull(); + assertThat(m.getEstimate()).isTrue(); + })); + } } diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisApiTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisApiTest.java index 1c6d144..5f12f49 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisApiTest.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisApiTest.java @@ -169,7 +169,7 @@ class DataAnalysisApiTest extends EngineFixture assertThat(denied.getStatus()).isEqualTo(403); assertThat(run.path(operation.equals("COLUMN_PROFILE") ? "columnProfile" : "tableProfile").isObject()).isTrue(); } - assertThat(invoker.listOperations().stream().filter(o -> o.pathTemplate().startsWith(DataAnalysisApi.PREFIX)).toList()).hasSize(7); + assertThat(invoker.listOperations().stream().filter(o -> o.pathTemplate().startsWith(DataAnalysisApi.PREFIX)).toList()).hasSize(8); var client = java.net.http.HttpClient.newHttpClient(); var response = client.send(java.net.http.HttpRequest.newBuilder(java.net.URI.create("http://127.0.0.1:" + server.getBoundPort() + "/qqq/v1/openapi.json")).build(), java.net.http.HttpResponse.BodyHandlers.ofString()); assertThat(response.statusCode()).isEqualTo(200); @@ -255,7 +255,7 @@ class DataAnalysisApiTest extends EngineFixture call(HttpMethod.POST, "/runs", Map.of("operation", "PIVOT"), 400); call(HttpMethod.POST, "/runs", com.fasterxml.jackson.databind.node.NullNode.getInstance(), 400); call(HttpMethod.POST, "/tables/{table}/column-profile", Map.of("fieldName", "amount", "filter", - Map.of("criteria", List.of(Map.of("fieldName", "secret", "operator", "EQUALS", "values", List.of("hidden"))))), 400); + Map.of("criteria", List.of(Map.of("fieldName", "secret", "operator", "EQUALS", "values", List.of("hidden"))))), 403); var absent = invoker.invoke(new UiApiRequest().withMethod(HttpMethod.GET).withPath(DataAnalysisApi.PREFIX + "/tables/{table}/capabilities") .withPathParam("table", "missing").withSessionUUID(caller).withUserTimezone("America/Chicago")); assertThat(absent.getStatus()).isEqualTo(404); @@ -269,7 +269,7 @@ class DataAnalysisApiTest extends EngineFixture } /******************************************************************************* - ** Generated fragment is committed verbatim and has only the seven operations. + ** Generated fragment is committed verbatim and has only the eight operations. *******************************************************************************/ @Test void contract() throws Exception @@ -286,5 +286,38 @@ class DataAnalysisApiTest extends EngineFixture assertThat(Files.exists(path)).as("Regenerate with ./mvnw -o test -Dtest=DataAnalysisApiTest#contract -Dopenapi.write=true -Dqqq.snapshot.version=4.1.0-DATAANALYSIS-SNAPSHOT").isTrue(); assertThat(Files.readString(path).replace("\r\n", "\n")).as("OpenAPI changed: regenerate with -Dopenapi.write=true").isEqualTo(yaml.replace("\r\n", "\n")); } + @Test + void pivotHttp() throws Exception + { + start(); + var body = Map.of("rows", List.of(Map.of("field", "status")), "columns", List.of(Map.of("field", "customer")), + "values", List.of(Map.of("operator", "COUNT"))); + JsonNode result = call(HttpMethod.POST, "/tables/{table}/pivot", body, 200); + assertThat(result.path("cells").size()).isEqualTo(4); + checkDrills(result); + checkDrills(call(HttpMethod.POST, "/tables/{table}/pivot", Map.of("rows", List.of(Map.of("field", "paid")), + "values", List.of(Map.of("operator", "COUNT")), "limitGroups", 1), 200)); + for(String field : List.of("note", "amount", "quantity")) + { + checkDrills(call(HttpMethod.POST, "/tables/{table}/pivot", Map.of("rows", List.of(Map.of("field", field)), + "values", List.of(Map.of("operator", "COUNT")), "limitGroups", 1), 200)); + } + for(String unit : List.of("DAY", "HOUR", "DAY_OF_WEEK", "HOUR_OF_DAY")) + { + checkDrills(call(HttpMethod.POST, "/tables/{table}/pivot", Map.of("rows", List.of(Map.of("field", "created", "dateBucket", unit)), + "values", List.of(Map.of("operator", "COUNT"))), 200)); + } + checkDrills(call(HttpMethod.POST, "/tables/{table}/pivot", Map.of("rows", List.of(Map.of("field", "amount", "numericBins", Map.of("count", 10))), + "values", List.of(Map.of("operator", "COUNT"))), 200)); + for(String field : List.of("secret", "missing")) + { + call(HttpMethod.POST, "/tables/{table}/pivot", Map.of("rows", List.of(Map.of("field", field)), "values", List.of(Map.of("operator", "COUNT"))), 400); + call(HttpMethod.POST, "/tables/{table}/pivot", Map.of("rows", List.of(Map.of("field", "status")), "values", List.of(Map.of("operator", "MIN", "field", field))), 400); + } + call(HttpMethod.POST, "/tables/{table}/pivot", Map.of(), 400); + call(HttpMethod.POST, "/tables/{table}/pivot", Map.of("rows", "status", "values", List.of(Map.of("operator", "COUNT"))), 400); + call(HttpMethod.POST, "/tables/{table}/pivot", Map.of("rows", List.of(Map.of("field", "status")), "values", List.of(Map.of("operator", "COUNT")), "limitGroups", 1.5), 400); + caller = "denied"; + call(HttpMethod.POST, "/tables/{table}/pivot", body, 403); + } } - diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java index 3275715..e16bbe4 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java @@ -61,7 +61,7 @@ import static org.assertj.core.api.Assertions.within; /******************************************************************************* ** Shared backend acceptance tests, including a generic drill equivalence walker. *******************************************************************************/ -abstract class EngineContractTest extends EngineFixture +abstract class EngineContractTest extends PivotContractTest { /******************************************************************************* ** Every emitted key across all profile kinds has a label, on every backend. @@ -249,7 +249,7 @@ abstract class EngineContractTest extends EngineFixture void permissionChecks() throws Exception { assertThat(service.capabilities("orders").getFields()).doesNotContain("secret"); - assertThatThrownBy(() -> service.columnProfile(column("secret"))).isInstanceOf(QPermissionDeniedException.class); + assertThatThrownBy(() -> service.columnProfile(column("secret"))).isInstanceOf(com.kingsrook.qqq.backend.core.exceptions.QBadRequestException.class); QContext.init(instance, TestSupport.user("alice").withPermission("orders.analyze")); assertThatThrownBy(() -> service.columnProfile(column("amount"))).isInstanceOf(QPermissionDeniedException.class); QContext.init(instance, TestSupport.user("alice").withPermission("orders.read")); @@ -258,7 +258,7 @@ abstract class EngineContractTest extends EngineFixture config.withTableDefaults(Map.of("orders", new TableAnalysisDefaults().withExcludedFields(Set.of("amount")))); service.getConfigResolver().invalidate(); assertThat(service.capabilities("orders").getFields()).doesNotContain("amount"); - assertThatThrownBy(() -> service.columnProfile(column("amount"))).isInstanceOf(QPermissionDeniedException.class); + assertThatThrownBy(() -> service.columnProfile(column("amount"))).isInstanceOf(com.kingsrook.qqq.backend.core.exceptions.QBadRequestException.class); assertThatThrownBy(() -> service.capabilities("missing")).isInstanceOf(QPermissionDeniedException.class); } diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/FieldAnalysisAccessTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/FieldAnalysisAccessTest.java new file mode 100644 index 0000000..fb25ada --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/FieldAnalysisAccessTest.java @@ -0,0 +1,244 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis; + +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import com.kingsrook.qbits.dataanalysis.engine.CoreTierFieldAccessResolver; +import com.kingsrook.qbits.dataanalysis.engine.DrillFilters; +import com.kingsrook.qbits.dataanalysis.engine.FieldAnalysisAccess; +import com.kingsrook.qbits.dataanalysis.engine.FieldAnalysisAccessResolver; +import com.kingsrook.qbits.dataanalysis.engine.FieldSecurityLockFieldAccessResolver; +import com.kingsrook.qbits.dataanalysis.result.DistinctValuesRequest; +import com.kingsrook.qbits.dataanalysis.result.ValueCount; +import com.kingsrook.qbits.dataanalysis.result.ColumnProfile; +import com.kingsrook.qbits.dataanalysis.result.ColumnProfileRequest; +import com.kingsrook.qbits.dataanalysis.result.Metric; +import com.kingsrook.qbits.dataanalysis.result.PivotCell; +import com.kingsrook.qbits.dataanalysis.result.PivotDimension; +import com.kingsrook.qbits.dataanalysis.result.PivotMeasure; +import com.kingsrook.qbits.dataanalysis.result.PivotRequest; +import com.kingsrook.qbits.dataanalysis.result.PivotResult; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.within; + +/******************************************************************************* + ** Exercise reflection without shadowing a future real core package on test classpaths. + *******************************************************************************/ +class FieldAnalysisAccessTest extends EngineFixture +{ + /******************************************************************************* + ** Test helper for field-access behavior. + *******************************************************************************/ + public static class TierApi + { + static String tier = "FULL"; + static boolean fail; + /******************************************************************************* + ** Verify or support the pivot and field-access contract. + *******************************************************************************/ + public static String getTier(QTableMetaData table, String field) + { + if(fail) + { + throw new IllegalStateException("unavailable"); + } + return (tier); + } + /******************************************************************************* + ** Verify or support the pivot and field-access contract. + *******************************************************************************/ + public static String getAccessFingerprint() + { + if(fail) + { + throw new IllegalStateException("unavailable"); + } + return ("policy-v1"); + } + } + + /******************************************************************************* + ** Test helper for field-access behavior. + *******************************************************************************/ + public static class TierOnlyApi + { + /******************************************************************************* + ** Verify or support the pivot and field-access contract. + *******************************************************************************/ + public static String getTier(QTableMetaData table, String field) + { + return ("FULL"); + } + } + + /******************************************************************************* + ** Verify or support the pivot and field-access contract. + *******************************************************************************/ + @Test + void legacyFallbackAndReflectiveTierMapping() throws Exception + { + var table = instance.getTable("orders"); + var fallback = new FieldSecurityLockFieldAccessResolver(); + assertThat(fallback.resolve(table, table.getField("amount"))).isEqualTo(FieldAnalysisAccess.FULL); + assertThat(fallback.resolve(table, table.getField("secret"))).isEqualTo(FieldAnalysisAccess.NONE); + // The snapshot has no tier package. Test the actual default selection. + assertThat(CoreTierFieldAccessResolver.defaultResolver()).isInstanceOf(FieldSecurityLockFieldAccessResolver.class); + var resolver = new CoreTierFieldAccessResolver(TierApi.class); + try + { + for(String tier : List.of("FULL", "REVEALABLE", "MASKED", "HIDDEN", "UNKNOWN")) + { + TierApi.tier = tier; + assertThat(resolver.resolve(table, table.getField("amount"))).isEqualTo(tier.equals("FULL") ? FieldAnalysisAccess.FULL : tier.equals("REVEALABLE") ? FieldAnalysisAccess.EXACT_MATCH_ONLY : FieldAnalysisAccess.NONE); + } + assertThat(resolver.getAccessFingerprint()).isEqualTo("policy-v1"); + TierApi.fail = true; + assertThat(resolver.resolve(table, table.getField("amount"))).isEqualTo(FieldAnalysisAccess.NONE); + assertThatThrownBy(resolver::getAccessFingerprint).isInstanceOf(IllegalStateException.class); + assertThat(new CoreTierFieldAccessResolver(TierOnlyApi.class).getAccessFingerprint()).isEmpty(); + } + finally + { + TierApi.fail = false; + TierApi.tier = "FULL"; + } + } + + /******************************************************************************* + ** Verify or support the pivot and field-access contract. + *******************************************************************************/ + @Test + void customPolicyAndUserFilterPassThrough() throws Exception + { + config.withFieldAnalysisAccessResolver(() -> (table, field) -> field.getName().equals("note") ? FieldAnalysisAccess.NONE + : field.getName().equals("paid") ? FieldAnalysisAccess.EXACT_MATCH_ONLY : new FieldSecurityLockFieldAccessResolver().resolve(table, field)); + service = new DataAnalysisService(config, Runnable::run); + assertThat(service.capabilities("orders").getFields()).doesNotContain("note", "paid", "secret"); + for(String field : List.of("note", "paid")) + { + assertThatThrownBy(() -> service.columnProfile(new ColumnProfileRequest().withTableName("orders").withFieldName(field))) + .isInstanceOf(com.kingsrook.qqq.backend.core.exceptions.QBadRequestException.class); + assertThatThrownBy(() -> service.pivot(new PivotRequest().withTableName("orders").withRows(List.of(new PivotDimension().withField(field))) + .withValues(List.of(new PivotMeasure().withOperator(PivotMeasure.Operator.COUNT))))) + .isInstanceOf(com.kingsrook.qqq.backend.core.exceptions.QBadRequestException.class); + assertThatThrownBy(() -> service.pivot(new PivotRequest().withTableName("orders").withRows(List.of(new PivotDimension().withField("status"))) + .withValues(List.of(new PivotMeasure().withOperator(PivotMeasure.Operator.MIN).withField(field))))) + .isInstanceOf(com.kingsrook.qqq.backend.core.exceptions.QBadRequestException.class); + } + var request = new PivotRequest().withTableName("orders").withRows(List.of(new PivotDimension().withField("status"))) + .withValues(List.of(new PivotMeasure().withOperator(PivotMeasure.Operator.COUNT))); + assertThat(JsonUtils.toJson(service.pivot(request))).doesNotContain("\"fieldName\":\"note\"", "\"fieldName\":\"paid\""); + var ownFilter = DrillFilters.condition("paid", QCriteriaOperator.EQUALS, true); + String before = JsonUtils.toJson(ownFilter); + PivotResult result = service.pivot(request.withFilter(ownFilter)); + assertThat(JsonUtils.toJson(ownFilter)).isEqualTo(before); + assertThat(((Number)result.getGrandTotals().getFirst().getValue()).intValue()).isEqualTo(6); + for(PivotCell cell : result.getCells()) + { + var drill = cell.getValues().getFirst().getDrill().getFilter(); + assertThat(JsonUtils.toJson(drill.getSubFilters().getFirst())).isEqualTo(before); + assertThat(JsonUtils.toJson(drill.getSubFilters().get(1))).doesNotContain("paid", "note"); + } + } + + /******************************************************************************* + ** Verify or support the pivot and field-access contract. + *******************************************************************************/ + @Test + void cacheMissesOnFieldTierAndCoreFingerprintChanges() throws Exception + { + AtomicReference tier = new AtomicReference<>(FieldAnalysisAccess.FULL); + AtomicReference fingerprint = new AtomicReference<>("first"); + config.withFieldAnalysisAccessResolver(() -> new FieldAnalysisAccessResolver() + { + /******************************************************************************* + ** Verify or support the pivot and field-access contract. + *******************************************************************************/ + @Override + public FieldAnalysisAccess resolve(QTableMetaData table, QFieldMetaData field) + { + return (field.getName().equals("quantity") ? tier.get() : new FieldSecurityLockFieldAccessResolver().resolve(table, field)); + } + /******************************************************************************* + ** Verify or support the pivot and field-access contract. + *******************************************************************************/ + @Override + public String getAccessFingerprint() + { + return (fingerprint.get()); + } + }); + service = new DataAnalysisService(config, Runnable::run); + ColumnProfileRequest request = new ColumnProfileRequest().withTableName("orders").withFieldName("amount").withExact(true); + String first = service.startExactRun(request); + assertThat(service.startExactRun(request)).isEqualTo(first); + tier.set(FieldAnalysisAccess.EXACT_MATCH_ONLY); + String second = service.startExactRun(request); + assertThat(second).isNotEqualTo(first); + assertThatThrownBy(() -> service.getRun(first)).isInstanceOf(com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException.class); + fingerprint.set("second"); + assertThat(service.startExactRun(request)).isNotEqualTo(second); + } + /******************************************************************************* + ** Verify or support the pivot and field-access contract. + *******************************************************************************/ + @Test + void generatedRepresentativeDrillsCannotUseRestrictedPrimaryKey() throws Exception + { + config.withFieldAnalysisAccessResolver(() -> (table, field) -> field.getName().equals("id") ? FieldAnalysisAccess.NONE + : new FieldSecurityLockFieldAccessResolver().resolve(table, field)); + service = new DataAnalysisService(config, Runnable::run); + ColumnProfile profile = service.columnProfile(new ColumnProfileRequest().withTableName("orders").withFieldName("status").withExact(true)); + Metric distinct = profile.getCounts().stream().filter(m -> m.getKey().equals("distinct")).findFirst().orElseThrow(); + assertThat(distinct.getDrill()).isNull(); + assertThat(profile.getCounts().getFirst().getDrill()).isNotNull(); + var ownFilter = DrillFilters.condition("id", QCriteriaOperator.EQUALS, 1); + profile = service.columnProfile(new ColumnProfileRequest().withTableName("orders").withFieldName("status").withFilter(ownFilter).withExact(true)); + assertThat(profile.getCounts().getFirst().getDrill()).isNotNull(); + assertThat(JsonUtils.toJson(profile.getCounts().getFirst().getDrill())).contains("id"); + } + /******************************************************************************* + ** Related label tiers must neither leak labels nor reuse a stale cache entry. + *******************************************************************************/ + @Test + void relatedLabelTierChangesInvalidateCache() throws Exception + { + AtomicReference tier = new AtomicReference<>(FieldAnalysisAccess.FULL); + config.withFieldAnalysisAccessResolver(() -> (table, field) -> table.getName().equals("customer") && field.getName().equals("name") + ? tier.get() : new FieldSecurityLockFieldAccessResolver().resolve(table, field)); + service = new DataAnalysisService(config, Runnable::run); + ColumnProfileRequest request = new ColumnProfileRequest().withTableName("orders").withFieldName("customer").withExact(true); + String first = service.startExactRun(request); + assertThat(service.columnProfile(request).getTopValues()).extracting(ValueCount::getLabel).contains("Alice Co"); + tier.set(FieldAnalysisAccess.NONE); + assertThat(service.startExactRun(request)).isNotEqualTo(first); + assertThat(service.columnProfile(request).getTopValues()).extracting(ValueCount::getLabel).containsExactly("1", "2"); + assertThat(service.distinctValues(new DistinctValuesRequest().withTableName("orders").withFieldName("customer")).getValues()).extracting(ValueCount::getLabel).containsExactly("1", "2"); + } +} diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/PivotContractTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/PivotContractTest.java new file mode 100644 index 0000000..0443a6e --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/PivotContractTest.java @@ -0,0 +1,350 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis; + +import java.math.BigDecimal; +import java.util.List; +import java.util.ArrayList; +import com.kingsrook.qbits.dataanalysis.result.Drill; +import com.kingsrook.qbits.dataanalysis.result.Metric; +import com.kingsrook.qbits.dataanalysis.result.PivotCell; +import com.kingsrook.qbits.dataanalysis.result.PivotDimension; +import com.kingsrook.qbits.dataanalysis.result.PivotHeader; +import com.kingsrook.qbits.dataanalysis.result.PivotMeasure; +import com.kingsrook.qbits.dataanalysis.result.PivotNumericBins; +import com.kingsrook.qbits.dataanalysis.result.PivotRequest; +import com.kingsrook.qbits.dataanalysis.result.PivotResult; +import com.kingsrook.qbits.dataanalysis.result.ValueType; +import com.kingsrook.qbits.dataanalysis.engine.DrillFilters; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; +import com.kingsrook.qqq.backend.core.actions.tables.CountAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.DateBucketUnit; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.within; + +/******************************************************************************* + ** Pivot contract inherited by every existing backend test configuration. + *******************************************************************************/ +abstract class PivotContractTest extends EngineFixture +{ + /******************************************************************************* + ** Verify or support the pivot and field-access contract. + *******************************************************************************/ + protected PivotDimension dimension(String field) + { + return (new PivotDimension().withField(field)); + } + + /******************************************************************************* + ** Verify or support the pivot and field-access contract. + *******************************************************************************/ + protected PivotMeasure measure(PivotMeasure.Operator operator, String field) + { + return (new PivotMeasure().withOperator(operator).withField(field)); + } + + /******************************************************************************* + ** Verify or support the pivot and field-access contract. + *******************************************************************************/ + protected PivotRequest pivot(List rows, List columns) + { + return (new PivotRequest().withTableName("orders").withRows(rows.stream().map(this::dimension).toList()) + .withColumns(columns.stream().map(this::dimension).toList()).withValues(List.of(measure(PivotMeasure.Operator.COUNT, null)))); + } + + /******************************************************************************* + ** Verify or support the pivot and field-access contract. + *******************************************************************************/ + private void equalNumber(Metric metric, String value) + { + assertThat(new BigDecimal(metric.getValue().toString())).isEqualByComparingTo(value); + } + + /******************************************************************************* + ** Verify or support the pivot and field-access contract. + *******************************************************************************/ + protected void countDrills(PivotResult result) throws Exception + { + List> groups = new ArrayList<>(result.getRowTotals()); + groups.addAll(result.getColumnTotals()); + groups.add(result.getGrandTotals()); + result.getCells().forEach(c -> groups.add(c.getValues())); + for(List metrics : groups) + { + for(Metric metric : metrics) + { + if(metric.getKey().equals("value0") && !metric.getSuppressed()) + { + assertThat(metric.getDrill()).isNotNull(); + Drill drill = JsonUtils.toObject(JsonUtils.toJson(metric.getDrill()), Drill.class); + CountInput input = new CountInput(drill.getTableName()).withFilter(drill.getFilter()); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + assertThat(new CountAction().execute(input).getCount()).as(JsonUtils.toJson(drill)).isEqualTo(((Number) metric.getValue()).intValue()); + QueryInput query = new QueryInput(drill.getTableName()).withFilter(drill.getFilter()); + query.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + assertThat(new QueryAction().execute(query).getRecords()).as(JsonUtils.toJson(drill)).hasSize(((Number) metric.getValue()).intValue()); + } + } + } + } + + /******************************************************************************* + ** Verify or support the pivot and field-access contract. + *******************************************************************************/ + @Test + void pivotShapesAndTotals() throws Exception + { + for(PivotRequest request : List.of(pivot(List.of("status"), List.of("customer")), pivot(List.of("status", "paid"), List.of("customer")), + pivot(List.of("status"), List.of("customer", "paid")))) + { + PivotResult result = service.pivot(request); + equalNumber(result.getGrandTotals().getFirst(), "12"); + assertThat(result.getCells().stream().mapToLong(c -> ((Number)c.getValues().getFirst().getValue()).longValue()).sum()).isEqualTo(12); + countDrills(result); + } + PivotResult result = service.pivot(pivot(List.of("status"), List.of("customer"))); + assertThat(result.getCells()).hasSize(4); + assertThat(result.getCells()).allSatisfy(c -> assertThat(((Number)c.getValues().getFirst().getValue()).longValue()).isIn(2L, 4L)); + assertThat(result.getRowTotals()).allSatisfy(m -> equalNumber(m.getFirst(), "6")); + result = service.pivot(pivot(List.of("status"), List.of()).withShowTotals(false)); + assertThat(result.getGrandTotals()).isEmpty(); + assertThat(result.getRowTotals()).isEmpty(); + assertThat(result.getColumnHeaders()).containsExactly(List.of()); + } + + /******************************************************************************* + ** Verify or support the pivot and field-access contract. + *******************************************************************************/ + @Test + void pivotOtherAndNonAdditiveMeasures() throws Exception + { + PivotRequest request = pivot(List.of("quantity"), List.of("status")).withLimitGroups(1).withValues(List.of( + measure(PivotMeasure.Operator.COUNT, null), measure(PivotMeasure.Operator.SUM, "amount"), measure(PivotMeasure.Operator.AVG, "amount"), + measure(PivotMeasure.Operator.COUNT_DISTINCT, "amount"), measure(PivotMeasure.Operator.PERCENTILE, "amount").withPercentile(new BigDecimal("0.5")))); + PivotResult result = service.pivot(request); + assertThat(result.getTruncatedRows()).isTrue(); + assertThat(result.getTruncatedColumns()).isTrue(); + assertThat(result.getRowHeaders().stream().flatMap(List::stream).map(PivotHeader::getLabel)).anyMatch(l -> l.equals("Other (5 groups)")); + equalNumber(result.getGrandTotals().get(1), "1170"); + equalNumber(result.getGrandTotals().get(3), "8"); + equalNumber(result.getGrandTotals().get(4), "20"); + countDrills(result); + for(PivotCell cell : result.getCells()) + { + var input = new com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput("orders").withFilter(cell.getValues().getFirst().getDrill().getFilter()); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + var numbers = new com.kingsrook.qqq.backend.core.actions.tables.QueryAction().execute(input).getRecords().stream().map(r -> r.getValueBigDecimal("amount")) + .filter(java.util.Objects::nonNull).sorted().toList(); + BigDecimal sum = numbers.stream().reduce(BigDecimal.ZERO, BigDecimal::add); + equalNumber(cell.getValues().get(1), sum.toString()); + assertThat(new BigDecimal(cell.getValues().get(2).getValue().toString()).doubleValue()).isCloseTo(sum.doubleValue() / numbers.size(), within(0.000000001)); + equalNumber(cell.getValues().get(3), Long.toString(numbers.stream().map(BigDecimal::stripTrailingZeros).distinct().count())); + BigDecimal median = numbers.size() % 2 == 1 ? numbers.get(numbers.size() / 2) : numbers.get(numbers.size() / 2 - 1).add(numbers.get(numbers.size() / 2)).divide(BigDecimal.valueOf(2)); + equalNumber(cell.getValues().get(4), median.toString()); + } + for(String field : List.of("paid", "note", "amount")) + { + countDrills(service.pivot(pivot(List.of(field), List.of()).withLimitGroups(1))); + } + } + + /******************************************************************************* + ** Verify or support the pivot and field-access contract. + *******************************************************************************/ + @Test + void pivotBuckets() throws Exception + { + for(DateBucketUnit unit : DateBucketUnit.values()) + { + PivotRequest request = pivot(List.of("created"), List.of("status")).withRows(List.of(dimension("created").withDateBucket(unit))); + PivotResult result = service.pivot(request); + equalNumber(result.getGrandTotals().getFirst(), "12"); + countDrills(result); + } + PivotResult dates = service.pivot(pivot(List.of("order_date"), List.of("status")).withRows(List.of(dimension("order_date").withDateBucket(DateBucketUnit.DAY)))); + assertThat(dates.getCells()).hasSize(4); + countDrills(dates); + for(PivotNumericBins bins : List.of(new PivotNumericBins().withCount(4).withMin(BigDecimal.ZERO).withMax(BigDecimal.valueOf(40)), new PivotNumericBins().withCount(10))) + { + countDrills(service.pivot(pivot(List.of("amount"), List.of("status")).withRows(List.of(dimension("amount").withNumericBins(bins))))); + } + } + + /******************************************************************************* + ** Verify or support the pivot and field-access contract. + *******************************************************************************/ + @Test + void pivotPrivacyAndUserFilter() throws Exception + { + config.withDefaultMinGroupSize(3); + service.getConfigResolver().invalidate(); + PivotResult result = service.pivot(pivot(List.of("status"), List.of("customer")).withValues(List.of(measure(PivotMeasure.Operator.COUNT, null), measure(PivotMeasure.Operator.SUM, "amount")))); + assertThat(result.getCells().stream().filter(c -> c.getValues().getFirst().getSuppressed())).hasSize(2); + result.getCells().stream().filter(c -> c.getValues().getFirst().getSuppressed()).forEach(c -> assertThat(c.getValues()).allSatisfy(m -> + { + assertThat(m.getValue()).isNull(); + assertThat(m.getDrill()).isNull(); + })); + assertThat(service.pivot(pivot(List.of("quantity"), List.of())).getRowHeaders()).isEmpty(); + config.withDefaultMinGroupSize(0); + service.getConfigResolver().invalidate(); + var filter = DrillFilters.condition("customer", QCriteriaOperator.EQUALS, 1); + String before = JsonUtils.toJson(filter); + result = service.pivot(pivot(List.of("status"), List.of()).withFilter(filter)); + equalNumber(result.getGrandTotals().getFirst(), "8"); + assertThat(JsonUtils.toJson(filter)).isEqualTo(before); + countDrills(result); + } + + /******************************************************************************* + ** Verify or support the pivot and field-access contract. + *******************************************************************************/ + @Test + void pivotJoinedDimensionAndPermissions() throws Exception + { + instance.addJoin(new QJoinMetaData().withName("ordersCustomer").withLeftTable("orders").withRightTable("customer") + .withType(JoinType.MANY_TO_ONE).withJoinOn(new JoinOn("customer", "id"))); + instance.getTable("orders").withExposedJoin(new ExposedJoin().withJoinTable("customer").withJoinPath(List.of("ordersCustomer"))); + PivotResult result = service.pivot(pivot(List.of("customer.name"), List.of("status"))); + assertThat(result.getRowHeaders().stream().flatMap(List::stream).map(PivotHeader::getValue)).containsExactlyInAnyOrder("Alice Co", "Bob Co"); + equalNumber(result.getGrandTotals().getFirst(), "12"); + assertThat(result.getWarnings()).isEmpty(); + assertThat(result.getCells()).allSatisfy(c -> assertThat(JsonUtils.toJson(c.getValues().getFirst().getDrill())).contains("customer.name")); + countDrills(result); + countDrills(service.pivot(pivot(List.of("customer.name"), List.of("status")).withLimitGroups(1) + .withFilter(DrillFilters.condition("paid", QCriteriaOperator.EQUALS, true)))); + instance.getJoins().put("ordersCustomer", instance.getJoin("ordersCustomer").flip()); + countDrills(service.pivot(pivot(List.of("customer.name"), List.of("status")))); + QContext.getQSession().getPermissions().remove("customer.read"); + assertThatThrownBy(() -> service.pivot(pivot(List.of("customer.name"), List.of()))).isInstanceOf(com.kingsrook.qqq.backend.core.exceptions.QBadRequestException.class); + } + + /******************************************************************************* + ** Reversing a many-to-one join multiplies records and disables every drill. + *******************************************************************************/ + @Test + void pivotToManyDimension() throws Exception + { + instance.addJoin(new QJoinMetaData().withName("ordersCustomer").withLeftTable("orders").withRightTable("customer") + .withType(JoinType.MANY_TO_ONE).withJoinOn(new JoinOn("customer", "id"))); + instance.getTable("customer").withExposedJoin(new ExposedJoin().withJoinTable("orders").withJoinPath(List.of("ordersCustomer"))); + QContext.getQSession().getPermissions().add("customer.analyze"); + for(int direction = 0; direction < 2; direction++) + { + PivotResult result = service.pivot(pivot(List.of("orders.status"), List.of()).withTableName("customer")); + equalNumber(result.getGrandTotals().getFirst(), "12"); + assertThat(result.getWarnings()).containsExactly("Counts include one row per related " + instance.getTable("orders").getLabel() + + "; drill-through is not available for this dimension."); + assertThat(result.getCells()).allSatisfy(c -> assertThat(c.getValues()).allSatisfy(m -> assertThat(m.getDrill()).isNull())); + assertThat(result.getRowTotals()).allSatisfy(metrics -> assertThat(metrics).allSatisfy(m -> assertThat(m.getDrill()).isNull())); + assertThat(result.getColumnTotals()).allSatisfy(metrics -> assertThat(metrics).allSatisfy(m -> assertThat(m.getDrill()).isNull())); + assertThat(result.getGrandTotals()).allSatisfy(m -> assertThat(m.getDrill()).isNull()); + instance.getJoins().put("ordersCustomer", instance.getJoin("ordersCustomer").flip()); + } + } + + /******************************************************************************* + ** Every hop must be to-one, including hops preceding a one-to-one target. + *******************************************************************************/ + @Test + void pivotMultiHopDimension() throws Exception + { + instance.addTable(new com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData().withName("region").withLabel("Region") + .withBackendName(instance.getTable("customer").getBackendName()).withPrimaryKeyField("id") + .withField(new com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData("id", com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType.INTEGER)) + .withField(new com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData("name", com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType.STRING))); + if(connection != null) + { + try(var statement = connection.createStatement()) + { + statement.execute("DROP TABLE IF EXISTS region"); + statement.execute("CREATE TABLE region (id INTEGER PRIMARY KEY, name VARCHAR(100))"); + } + } + new com.kingsrook.qqq.backend.core.instances.QInstanceEnricher(instance).enrich(); + TestSupport.insert("region", new com.kingsrook.qqq.backend.core.model.data.QRecord().withValue("id", 1).withValue("name", "East")); + TestSupport.insert("region", new com.kingsrook.qqq.backend.core.model.data.QRecord().withValue("id", 2).withValue("name", "West")); + instance.addJoin(new QJoinMetaData().withName("ordersCustomer").withLeftTable("orders").withRightTable("customer") + .withType(JoinType.MANY_TO_ONE).withJoinOn(new JoinOn("customer", "id"))); + instance.addJoin(new QJoinMetaData().withName("customerRegion").withLeftTable("region").withRightTable("customer") + .withType(JoinType.ONE_TO_ONE).withJoinOn(new JoinOn("id", "id"))); + instance.getTable("orders").withExposedJoin(new ExposedJoin().withJoinTable("region").withJoinPath(List.of("ordersCustomer", "customerRegion"))); + PivotResult result = service.pivot(pivot(List.of("region.name"), List.of("status"))); + assertThat(result.getWarnings()).containsExactly("Drill-through is not available for fields reached through more than one join (Region)."); + assertThat(result.getCells()).allSatisfy(c -> assertThat(c.getValues().getFirst().getDrill()).isNull()); + instance.getJoin("ordersCustomer").withType(JoinType.MANY_TO_MANY); + result = service.pivot(pivot(List.of("region.name"), List.of("status"))); + assertThat(result.getWarnings()).containsExactly("Counts include one row per related Region; drill-through is not available for this dimension."); + assertThat(result.getCells()).allSatisfy(c -> assertThat(c.getValues().getFirst().getDrill()).isNull()); + } + + /******************************************************************************* + ** Verify or support the pivot and field-access contract. + *******************************************************************************/ + @Test + void pivotValidation() throws Exception + { + for(String field : List.of("secret", "missing", "missing.field")) + { + assertThatThrownBy(() -> service.pivot(pivot(List.of(field), List.of()))).isInstanceOf(com.kingsrook.qqq.backend.core.exceptions.QBadRequestException.class); + assertThatThrownBy(() -> service.pivot(pivot(List.of("status"), List.of()).withValues(List.of(measure(PivotMeasure.Operator.MIN, field))))) + .isInstanceOf(com.kingsrook.qqq.backend.core.exceptions.QBadRequestException.class); + } + for(PivotRequest bad : List.of(pivot(List.of(), List.of()), pivot(List.of("status"), List.of()).withLimitGroups(201), + pivot(List.of("status"), List.of("status")), pivot(List.of("status"), List.of()).withRows(List.of(dimension("status").withDateBucket(DateBucketUnit.YEAR))), + pivot(List.of("status"), List.of()).withValues(List.of(measure(PivotMeasure.Operator.PERCENTILE, "amount").withPercentile(BigDecimal.TEN))))) + { + assertThatThrownBy(() -> service.pivot(bad)).isInstanceOf(com.kingsrook.qqq.backend.core.exceptions.QBadRequestException.class); + } + } + /******************************************************************************* + ** Verify or support the pivot and field-access contract. + *******************************************************************************/ + @Test + void pivotTypedExtremesAndEmptyScope() throws Exception + { + PivotResult result = service.pivot(pivot(List.of("status"), List.of()).withValues(List.of(measure(PivotMeasure.Operator.COUNT, "amount"), + measure(PivotMeasure.Operator.MIN, "order_date"), measure(PivotMeasure.Operator.MAX, "created"), measure(PivotMeasure.Operator.MIN, "amount"), + measure(PivotMeasure.Operator.MAX, "amount")))); + equalNumber(result.getGrandTotals().getFirst(), "12"); + equalNumber(result.getGrandTotals().get(3), "-10"); + equalNumber(result.getGrandTotals().get(4), "1000"); + assertThat(result.getGrandTotals().get(1).getValueType()).isEqualTo(ValueType.DATE); + assertThat(result.getGrandTotals().get(2).getValueType()).isEqualTo(ValueType.DATE_TIME); + countDrills(result); + result = service.pivot(pivot(List.of("status"), List.of()).withFilter(DrillFilters.condition("id", QCriteriaOperator.EQUALS, -1))); + assertThat(result.getCells()).isEmpty(); + equalNumber(result.getGrandTotals().getFirst(), "0"); + countDrills(result); + } +} -- 2.54.0 From 20b8223ce8eb79788944629d2acc21fcf6946dc7 Mon Sep 17 00:00:00 2001 From: James Maes Date: Tue, 13 Oct 2026 19:37:03 -0500 Subject: [PATCH 05/10] feat(compare): compare, distribution shift, correlation, scatter and crosstabs Extension 1.2.0: compare two filters field by field with deltas and significance (two-proportion z, Welch's t, chi-square with validity rules and plain sentences), distribution shift over time with PSI, a pushed-down Pearson correlation matrix with sampled scatter and trend, and crosstabs with chi-square and Cramer's V. Pure-Java statistics checked against scipy reference values; the privacy guard, field access and drills apply as for profiles. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 + README.md | 23 +- .../dataanalysis/DataAnalysisService.java | 58 ++ .../dataanalysis/api/DataAnalysisApi.java | 188 ++++- .../dataanalysis/api/DataAnalysisSchemas.java | 5 + .../dataanalysis/api/RelationshipSchemas.java | 663 ++++++++++++++++++ .../engine/ComparisonExplorer.java | 254 +++++++ .../dataanalysis/engine/MetricLabels.java | 2 + .../engine/RelationshipExplorer.java | 255 +++++++ .../engine/RelationshipSupport.java | 241 +++++++ .../dataanalysis/engine/ShiftExplorer.java | 213 ++++++ .../qbits/dataanalysis/result/Chart.java | 18 + .../dataanalysis/result/CompareRequest.java | 213 ++++++ .../dataanalysis/result/CompareResult.java | 55 ++ .../dataanalysis/result/ComparisonSide.java | 77 ++ .../dataanalysis/result/ComparisonValue.java | 186 +++++ .../dataanalysis/result/CorrelationCell.java | 142 ++++ .../result/CorrelationRequest.java | 125 ++++ .../result/CorrelationResult.java | 99 +++ .../dataanalysis/result/CrosstabCell.java | 142 ++++ .../dataanalysis/result/CrosstabRequest.java | 168 +++++ .../dataanalysis/result/CrosstabResult.java | 121 ++++ .../dataanalysis/result/FieldComparison.java | 121 ++++ .../dataanalysis/result/ScatterRequest.java | 168 +++++ .../dataanalysis/result/ScatterResult.java | 165 +++++ .../dataanalysis/result/ShiftBucket.java | 121 ++++ .../dataanalysis/result/ShiftRequest.java | 168 +++++ .../dataanalysis/result/ShiftResult.java | 121 ++++ .../dataanalysis/result/Significance.java | 186 +++++ .../qbits/dataanalysis/result/TrendLine.java | 98 +++ .../dataanalysis/stats/SpecialFunctions.java | 179 +++++ .../qbits/dataanalysis/stats/Statistics.java | 286 ++++++++ .../openapi/qqq-v1/data-analysis.yaml | 573 ++++++++++++++- .../dataanalysis/DataAnalysisApiTest.java | 99 ++- .../dataanalysis/EngineContractTest.java | 2 +- .../RelationshipContractTest.java | 370 ++++++++++ .../RelationshipFallbackTest.java | 133 ++++ .../qbits/dataanalysis/StatisticsTest.java | 148 ++++ 38 files changed, 6177 insertions(+), 14 deletions(-) create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/api/RelationshipSchemas.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/ComparisonExplorer.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/RelationshipExplorer.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/RelationshipSupport.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/ShiftExplorer.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/CompareRequest.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/CompareResult.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/ComparisonSide.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/ComparisonValue.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/CorrelationCell.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/CorrelationRequest.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/CorrelationResult.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/CrosstabCell.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/CrosstabRequest.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/CrosstabResult.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/FieldComparison.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/ScatterRequest.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/ScatterResult.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/ShiftBucket.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/ShiftRequest.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/ShiftResult.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/Significance.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/TrendLine.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/stats/SpecialFunctions.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/stats/Statistics.java create mode 100644 src/test/java/com/kingsrook/qbits/dataanalysis/RelationshipContractTest.java create mode 100644 src/test/java/com/kingsrook/qbits/dataanalysis/RelationshipFallbackTest.java create mode 100644 src/test/java/com/kingsrook/qbits/dataanalysis/StatisticsTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 66c54b8..f831652 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ Following [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and [Semantic ### Added +- Phase d compare, shift, correlation, scatter and crosstab service operations and HTTP specs; extension version 1.2.0 with typed `DataAnalysis*V1` schemas and regenerated OpenAPI. +- Pure-Java two-proportion, Welch, chi-square/Cramer V, PSI and Pearson statistics, including regularized incomplete beta/gamma and independent SciPy references. +- Core-pushed numeric moments and batched correlations, sample trend lines, calendar distribution shifts, category pooling, privacy suppression and exact count drills. +- Shared memory/H2/PostgreSQL relationship contracts, forced core streamed-fallback checks, batching checks and HTTP authorization/serialization/drill tests. + - Phase c pivot backend and `POST /qbit/data-analysis/tables/{table}/pivot`, extension version 1.1.0, typed schemas and regenerated OpenAPI fragment. - Row/column dimensions, calendar/numeric buckets, seven measure operators, sparse cells, exact Other recomputation, totals, privacy suppression and count-equivalent drills. - Exposed-join dimensions and measures with READ/FULL checks; filter-only drills for to-one paths and explicit null-drill warnings for to-many paths whose joined-row counts can exceed record counts. diff --git a/README.md b/README.md index 48c0ff4..fe664f0 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ Statistical analysis and data dives on any QQQ table, filter or saved view: prof ## What works now -The host API provides column and table profiles, distinct-value pages, duplicate groups, drill-through filters, effective per-table configuration, privacy suppression, and asynchronous exact runs with a persistent per-user cache. Metadata, six visible tables, owner locks, PostgreSQL DDL generation and the Analyses home are also available. +The host API provides column and table profiles, pivots, comparisons, distribution shifts, correlations, scatter samples, crosstabs, distinct-value pages, duplicate groups, drill-through filters, privacy suppression, and asynchronous exact profile runs with a persistent per-user cache. Metadata, six visible tables, owner locks, PostgreSQL DDL generation and the Analyses home are also available. -Seven HTTP operations are available under `/qqq/v1/qbit/data-analysis`; the Analyze UI is a later phase. Pivots, comparisons, anomalies, saved-analysis execution, scheduling and exports remain later phases. +Thirteen HTTP operations are available under `/qqq/v1/qbit/data-analysis`; the Analyze UI is a later phase. Pivot and relationship backends are available. Anomalies, saved-analysis execution, scheduling and exports remain later phases. ## Requirements @@ -138,7 +138,7 @@ Count drills combine an untouched copy of the request filter with their producin ## HTTP endpoints -Extension `data-analysis` version **1.1.0**, prefix `/qbit/data-analysis` inside `/qqq/v1`: +Extension `data-analysis` version **1.2.0**, prefix `/qbit/data-analysis` inside `/qqq/v1`: | Method | Path after `/qqq/v1` | Body or purpose | | --- | --- | --- | @@ -148,6 +148,11 @@ Extension `data-analysis` version **1.1.0**, prefix `/qbit/data-analysis` inside | POST | `/qbit/data-analysis/tables/{table}/table-profile` | `{filter, fields}` | | POST | `/qbit/data-analysis/tables/{table}/duplicates` | `{fields, filter, limit}`; default limit 50, maximum 500 | | POST | `/qbit/data-analysis/tables/{table}/pivot` | `{rows, columns, values, filter, showTotals, limitGroups}`; 1–3 row dimensions, 0–2 column dimensions, 1–6 measures | +| POST | `/qbit/data-analysis/tables/{table}/compare` | `{filterA, filterB, labelA, labelB, fields}`; null filter selects the whole permitted table | +| POST | `/qbit/data-analysis/tables/{table}/shift` | `{field, dateField, bucket, filter}`; calendar buckets, top 8 categories or quartiles, first/last complete-bucket PSI | +| POST | `/qbit/data-analysis/tables/{table}/correlation` | `{fields, filter}`; 2–12 distinct numeric fields, pairwise r and n, scatter request per cell | +| POST | `/qbit/data-analysis/tables/{table}/scatter` | `{xField, yField, filter, limit}`; default/maximum 2000 points and a sample trend line | +| POST | `/qbit/data-analysis/tables/{table}/crosstab` | `{rowField, columnField, filter, limit}`; default/maximum 20 categories per axis plus Other | | POST | `/qbit/data-analysis/runs` | `{operation: COLUMN_PROFILE or TABLE_PROFILE, tableName, request}` returns `{runId, status}` | | GET | `/qbit/data-analysis/runs/{runId}` | Status, timestamps, error, typed `columnProfile` or `tableProfile` | @@ -187,6 +192,16 @@ Variance, interpolated percentiles, string lengths/shapes, date classification, Data versions use filtered COUNT plus MAX of a readable modification-date-like field, otherwise MAX(primary key). This is the specified inexpensive invalidation heuristic, not snapshot isolation; updates on a table without a modification timestamp can remain cached until TTL expiry. Concurrent writes can also change drill counts after a profile is produced. Results always retain `asOf`. +## Compare and relationships + +All five operations resolve FULL fields before reading source data and use core actions with `PermissionEnforcement.ENFORCE`. Core applies table permissions and record locks. Compare batches numeric COUNT, AVG, VAR_SAMP and quartiles per population; categorical comparisons use a common top-10-plus-Other distribution and pool categories with expected counts below five. Deltas are B minus A, and percent deltas divide by absolute A (null when A is zero). Significance uses a strict p < 0.05 threshold; invalid or privacy-suppressed tests return a plain reason. + +Correlation batches all CORR pairs into one aggregate call. Pairwise non-null n values use enforced core count actions, since the current aggregate API has no per-measure filter. Unsupported aggregates use core's bounded streamed fallback; a truncated fallback returns 400 with a request to narrow the filter, rather than publishing a partial significance test. Scatter uses the first N paired records ordered by primary key because the current core QueryInput has no sampling API. Its least-squares line describes that displayed sample. Point drills contain only primary-key equality and are disabled whenever privacy is enabled or the primary key is not FULL. + +Shift uses session-timezone calendar buckets. Only buckets whose end has passed enter PSI; categories share one top-8-plus-Other basis, and numeric PSI uses the first complete bucket's quartile boundaries for both populations. Zero shares are floored at 0.000001. PSI is descriptive, not a significance test. Category time series explicitly declare `stacked: true`. + +The minimum-group-size guard removes small numeric summaries, suppresses crosstab cells and revealing marginal shares, disables drills on privacy-merged Other groups, and suppresses scatter points below the minimum. Statistical tests and PSI are unavailable when protected values would contribute. No runtime statistics dependency is added; beta/gamma, Welch, chi-square, proportions, PSI and Pearson are pure Java, checked against independently computed SciPy references. + ## Adopting in another app 1. Add a backend, install the producer and run the usual `QInstanceValidator`. @@ -195,7 +210,7 @@ Data versions use filtered COUNT plus MAX of a readable modification-date-like f 4. Review per-table include/exclude sets, limits and minimum group sizes before enabling analysis. Integrate Saved Analyses and Watches with host ESB/Quick Search when those optional host features are installed; this scaffold does not add those dependencies. 5. Use the Analyses home as the owner of analysis data. Other dashboards should link there until saved-analysis widgets are implemented. -Phase **b** profiles, drill-through, runs/cache and HTTP endpoints, and phase **c** pivot backend are implemented. Remaining work: **b** analysis UI; **c** pivot UI; **d** comparisons and relationships; **e** watch evaluation and alerts; **f** saved-analysis execution, sharing, stars integration, schedules, chart widgets, full home and exports. The current home summarizes stored rows; the host facade executes profiles. The home does not send deliveries. +Phase **b** profiles, drill-through, runs/cache and HTTP endpoints, phase **c** pivot backend, and phase **d** compare/relationships backend are implemented. Remaining work: **b** analysis UI; **c** pivot UI; **d** comparison/relationship UI; **e** watch evaluation and alerts; **f** saved-analysis execution, sharing, stars integration, schedules, chart widgets, full home and exports. The current home summarizes stored rows; the host facade executes profiles. The home does not send deliveries. ## Build diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisService.java b/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisService.java index 729b33a..6929f11 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisService.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisService.java @@ -29,6 +29,7 @@ import com.kingsrook.qbits.dataanalysis.engine.AnalysisDrillGuard; import com.kingsrook.qbits.dataanalysis.engine.AnalysisLabels; import com.kingsrook.qbits.dataanalysis.engine.AnalysisSecurity; import com.kingsrook.qbits.dataanalysis.engine.ColumnProfiler; +import com.kingsrook.qbits.dataanalysis.engine.ComparisonExplorer; import com.kingsrook.qbits.dataanalysis.engine.CoreAnalysisBackend; import com.kingsrook.qbits.dataanalysis.engine.DistinctValues; import com.kingsrook.qbits.dataanalysis.engine.DrillFilters; @@ -36,12 +37,20 @@ import com.kingsrook.qbits.dataanalysis.engine.DuplicateFinder; import com.kingsrook.qbits.dataanalysis.engine.EffectiveConfigResolver; import com.kingsrook.qbits.dataanalysis.engine.FieldAnalysisAccessResolver; import com.kingsrook.qbits.dataanalysis.engine.PivotExplorer; +import com.kingsrook.qbits.dataanalysis.engine.RelationshipExplorer; +import com.kingsrook.qbits.dataanalysis.engine.ShiftExplorer; import com.kingsrook.qbits.dataanalysis.engine.StandInAnalysisBackend; import com.kingsrook.qbits.dataanalysis.engine.TableProfiler; import com.kingsrook.qbits.dataanalysis.result.AnalysisRequest; import com.kingsrook.qbits.dataanalysis.result.Capabilities; import com.kingsrook.qbits.dataanalysis.result.ColumnProfile; import com.kingsrook.qbits.dataanalysis.result.ColumnProfileRequest; +import com.kingsrook.qbits.dataanalysis.result.CompareRequest; +import com.kingsrook.qbits.dataanalysis.result.CompareResult; +import com.kingsrook.qbits.dataanalysis.result.CorrelationRequest; +import com.kingsrook.qbits.dataanalysis.result.CorrelationResult; +import com.kingsrook.qbits.dataanalysis.result.CrosstabRequest; +import com.kingsrook.qbits.dataanalysis.result.CrosstabResult; import com.kingsrook.qbits.dataanalysis.result.DistinctValuesPage; import com.kingsrook.qbits.dataanalysis.result.DistinctValuesRequest; import com.kingsrook.qbits.dataanalysis.result.DuplicatesRequest; @@ -49,6 +58,10 @@ import com.kingsrook.qbits.dataanalysis.result.DuplicatesResult; import com.kingsrook.qbits.dataanalysis.result.PivotRequest; import com.kingsrook.qbits.dataanalysis.result.PivotResult; import com.kingsrook.qbits.dataanalysis.result.RunStatus; +import com.kingsrook.qbits.dataanalysis.result.ScatterRequest; +import com.kingsrook.qbits.dataanalysis.result.ScatterResult; +import com.kingsrook.qbits.dataanalysis.result.ShiftRequest; +import com.kingsrook.qbits.dataanalysis.result.ShiftResult; import com.kingsrook.qbits.dataanalysis.result.TableProfile; import com.kingsrook.qbits.dataanalysis.result.TableProfileRequest; import com.kingsrook.qbits.dataanalysis.run.AnalysisRunService; @@ -280,4 +293,49 @@ public class DataAnalysisService { return (AnalysisLabels.policy(table, fieldAccess)); } + + /******************************************************************************* + ** Execute compare under the current user's FULL-field and record policy. + *******************************************************************************/ + public CompareResult compare(CompareRequest request) throws QException + { + validate(request); + return (new ComparisonExplorer(security, configs.resolve(request.getTableName()), request.getTableName()).compare(request)); + } + + /******************************************************************************* + ** Execute correlation under the current user's FULL-field and record policy. + *******************************************************************************/ + public CorrelationResult correlation(CorrelationRequest request) throws QException + { + validate(request); + return (new RelationshipExplorer(security, configs.resolve(request.getTableName()), request.getTableName()).correlation(request)); + } + + /******************************************************************************* + ** Execute crosstab under the current user's FULL-field and record policy. + *******************************************************************************/ + public CrosstabResult crosstab(CrosstabRequest request) throws QException + { + validate(request); + return (new RelationshipExplorer(security, configs.resolve(request.getTableName()), request.getTableName()).crosstab(request)); + } + + /******************************************************************************* + ** Execute scatter under the current user's FULL-field and record policy. + *******************************************************************************/ + public ScatterResult scatter(ScatterRequest request) throws QException + { + validate(request); + return (new RelationshipExplorer(security, configs.resolve(request.getTableName()), request.getTableName()).scatter(request)); + } + + /******************************************************************************* + ** Execute shift under the current user's FULL-field and record policy. + *******************************************************************************/ + public ShiftResult shift(ShiftRequest request) throws QException + { + validate(request); + return (new ShiftExplorer(security, configs.resolve(request.getTableName()), request.getTableName()).shift(request)); + } } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisApi.java b/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisApi.java index 9951b86..b84e8c2 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisApi.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisApi.java @@ -28,9 +28,14 @@ import com.kingsrook.qbits.dataanalysis.engine.CoreAnalysisBackend; import com.kingsrook.qbits.dataanalysis.result.AnalysisRequest; import com.kingsrook.qbits.dataanalysis.result.ChartKind; import com.kingsrook.qbits.dataanalysis.result.ColumnProfileRequest; +import com.kingsrook.qbits.dataanalysis.result.CompareRequest; +import com.kingsrook.qbits.dataanalysis.result.CorrelationRequest; +import com.kingsrook.qbits.dataanalysis.result.CrosstabRequest; import com.kingsrook.qbits.dataanalysis.result.DistinctValuesRequest; import com.kingsrook.qbits.dataanalysis.result.DuplicatesRequest; import com.kingsrook.qbits.dataanalysis.result.PivotRequest; +import com.kingsrook.qbits.dataanalysis.result.ScatterRequest; +import com.kingsrook.qbits.dataanalysis.result.ShiftRequest; import com.kingsrook.qbits.dataanalysis.result.TableProfileRequest; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; @@ -52,7 +57,7 @@ import com.kingsrook.qqq.openapi.model.Schema; import com.kingsrook.qqq.openapi.model.Type; /******************************************************************************* - ** Session-bound profile operations registered only with optional Javalin present. + ** Session-bound analysis operations registered only with optional Javalin present. *******************************************************************************/ public class DataAnalysisApi { @@ -71,14 +76,15 @@ public class DataAnalysisApi *******************************************************************************/ public static MiddlewareSpecExtension extension() { - return (new MiddlewareSpecExtension().withName("data-analysis").withSemanticVersion("1.1.0") - .withDescription("Profiles, pivots, distinct values, duplicate groups and exact analysis runs as the signed-in user.") + return (new MiddlewareSpecExtension().withName("data-analysis").withSemanticVersion("1.2.0") + .withDescription("Profiles, pivots, comparisons, distribution shifts, correlations, scatter samples and crosstabs as the signed-in user.") .withSpec(CapabilitiesSpec.class).withSpec(ColumnProfileSpec.class).withSpec(DistinctValuesSpec.class) + .withSpec(CompareSpec.class).withSpec(ShiftSpec.class).withSpec(CorrelationSpec.class).withSpec(ScatterSpec.class).withSpec(CrosstabSpec.class) .withSpec(PivotSpec.class).withSpec(TableProfileSpec.class).withSpec(DuplicatesSpec.class).withSpec(StartRunSpec.class).withSpec(GetRunSpec.class)); } /******************************************************************************* - ** Common schema catalog and path parameters for the seven JSON specs. + ** Common schema catalog and path parameters for the analysis JSON specs. *******************************************************************************/ public abstract static class ProfileSpec extends AbstractJsonEndpointSpec { @@ -577,4 +583,178 @@ public class DataAnalysisApi return (transport(DataAnalysisRuntime.service().pivot(pivot), DataAnalysisSchemas.DataAnalysisPivotV1.class)); } } + + /******************************************************************************* + ** Strict JSON types before Jackson conversion; engine validates FULL fields. + *******************************************************************************/ + private static T relationshipRequest(String tableName, Map body, Class type) throws Exception + { + table(tableName); + DataAnalysisRuntime.service().capabilities(tableName); + try + { + for(String name : List.of("field", "dateField", "bucket", "xField", "yField", "rowField", "columnField", "labelA", "labelB")) + { + if(body.containsKey(name) && !(body.get(name) instanceof String)) + { + throw new IllegalArgumentException("Expected a field name or label."); + } + } + if(body.containsKey("fields") && (!(body.get("fields") instanceof List fields) || fields.stream().anyMatch(f -> !(f instanceof String)))) + { + throw new IllegalArgumentException("Expected an array of field names."); + } + if(body.containsKey("limit")) + { + integer(body, "limit", 2000, 1, type == CrosstabRequest.class ? 20 : 2000); + } + for(String name : List.of("filter", "filterA", "filterB")) + { + Object filter = body.get(name); + if(filter != null) + { + if(!(filter instanceof Map)) + { + throw new IllegalArgumentException("Expected a query filter object."); + } + validateFilter(JsonUtils.toObject(JsonUtils.toJson(filter), QQueryFilter.class)); + } + } + return (JsonUtils.toObject(JsonUtils.toJson(body), type)); + } + catch(Exception e) + { + throw new QBadRequestException("Invalid relationship request."); + } + } + + /******************************************************************************* + ** Compare filtered A with filtered B (null means the whole permitted table). Return means, medians or category shares, B-minus-A deltas, valid significance tests and count-equivalent drills. Only FULL fields are accepted. + *******************************************************************************/ + public static class CompareSpec extends ProfileSpec + { + /******************************************************************************* + ** Public constructor for endpoint and assistant discovery. + *******************************************************************************/ + public CompareSpec() + { + super("/tables/{table}/compare", HttpMethod.POST, "Compare two populations", + "Compare filtered A with filtered B (null means the whole permitted table). Return means, medians or category shares, B-minus-A deltas, valid significance tests and count-equivalent drills. Only FULL fields are accepted.", "CompareRequest", "Compare"); + } + + /******************************************************************************* + ** Execute with the calling session and versioned transport contract. + *******************************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + String tableName = input.getPathParam("table"); + CompareRequest request = relationshipRequest(tableName, input.getBody(), CompareRequest.class).withTableName(tableName); + return (transport(DataAnalysisRuntime.service().compare(request), RelationshipSchemas.DataAnalysisCompareV1.class)); + } + } + + /******************************************************************************* + ** Compare category shares or numeric quartiles across session-timezone calendar buckets. Compute descriptive PSI between the first and last complete nonempty buckets using fixed categories or baseline quartile bins. + *******************************************************************************/ + public static class ShiftSpec extends ProfileSpec + { + /******************************************************************************* + ** Public constructor for endpoint and assistant discovery. + *******************************************************************************/ + public ShiftSpec() + { + super("/tables/{table}/shift", HttpMethod.POST, "Measure distribution shift over time", + "Compare category shares or numeric quartiles across session-timezone calendar buckets. Compute descriptive PSI between the first and last complete nonempty buckets using fixed categories or baseline quartile bins.", "ShiftRequest", "Shift"); + } + + /******************************************************************************* + ** Execute with the calling session and versioned transport contract. + *******************************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + String tableName = input.getPathParam("table"); + ShiftRequest request = relationshipRequest(tableName, input.getBody(), ShiftRequest.class).withTableName(tableName); + return (transport(DataAnalysisRuntime.service().shift(request), RelationshipSchemas.DataAnalysisShiftV1.class)); + } + } + + /******************************************************************************* + ** Compute a Pearson correlation matrix for 2 to 12 distinct FULL numeric fields, using paired non-null records. Every cell includes n and parameters for the scatter endpoint; core pushes down CORR or uses bounded streaming. + *******************************************************************************/ + public static class CorrelationSpec extends ProfileSpec + { + /******************************************************************************* + ** Public constructor for endpoint and assistant discovery. + *******************************************************************************/ + public CorrelationSpec() + { + super("/tables/{table}/correlation", HttpMethod.POST, "Find numeric relationships", + "Compute a Pearson correlation matrix for 2 to 12 distinct FULL numeric fields, using paired non-null records. Every cell includes n and parameters for the scatter endpoint; core pushes down CORR or uses bounded streaming.", "CorrelationRequest", "Correlation"); + } + + /******************************************************************************* + ** Execute with the calling session and versioned transport contract. + *******************************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + String tableName = input.getPathParam("table"); + CorrelationRequest request = relationshipRequest(tableName, input.getBody(), CorrelationRequest.class).withTableName(tableName); + return (transport(DataAnalysisRuntime.service().correlation(request), RelationshipSchemas.DataAnalysisCorrelationV1.class)); + } + } + + /******************************************************************************* + ** Return at most 2000 paired numeric points as the user, ordered by primary key when core query sampling is unavailable, plus a least-squares line fitted to the sample. Privacy disables individual drills and suppresses small populations. + *******************************************************************************/ + public static class ScatterSpec extends ProfileSpec + { + /******************************************************************************* + ** Public constructor for endpoint and assistant discovery. + *******************************************************************************/ + public ScatterSpec() + { + super("/tables/{table}/scatter", HttpMethod.POST, "Inspect a numeric relationship", + "Return at most 2000 paired numeric points as the user, ordered by primary key when core query sampling is unavailable, plus a least-squares line fitted to the sample. Privacy disables individual drills and suppresses small populations.", "ScatterRequest", "Scatter"); + } + + /******************************************************************************* + ** Execute with the calling session and versioned transport contract. + *******************************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + String tableName = input.getPathParam("table"); + ScatterRequest request = relationshipRequest(tableName, input.getBody(), ScatterRequest.class).withTableName(tableName); + return (transport(DataAnalysisRuntime.service().scatter(request), RelationshipSchemas.DataAnalysisScatterV1.class)); + } + } + + /******************************************************************************* + ** Count combinations of two FULL fields with row and column shares, up to 20 categories per axis plus Other. Test independence using chi-square and Cramer V after pooling low expected counts; privacy suppresses small cells and their drills. + *******************************************************************************/ + public static class CrosstabSpec extends ProfileSpec + { + /******************************************************************************* + ** Public constructor for endpoint and assistant discovery. + *******************************************************************************/ + public CrosstabSpec() + { + super("/tables/{table}/crosstab", HttpMethod.POST, "Cross-tabulate two fields", + "Count combinations of two FULL fields with row and column shares, up to 20 categories per axis plus Other. Test independence using chi-square and Cramer V after pooling low expected counts; privacy suppresses small cells and their drills.", "CrosstabRequest", "Crosstab"); + } + + /******************************************************************************* + ** Execute with the calling session and versioned transport contract. + *******************************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + String tableName = input.getPathParam("table"); + CrosstabRequest request = relationshipRequest(tableName, input.getBody(), CrosstabRequest.class).withTableName(tableName); + return (transport(DataAnalysisRuntime.service().crosstab(request), RelationshipSchemas.DataAnalysisCrosstabV1.class)); + } + } } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java b/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java index 19faa6e..8c9c5b8 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java @@ -114,6 +114,7 @@ public class DataAnalysisSchemas result.put("DataAnalysisPivotMeasureInfoV1", new DataAnalysisPivotMeasureInfoV1().toSchema()); result.put("DataAnalysisPivotCellV1", new DataAnalysisPivotCellV1().toSchema()); result.put("DataAnalysisPivotV1", new DataAnalysisPivotV1().toSchema()); + result.putAll(RelationshipSchemas.components()); return (result); } @@ -181,6 +182,9 @@ public class DataAnalysisSchemas @OpenAPIDescription("Semantic field kind: NUMERIC, DATE, DATE_TIME, TEXT, BOOLEAN or CATEGORY.") public ChartKind kind; + @OpenAPIDescription("Stack category series to a total share of one when true.") + public Boolean stacked; + @OpenAPIDescription("Alternatives.") public List alternatives; @@ -203,6 +207,7 @@ public class DataAnalysisSchemas public Schema toSchema() { Map properties = new LinkedHashMap<>(); + properties.put("stacked", new Schema().withType(Type.BOOLEAN).withNullable(true).withDescription("Stack category series to a total share of one when true.")); properties.put("kind", new Schema().withType(Type.STRING).withEnumValues(java.util.Arrays.stream(ChartKind.values()).map(Enum::name).toList()).withDescription("Semantic field kind: NUMERIC, DATE, DATE_TIME, TEXT, BOOLEAN or CATEGORY.").withNullable(true)); properties.put("alternatives", new Schema().withType(Type.ARRAY).withItems(new Schema().withType(Type.STRING).withEnumValues(java.util.Arrays.stream(ChartKind.values()).map(Enum::name).toList())).withDescription("Alternatives.").withNullable(true)); properties.put("x", ref("DataAnalysisAxisV1").withDescription("X.").withNullable(true)); diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/api/RelationshipSchemas.java b/src/main/java/com/kingsrook/qbits/dataanalysis/api/RelationshipSchemas.java new file mode 100644 index 0000000..48132bb --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/api/RelationshipSchemas.java @@ -0,0 +1,663 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.api; + +import java.io.Serializable; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.middleware.javalin.schemabuilder.ToSchema; +import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIDescription; +import com.kingsrook.qqq.openapi.model.Schema; +import com.kingsrook.qqq.openapi.model.Type; + +/******************************************************************************* + ** Explicit version-one contracts for compare and relationships. + *******************************************************************************/ +public class RelationshipSchemas +{ + /******************************************************************************* + ** Register every referenced response and request component. + *******************************************************************************/ + public static Map components() + { + Map result = new LinkedHashMap<>(); + result.put("DataAnalysisCompareRequestV1", new DataAnalysisCompareRequestV1().toSchema()); + result.put("DataAnalysisShiftRequestV1", new DataAnalysisShiftRequestV1().toSchema()); + result.put("DataAnalysisCorrelationRequestV1", new DataAnalysisCorrelationRequestV1().toSchema()); + result.put("DataAnalysisScatterRequestV1", new DataAnalysisScatterRequestV1().toSchema()); + result.put("DataAnalysisCrosstabRequestV1", new DataAnalysisCrosstabRequestV1().toSchema()); + result.put("DataAnalysisSignificanceV1", new DataAnalysisSignificanceV1().toSchema()); + result.put("DataAnalysisComparisonSideV1", new DataAnalysisComparisonSideV1().toSchema()); + result.put("DataAnalysisComparisonValueV1", new DataAnalysisComparisonValueV1().toSchema()); + result.put("DataAnalysisFieldComparisonV1", new DataAnalysisFieldComparisonV1().toSchema()); + result.put("DataAnalysisCompareV1", new DataAnalysisCompareV1().toSchema()); + result.put("DataAnalysisShiftBucketV1", new DataAnalysisShiftBucketV1().toSchema()); + result.put("DataAnalysisShiftV1", new DataAnalysisShiftV1().toSchema()); + result.put("DataAnalysisCorrelationCellV1", new DataAnalysisCorrelationCellV1().toSchema()); + result.put("DataAnalysisCorrelationV1", new DataAnalysisCorrelationV1().toSchema()); + result.put("DataAnalysisTrendLineV1", new DataAnalysisTrendLineV1().toSchema()); + result.put("DataAnalysisScatterV1", new DataAnalysisScatterV1().toSchema()); + result.put("DataAnalysisCrosstabCellV1", new DataAnalysisCrosstabCellV1().toSchema()); + result.put("DataAnalysisCrosstabV1", new DataAnalysisCrosstabV1().toSchema()); + return (result); + } + + /******************************************************************************* + ** Reuse the common metric, filter and chart contracts. + *******************************************************************************/ + private static Schema ref(String name) + { + return (new Schema().withRef("#/components/schemas/" + name)); + } + + /******************************************************************************* + ** CompareRequest version-one transport model. + *******************************************************************************/ + public static class DataAnalysisCompareRequestV1 implements ToSchema + { + @OpenAPIDescription("Population A filter; null selects the whole permitted table.") + public QQueryFilter filterA; + + @OpenAPIDescription("Population B filter; null selects the whole permitted table.") + public QQueryFilter filterB; + + @OpenAPIDescription("Display name of population A; default A.") + public String labelA; + + @OpenAPIDescription("Display name of population B; default B.") + public String labelB; + + @OpenAPIDescription("Distinct FULL fields to analyze; correlation requires 2 to 12 numeric fields.") + public List fields; + + /******************************************************************************* + ** Explicit references preserve the standalone OpenAPI fragment. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("filterA", ref("QueryFilter").withDescription("Population A filter; null selects the whole permitted table.").withNullable(true)); + properties.put("filterB", ref("QueryFilter").withDescription("Population B filter; null selects the whole permitted table.").withNullable(true)); + properties.put("labelA", new Schema().withType(Type.STRING).withDescription("Display name of population A; default A.")); + properties.put("labelB", new Schema().withType(Type.STRING).withDescription("Display name of population B; default B.")); + properties.put("fields", new Schema().withType(Type.ARRAY).withItems(new Schema().withType(Type.STRING)).withDescription("Distinct FULL fields to analyze; correlation requires 2 to 12 numeric fields.")); + return (new DataAnalysisSchemas.RequiredSchema(List.of("fields")).withType(Type.OBJECT).withProperties(properties)); + } + } + + /******************************************************************************* + ** ShiftRequest version-one transport model. + *******************************************************************************/ + public static class DataAnalysisShiftRequestV1 implements ToSchema + { + @OpenAPIDescription("Current-user query filter; record locks and core field rules apply.") + public QQueryFilter filter; + + @OpenAPIDescription("FULL field whose distribution is compared over time.") + public String field; + + @OpenAPIDescription("FULL DATE or DATE_TIME field for session-timezone calendar buckets.") + public String dateField; + + @OpenAPIDescription("Calendar unit: HOUR, DAY, WEEK, MONTH, QUARTER or YEAR; default MONTH.") + public String bucket; + + /******************************************************************************* + ** Explicit references preserve the standalone OpenAPI fragment. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("filter", ref("QueryFilter").withDescription("Current-user query filter; record locks and core field rules apply.").withNullable(true)); + properties.put("field", new Schema().withType(Type.STRING).withDescription("FULL field whose distribution is compared over time.")); + properties.put("dateField", new Schema().withType(Type.STRING).withDescription("FULL DATE or DATE_TIME field for session-timezone calendar buckets.")); + properties.put("bucket", new Schema().withType(Type.STRING).withDescription("Calendar unit: HOUR, DAY, WEEK, MONTH, QUARTER or YEAR; default MONTH.")); + return (new DataAnalysisSchemas.RequiredSchema(List.of("field", "dateField")).withType(Type.OBJECT).withProperties(properties)); + } + } + + /******************************************************************************* + ** CorrelationRequest version-one transport model. + *******************************************************************************/ + public static class DataAnalysisCorrelationRequestV1 implements ToSchema + { + @OpenAPIDescription("Current-user query filter; record locks and core field rules apply.") + public QQueryFilter filter; + + @OpenAPIDescription("Distinct FULL fields to analyze; correlation requires 2 to 12 numeric fields.") + public List fields; + + /******************************************************************************* + ** Explicit references preserve the standalone OpenAPI fragment. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("filter", ref("QueryFilter").withDescription("Current-user query filter; record locks and core field rules apply.").withNullable(true)); + properties.put("fields", new Schema().withType(Type.ARRAY).withItems(new Schema().withType(Type.STRING)).withDescription("Distinct FULL fields to analyze; correlation requires 2 to 12 numeric fields.")); + return (new DataAnalysisSchemas.RequiredSchema(List.of("fields")).withType(Type.OBJECT).withProperties(properties)); + } + } + + /******************************************************************************* + ** ScatterRequest version-one transport model. + *******************************************************************************/ + public static class DataAnalysisScatterRequestV1 implements ToSchema + { + @OpenAPIDescription("Table for the scatter link; route parameter takes precedence.") + public String tableName; + + @OpenAPIDescription("Compatibility flag; scatter always returns the bounded sample.") + public Boolean exact; + @OpenAPIDescription("Current-user query filter; record locks and core field rules apply.") + public QQueryFilter filter; + + @OpenAPIDescription("FULL numeric field on the x axis.") + public String xField; + + @OpenAPIDescription("FULL numeric field on the y axis.") + public String yField; + + @OpenAPIDescription("Maximum sample points (1 to 2000, default 2000) or kept crosstab categories per axis (1 to 20, default 20).") + public Integer limit; + + /******************************************************************************* + ** Explicit references preserve the standalone OpenAPI fragment. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("tableName", new Schema().withType(Type.STRING).withDescription("Table for the scatter link; route parameter takes precedence.").withNullable(true)); + properties.put("exact", new Schema().withType(Type.BOOLEAN).withDescription("Compatibility flag; scatter always returns the bounded sample.").withNullable(true)); + properties.put("filter", ref("QueryFilter").withDescription("Current-user query filter; record locks and core field rules apply.").withNullable(true)); + properties.put("xField", new Schema().withType(Type.STRING).withDescription("FULL numeric field on the x axis.")); + properties.put("yField", new Schema().withType(Type.STRING).withDescription("FULL numeric field on the y axis.")); + properties.put("limit", new Schema().withType(Type.INTEGER).withDescription("Maximum sample points (1 to 2000, default 2000) or kept crosstab categories per axis (1 to 20, default 20).")); + return (new DataAnalysisSchemas.RequiredSchema(List.of("xField", "yField")).withType(Type.OBJECT).withProperties(properties)); + } + } + + /******************************************************************************* + ** CrosstabRequest version-one transport model. + *******************************************************************************/ + public static class DataAnalysisCrosstabRequestV1 implements ToSchema + { + @OpenAPIDescription("Current-user query filter; record locks and core field rules apply.") + public QQueryFilter filter; + + @OpenAPIDescription("FULL field defining crosstab rows.") + public String rowField; + + @OpenAPIDescription("FULL field defining crosstab columns.") + public String columnField; + + @OpenAPIDescription("Maximum sample points (1 to 2000, default 2000) or kept crosstab categories per axis (1 to 20, default 20).") + public Integer limit; + + /******************************************************************************* + ** Explicit references preserve the standalone OpenAPI fragment. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("filter", ref("QueryFilter").withDescription("Current-user query filter; record locks and core field rules apply.").withNullable(true)); + properties.put("rowField", new Schema().withType(Type.STRING).withDescription("FULL field defining crosstab rows.")); + properties.put("columnField", new Schema().withType(Type.STRING).withDescription("FULL field defining crosstab columns.")); + properties.put("limit", new Schema().withType(Type.INTEGER).withDescription("Maximum sample points (1 to 2000, default 2000) or kept crosstab categories per axis (1 to 20, default 20).")); + return (new DataAnalysisSchemas.RequiredSchema(List.of("rowField", "columnField")).withType(Type.OBJECT).withProperties(properties)); + } + } + + /******************************************************************************* + ** Significance version-one transport model. + *******************************************************************************/ + public static class DataAnalysisSignificanceV1 implements ToSchema + { + @OpenAPIDescription("Test.") + public String test; + + @OpenAPIDescription("Statistic.") + public Double statistic; + + @OpenAPIDescription("Degrees of freedom; null when undefined.") + public Double df; + + @OpenAPIDescription("Two-sided p-value; p >= 0.05 is never significant.") + public Double pValue; + + @OpenAPIDescription("Whether all test validity and privacy requirements hold.") + public Boolean applicable; + + @OpenAPIDescription("Plain-language result or reason the statistic is not applicable.") + public String sentence; + + @OpenAPIDescription("Cramer association strength computed on the pooled contingency table; null when invalid.") + public Double cramersV; + + /******************************************************************************* + ** Explicit references preserve the standalone OpenAPI fragment. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("test", new Schema().withType(Type.STRING).withDescription("Test.").withNullable(true)); + properties.put("statistic", new Schema().withType(Type.NUMBER).withDescription("Statistic.").withNullable(true)); + properties.put("df", new Schema().withType(Type.NUMBER).withDescription("Degrees of freedom; null when undefined.").withNullable(true)); + properties.put("pValue", new Schema().withType(Type.NUMBER).withDescription("Two-sided p-value; p >= 0.05 is never significant.").withNullable(true)); + properties.put("applicable", new Schema().withType(Type.BOOLEAN).withDescription("Whether all test validity and privacy requirements hold.").withNullable(true)); + properties.put("sentence", new Schema().withType(Type.STRING).withDescription("Plain-language result or reason the statistic is not applicable.").withNullable(true)); + properties.put("cramersV", new Schema().withType(Type.NUMBER).withDescription("Cramer association strength computed on the pooled contingency table; null when invalid.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withProperties(properties)); + } + } + + /******************************************************************************* + ** ComparisonSide version-one transport model. + *******************************************************************************/ + public static class DataAnalysisComparisonSideV1 implements ToSchema + { + @OpenAPIDescription("Label.") + public String label; + + @OpenAPIDescription("Typed metrics with exact population drills unless privacy suppresses them.") + public List metrics; + + /******************************************************************************* + ** Explicit references preserve the standalone OpenAPI fragment. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("label", new Schema().withType(Type.STRING).withDescription("Label.").withNullable(true)); + properties.put("metrics", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisMetricV1")).withDescription("Typed metrics with exact population drills unless privacy suppresses them.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withProperties(properties)); + } + } + + /******************************************************************************* + ** ComparisonValue version-one transport model. + *******************************************************************************/ + public static class DataAnalysisComparisonValueV1 implements ToSchema + { + @OpenAPIDescription("Value.") + public Serializable value; + + @OpenAPIDescription("Label.") + public String label; + + @OpenAPIDescription("Side a.") + public DataAnalysisComparisonSideV1 sideA; + + @OpenAPIDescription("Side b.") + public DataAnalysisComparisonSideV1 sideB; + + @OpenAPIDescription("B minus A, in original units (share differences are fractions).") + public DataAnalysisSchemas.DataAnalysisMetricV1 delta; + + @OpenAPIDescription("(B minus A) divided by absolute A, a fraction; null at a zero baseline.") + public DataAnalysisSchemas.DataAnalysisMetricV1 percentDelta; + + @OpenAPIDescription("Test assumptions, two-sided p-value, degrees of freedom and plain-language conclusion.") + public DataAnalysisSignificanceV1 significance; + + /******************************************************************************* + ** Explicit references preserve the standalone OpenAPI fragment. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("value", new Schema().withDescription("Value.").withNullable(true)); + properties.put("label", new Schema().withType(Type.STRING).withDescription("Label.").withNullable(true)); + properties.put("sideA", ref("DataAnalysisComparisonSideV1").withDescription("Side a.").withNullable(true)); + properties.put("sideB", ref("DataAnalysisComparisonSideV1").withDescription("Side b.").withNullable(true)); + properties.put("delta", ref("DataAnalysisMetricV1").withDescription("B minus A, in original units (share differences are fractions).").withNullable(true)); + properties.put("percentDelta", ref("DataAnalysisMetricV1").withDescription("(B minus A) divided by absolute A, a fraction; null at a zero baseline.").withNullable(true)); + properties.put("significance", ref("DataAnalysisSignificanceV1").withDescription("Test assumptions, two-sided p-value, degrees of freedom and plain-language conclusion.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withProperties(properties)); + } + } + + /******************************************************************************* + ** FieldComparison version-one transport model. + *******************************************************************************/ + public static class DataAnalysisFieldComparisonV1 implements ToSchema + { + @OpenAPIDescription("FULL field whose distribution is compared over time.") + public String field; + + @OpenAPIDescription("Values.") + public List values; + + @OpenAPIDescription("Test assumptions, two-sided p-value, degrees of freedom and plain-language conclusion.") + public DataAnalysisSignificanceV1 significance; + + @OpenAPIDescription("Semantic series; category comparisons are grouped and category time series are stacked.") + public DataAnalysisSchemas.DataAnalysisChartV1 chart; + + /******************************************************************************* + ** Explicit references preserve the standalone OpenAPI fragment. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("field", new Schema().withType(Type.STRING).withDescription("FULL field whose distribution is compared over time.").withNullable(true)); + properties.put("values", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisComparisonValueV1")).withDescription("Values.").withNullable(true)); + properties.put("significance", ref("DataAnalysisSignificanceV1").withDescription("Test assumptions, two-sided p-value, degrees of freedom and plain-language conclusion.").withNullable(true)); + properties.put("chart", ref("DataAnalysisChartV1").withDescription("Semantic series; category comparisons are grouped and category time series are stacked.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withProperties(properties)); + } + } + + /******************************************************************************* + ** CompareResult version-one transport model. + *******************************************************************************/ + public static class DataAnalysisCompareV1 implements ToSchema + { + @OpenAPIDescription("Distinct FULL fields to analyze; correlation requires 2 to 12 numeric fields.") + public List fields; + + /******************************************************************************* + ** Explicit references preserve the standalone OpenAPI fragment. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("fields", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisFieldComparisonV1")).withDescription("Distinct FULL fields to analyze; correlation requires 2 to 12 numeric fields.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withProperties(properties)); + } + } + + /******************************************************************************* + ** ShiftBucket version-one transport model. + *******************************************************************************/ + public static class DataAnalysisShiftBucketV1 implements ToSchema + { + @OpenAPIDescription("Value.") + public Serializable value; + + @OpenAPIDescription("Label.") + public String label; + + @OpenAPIDescription("True when the calendar bucket end has passed in the session timezone.") + public Boolean complete; + + @OpenAPIDescription("Typed metrics with exact population drills unless privacy suppresses them.") + public List metrics; + + /******************************************************************************* + ** Explicit references preserve the standalone OpenAPI fragment. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("value", new Schema().withDescription("Value.").withNullable(true)); + properties.put("label", new Schema().withType(Type.STRING).withDescription("Label.").withNullable(true)); + properties.put("complete", new Schema().withType(Type.BOOLEAN).withDescription("True when the calendar bucket end has passed in the session timezone.").withNullable(true)); + properties.put("metrics", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisMetricV1")).withDescription("Typed metrics with exact population drills unless privacy suppresses them.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withProperties(properties)); + } + } + + /******************************************************************************* + ** ShiftResult version-one transport model. + *******************************************************************************/ + public static class DataAnalysisShiftV1 implements ToSchema + { + @OpenAPIDescription("Buckets.") + public List buckets; + + @OpenAPIDescription("Population stability index, null when inapplicable; zero shares are floored at 0.000001.") + public DataAnalysisSchemas.DataAnalysisMetricV1 psi; + + @OpenAPIDescription("Plain-language result or reason the statistic is not applicable.") + public String sentence; + + @OpenAPIDescription("Semantic series; category comparisons are grouped and category time series are stacked.") + public DataAnalysisSchemas.DataAnalysisChartV1 chart; + + /******************************************************************************* + ** Explicit references preserve the standalone OpenAPI fragment. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("buckets", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisShiftBucketV1")).withDescription("Buckets.").withNullable(true)); + properties.put("psi", ref("DataAnalysisMetricV1").withDescription("Population stability index, null when inapplicable; zero shares are floored at 0.000001.").withNullable(true)); + properties.put("sentence", new Schema().withType(Type.STRING).withDescription("Plain-language result or reason the statistic is not applicable.").withNullable(true)); + properties.put("chart", ref("DataAnalysisChartV1").withDescription("Semantic series; category comparisons are grouped and category time series are stacked.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withProperties(properties)); + } + } + + /******************************************************************************* + ** CorrelationCell version-one transport model. + *******************************************************************************/ + public static class DataAnalysisCorrelationCellV1 implements ToSchema + { + @OpenAPIDescription("FULL numeric field on the x axis.") + public String xField; + + @OpenAPIDescription("FULL numeric field on the y axis.") + public String yField; + + @OpenAPIDescription("Pairwise non-null population count; null when privacy suppresses it.") + public Long n; + + @OpenAPIDescription("Pearson correlation; null for insufficient data, zero variance or privacy suppression.") + public Double r; + + @OpenAPIDescription("POST the included parameters to the table scatter endpoint.") + public DataAnalysisScatterRequestV1 scatter; + + /******************************************************************************* + ** Explicit references preserve the standalone OpenAPI fragment. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("xField", new Schema().withType(Type.STRING).withDescription("FULL numeric field on the x axis.").withNullable(true)); + properties.put("yField", new Schema().withType(Type.STRING).withDescription("FULL numeric field on the y axis.").withNullable(true)); + properties.put("n", new Schema().withType(Type.INTEGER).withDescription("Pairwise non-null population count; null when privacy suppresses it.").withNullable(true)); + properties.put("r", new Schema().withType(Type.NUMBER).withDescription("Pearson correlation; null for insufficient data, zero variance or privacy suppression.").withNullable(true)); + properties.put("scatter", ref("DataAnalysisScatterRequestV1").withDescription("POST the included parameters to the table scatter endpoint.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withProperties(properties)); + } + } + + /******************************************************************************* + ** CorrelationResult version-one transport model. + *******************************************************************************/ + public static class DataAnalysisCorrelationV1 implements ToSchema + { + @OpenAPIDescription("Distinct FULL fields to analyze; correlation requires 2 to 12 numeric fields.") + public List fields; + + @OpenAPIDescription("Cells.") + public List cells; + + @OpenAPIDescription("Semantic series; category comparisons are grouped and category time series are stacked.") + public DataAnalysisSchemas.DataAnalysisChartV1 chart; + + /******************************************************************************* + ** Explicit references preserve the standalone OpenAPI fragment. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("fields", new Schema().withType(Type.ARRAY).withItems(new Schema().withType(Type.STRING)).withDescription("Distinct FULL fields to analyze; correlation requires 2 to 12 numeric fields.").withNullable(true)); + properties.put("cells", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisCorrelationCellV1")).withDescription("Cells.").withNullable(true)); + properties.put("chart", ref("DataAnalysisChartV1").withDescription("Semantic series; category comparisons are grouped and category time series are stacked.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withProperties(properties)); + } + } + + /******************************************************************************* + ** TrendLine version-one transport model. + *******************************************************************************/ + public static class DataAnalysisTrendLineV1 implements ToSchema + { + @OpenAPIDescription("Slope.") + public Double slope; + + @OpenAPIDescription("Intercept.") + public Double intercept; + + @OpenAPIDescription("Pearson correlation; null for insufficient data, zero variance or privacy suppression.") + public Double r; + + /******************************************************************************* + ** Explicit references preserve the standalone OpenAPI fragment. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("slope", new Schema().withType(Type.NUMBER).withDescription("Slope.").withNullable(true)); + properties.put("intercept", new Schema().withType(Type.NUMBER).withDescription("Intercept.").withNullable(true)); + properties.put("r", new Schema().withType(Type.NUMBER).withDescription("Pearson correlation; null for insufficient data, zero variance or privacy suppression.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withProperties(properties)); + } + } + + /******************************************************************************* + ** ScatterResult version-one transport model. + *******************************************************************************/ + public static class DataAnalysisScatterV1 implements ToSchema + { + @OpenAPIDescription("Pairwise non-null population count; null when privacy suppresses it.") + public Long n; + + @OpenAPIDescription("Number of displayed paired records.") + public Integer sampleSize; + + @OpenAPIDescription("Sampling method; core query currently uses a deterministic primary-key prefix.") + public String sampling; + + @OpenAPIDescription("Points.") + public List points; + + @OpenAPIDescription("Least-squares slope, intercept and r fitted to the displayed sample.") + public DataAnalysisTrendLineV1 trend; + + @OpenAPIDescription("Semantic series; category comparisons are grouped and category time series are stacked.") + public DataAnalysisSchemas.DataAnalysisChartV1 chart; + + /******************************************************************************* + ** Explicit references preserve the standalone OpenAPI fragment. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("n", new Schema().withType(Type.INTEGER).withDescription("Pairwise non-null population count; null when privacy suppresses it.").withNullable(true)); + properties.put("sampleSize", new Schema().withType(Type.INTEGER).withDescription("Number of displayed paired records.").withNullable(true)); + properties.put("sampling", new Schema().withType(Type.STRING).withDescription("Sampling method; core query currently uses a deterministic primary-key prefix.").withNullable(true)); + properties.put("points", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisPointV1")).withDescription("Points.").withNullable(true)); + properties.put("trend", ref("DataAnalysisTrendLineV1").withDescription("Least-squares slope, intercept and r fitted to the displayed sample.").withNullable(true)); + properties.put("chart", ref("DataAnalysisChartV1").withDescription("Semantic series; category comparisons are grouped and category time series are stacked.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withProperties(properties)); + } + } + + /******************************************************************************* + ** CrosstabCell version-one transport model. + *******************************************************************************/ + public static class DataAnalysisCrosstabCellV1 implements ToSchema + { + @OpenAPIDescription("Row index.") + public Integer rowIndex; + + @OpenAPIDescription("Column index.") + public Integer columnIndex; + + @OpenAPIDescription("Cell count with its exact drill, or a suppressed metric.") + public DataAnalysisSchemas.DataAnalysisMetricV1 count; + + @OpenAPIDescription("Cell count divided by row total; suppressed if it could disclose a hidden cell.") + public DataAnalysisSchemas.DataAnalysisMetricV1 rowShare; + + @OpenAPIDescription("Cell count divided by column total; suppressed if it could disclose a hidden cell.") + public DataAnalysisSchemas.DataAnalysisMetricV1 columnShare; + + /******************************************************************************* + ** Explicit references preserve the standalone OpenAPI fragment. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("rowIndex", new Schema().withType(Type.INTEGER).withDescription("Row index.").withNullable(true)); + properties.put("columnIndex", new Schema().withType(Type.INTEGER).withDescription("Column index.").withNullable(true)); + properties.put("count", ref("DataAnalysisMetricV1").withDescription("Cell count with its exact drill, or a suppressed metric.").withNullable(true)); + properties.put("rowShare", ref("DataAnalysisMetricV1").withDescription("Cell count divided by row total; suppressed if it could disclose a hidden cell.").withNullable(true)); + properties.put("columnShare", ref("DataAnalysisMetricV1").withDescription("Cell count divided by column total; suppressed if it could disclose a hidden cell.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withProperties(properties)); + } + } + + /******************************************************************************* + ** CrosstabResult version-one transport model. + *******************************************************************************/ + public static class DataAnalysisCrosstabV1 implements ToSchema + { + @OpenAPIDescription("Rows.") + public List rows; + + @OpenAPIDescription("Columns.") + public List columns; + + @OpenAPIDescription("Cells.") + public List cells; + + @OpenAPIDescription("Test assumptions, two-sided p-value, degrees of freedom and plain-language conclusion.") + public DataAnalysisSignificanceV1 significance; + + /******************************************************************************* + ** Explicit references preserve the standalone OpenAPI fragment. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("rows", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisPivotHeaderV1")).withDescription("Rows.").withNullable(true)); + properties.put("columns", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisPivotHeaderV1")).withDescription("Columns.").withNullable(true)); + properties.put("cells", new Schema().withType(Type.ARRAY).withItems(ref("DataAnalysisCrosstabCellV1")).withDescription("Cells.").withNullable(true)); + properties.put("significance", ref("DataAnalysisSignificanceV1").withDescription("Test assumptions, two-sided p-value, degrees of freedom and plain-language conclusion.").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withProperties(properties)); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ComparisonExplorer.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ComparisonExplorer.java new file mode 100644 index 0000000..ce5676a --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ComparisonExplorer.java @@ -0,0 +1,254 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.math.BigDecimal; +import java.math.MathContext; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.dataanalysis.result.ChartKind; +import com.kingsrook.qbits.dataanalysis.result.CompareRequest; +import com.kingsrook.qbits.dataanalysis.result.CompareResult; +import com.kingsrook.qbits.dataanalysis.result.ComparisonSide; +import com.kingsrook.qbits.dataanalysis.result.ComparisonValue; +import com.kingsrook.qbits.dataanalysis.result.FieldComparison; +import com.kingsrook.qbits.dataanalysis.result.Metric; +import com.kingsrook.qbits.dataanalysis.result.Point; +import com.kingsrook.qbits.dataanalysis.result.Series; +import com.kingsrook.qbits.dataanalysis.result.ValueType; +import com.kingsrook.qbits.dataanalysis.stats.Statistics; +import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; + +/******************************************************************************* + ** Two populations share category boundaries and exact producing predicates. + *******************************************************************************/ +public class ComparisonExplorer extends RelationshipSupport +{ + /******************************************************************************* + ** Bind request-local policy without modifying session metadata. + *******************************************************************************/ + public ComparisonExplorer(AnalysisSecurity security, EffectiveConfig config, String table) + { + super(security, config, table); + } + + /******************************************************************************* + ** Resolve every field before issuing source queries. + *******************************************************************************/ + public CompareResult compare(CompareRequest request) throws QException + { + if(request.getFields() == null || request.getFields().isEmpty() || request.getFields().size() > 100 + || request.getFields().stream().distinct().count() != request.getFields().size()) + { + throw new QBadRequestException("Choose 1 to 100 distinct analysis fields."); + } + for(String field : request.getFields()) + { + full(field); + } + CompareResult result = new CompareResult(); + for(String field : request.getFields()) + { + result.getFields().add(numeric(full(field)) ? numericComparison(request, field) : categoryComparison(request, field)); + } + return (result); + } + + /******************************************************************************* + ** All moments and quartiles are computed together by core, including VAR_SAMP. + *******************************************************************************/ + protected Map moments(String field, QQueryFilter filter) throws QException + { + Map measures = new LinkedHashMap<>(); + measures.put("count", measure(field, AggregateOperator.COUNT)); + measures.put("mean", measure(field, AggregateOperator.AVG)); + measures.put("variance", measure(field, AggregateOperator.VAR_SAMP)); + measures.put("min", measure(field, AggregateOperator.MIN)); + measures.put("max", measure(field, AggregateOperator.MAX)); + for(int p : List.of(25, 50, 75)) + { + measures.put("p" + p, measure(field, AggregateOperator.PERCENTILE_CONT).withPercentile(BigDecimal.valueOf(p, 2))); + } + var row = summarize(filter, new ArrayList<>(measures.values())); + Map result = new LinkedHashMap<>(); + measures.forEach((name, aggregate) -> result.put(name, decimal(row.getAggregateValues().get(aggregate)))); + result.putIfAbsent("count", BigDecimal.ZERO); + return (result); + } + + /******************************************************************************* + ** Two BOX series retain all five summary values; mean and median get drills. + *******************************************************************************/ + private FieldComparison numericComparison(CompareRequest request, String field) throws QException + { + Map a = moments(field, request.getFilterA()); + Map b = moments(field, request.getFilterB()); + long nA = a.get("count").longValue(); + long nB = b.get("count").longValue(); + ComparisonSide sideA = numericSide(field, request.getLabelA(), request.getFilterA(), a); + ComparisonSide sideB = numericSide(field, request.getLabelB(), request.getFilterB(), b); + ComparisonValue value = new ComparisonValue().withSideA(sideA).withSideB(sideB).withLabel(field); + boolean hidden = nA < config.getMinGroupSize() || nB < config.getMinGroupSize(); + if(!hidden) + { + deltas(value, a.get("mean"), b.get("mean")); + } + Statistics.Result test = hidden ? Statistics.unavailable("Welch's t-test", "privacy guard") + : Statistics.welch(nA, number(a, "mean"), number(a, "variance"), nB, number(b, "mean"), number(b, "variance")); + return (new FieldComparison().withField(field).withValues(List.of(value)).withSignificance(significance(test)) + .withChart(chart(ChartKind.BOX, List.of(box(sideA), box(sideB)), "Two box summaries; delta is B minus A."))); + } + + /******************************************************************************* + ** Null moments remain invalid test inputs rather than becoming zero variance. + *******************************************************************************/ + private static double number(Map values, String key) + { + return (values.get(key) == null ? Double.NaN : values.get(key).doubleValue()); + } + + /******************************************************************************* + ** Count means non-null numeric records; every summary drills to that set. + *******************************************************************************/ + protected ComparisonSide numericSide(String field, String label, QQueryFilter filter, Map values) + { + ComparisonSide result = new ComparisonSide().withLabel(label); + for(String key : List.of("count", "mean", "min", "p25", "p50", "p75", "max")) + { + String name = key.equals("p50") ? "median" : key; + result.getMetrics().add(metric(name, values.get(key), key.equals("count") ? ValueType.COUNT : ValueType.DECIMAL, filter, DrillFilters.nonNull(field, false))); + } + guard(result.getMetrics(), values.get("count").longValue()); + return (result); + } + + /******************************************************************************* + ** Only guarded summary metrics enter charts. + *******************************************************************************/ + private Series box(ComparisonSide side) + { + Series series = new Series().withName(side.getLabel()); + side.getMetrics().stream().filter(m -> !List.of("count", "mean").contains(m.getKey()) && m.getValue() != null) + .forEach(m -> series.getPoints().add(new Point().withX(m.getKey()).withY(decimal(m.getValue())).withLabel(m.getLabel()).withDrill(m.getDrill()))); + return (series); + } + + /******************************************************************************* + ** Top ten across the union gives both populations the same categories. + ** Expected counts below five pool into Other before display and inference. + *******************************************************************************/ + private FieldComparison categoryComparison(CompareRequest request, String field) throws QException + { + QQueryFilter union = request.getFilterA() == null || request.getFilterB() == null ? null + : new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR).withSubFilter(DrillFilters.copy(request.getFilterA())).withSubFilter(DrillFilters.copy(request.getFilterB())); + long nA = size(request.getFilterA()); + long nB = size(request.getFilterB()); + List kept = new ArrayList<>(); + List other = new ArrayList<>(); + boolean privateOther = false; + for(Category category : categories(field, union, 10)) + { + long a = size(DrillFilters.and(request.getFilterA(), category.condition())); + long b = size(DrillFilters.and(request.getFilterB(), category.condition())); + boolean hidden = category.privateGroup() || a < config.getMinGroupSize() || b < config.getMinGroupSize(); + double pooled = nA + nB == 0 ? 0 : (double) (a + b) / (nA + nB); + if(hidden || Math.min(nA * pooled, nB * pooled) < 5 || category.label().equals("Other") && category.value() == null) + { + other.add(category); + privateOther |= hidden; + } + else + { + kept.add(category); + } + } + if(!other.isEmpty()) + { + kept.add(merge(other, privateOther)); + } + double[][] counts = new double[2][kept.size()]; + FieldComparison result = new FieldComparison().withField(field); + Series seriesA = new Series().withName(request.getLabelA()); + Series seriesB = new Series().withName(request.getLabelB()); + for(int i = 0; i < kept.size(); i++) + { + Category category = kept.get(i); + long a = size(DrillFilters.and(request.getFilterA(), category.condition())); + long b = size(DrillFilters.and(request.getFilterB(), category.condition())); + counts[0][i] = a; + counts[1][i] = b; + ComparisonSide sideA = categorySide(request.getLabelA(), request.getFilterA(), category, a, nA); + ComparisonSide sideB = categorySide(request.getLabelB(), request.getFilterB(), category, b, nB); + ComparisonValue value = new ComparisonValue().withValue(category.value()).withLabel(category.label()).withSideA(sideA).withSideB(sideB); + boolean hidden = a < config.getMinGroupSize() || b < config.getMinGroupSize(); + if(!hidden) + { + deltas(value, ProfileSupport.share(a, nA), ProfileSupport.share(b, nB)); + } + value.withSignificance(significance(hidden || category.privateGroup() ? Statistics.unavailable("Two-proportion z-test", "privacy guard") : Statistics.proportions(a, nA, b, nB))); + result.getValues().add(value); + categoryPoint(seriesA, category, sideA); + categoryPoint(seriesB, category, sideB); + } + result.withSignificance(significance(privateOther ? Statistics.unavailable("Chi-square homogeneity", "privacy guard") : Statistics.chiSquare(counts, true))); + return (result.withChart(chart(ChartKind.BAR, List.of(seriesA, seriesB), "Grouped bars; delta is B minus A. Low expected counts are merged into Other."))); + } + + /******************************************************************************* + ** Sensitive merged categories reveal no predicate or original value. + *******************************************************************************/ + private ComparisonSide categorySide(String label, QQueryFilter filter, Category category, long n, long total) + { + ComparisonSide side = new ComparisonSide().withLabel(label).withMetrics(List.of(metric("count", n, ValueType.COUNT, filter, category.condition()), + metric("share", ProfileSupport.share(n, total), ValueType.PERCENT, filter, category.condition()))); + if(category.privateGroup()) + { + side.getMetrics().forEach(m -> m.withDrill(null)); + } + guard(side.getMetrics(), n); + return (side); + } + + /******************************************************************************* + ** Charts reflect only released count metrics. + *******************************************************************************/ + private void categoryPoint(Series series, Category category, ComparisonSide side) + { + Metric count = side.getMetrics().getFirst(); + series.getPoints().add(new Point().withX(category.label()).withLabel(category.label()).withY(decimal(count.getValue())).withDrill(count.getDrill())); + } + + /******************************************************************************* + ** Percent delta is a fraction relative to |A|; zero baseline is undefined. + *******************************************************************************/ + private void deltas(ComparisonValue value, BigDecimal a, BigDecimal b) + { + BigDecimal delta = a == null || b == null ? null : b.subtract(a); + value.withDelta(ProfileSupport.metric("delta", delta, ValueType.DECIMAL)); + value.withPercentDelta(ProfileSupport.metric("percentDelta", delta == null || a.signum() == 0 ? null : delta.divide(a.abs(), MathContext.DECIMAL128), ValueType.PERCENT)); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MetricLabels.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MetricLabels.java index 2b7ce09..964d953 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MetricLabels.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MetricLabels.java @@ -30,6 +30,8 @@ import com.kingsrook.qbits.dataanalysis.result.ValueType; public final class MetricLabels { private static final Map LABELS = Map.ofEntries( + Map.entry("count", "Count"), Map.entry("delta", "Difference"), Map.entry("percentDelta", "Percent change"), + Map.entry("psi", "Population stability index"), Map.entry("rowShare", "Row share"), Map.entry("columnShare", "Column share"), Map.entry("rows", "Rows"), Map.entry("nonNull", "Non-empty"), Map.entry("nulls", "Nulls"), Map.entry("blanks", "Blanks"), Map.entry("distinct", "Distinct values"), Map.entry("uniqueness", "Uniqueness"), Map.entry("completeness", "Completeness"), Map.entry("share", "Share"), diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/RelationshipExplorer.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/RelationshipExplorer.java new file mode 100644 index 0000000..36eeacf --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/RelationshipExplorer.java @@ -0,0 +1,255 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import com.kingsrook.qbits.dataanalysis.result.ChartKind; +import com.kingsrook.qbits.dataanalysis.result.CorrelationCell; +import com.kingsrook.qbits.dataanalysis.result.CorrelationRequest; +import com.kingsrook.qbits.dataanalysis.result.CorrelationResult; +import com.kingsrook.qbits.dataanalysis.result.CrosstabCell; +import com.kingsrook.qbits.dataanalysis.result.CrosstabRequest; +import com.kingsrook.qbits.dataanalysis.result.CrosstabResult; +import com.kingsrook.qbits.dataanalysis.result.Metric; +import com.kingsrook.qbits.dataanalysis.result.PivotHeader; +import com.kingsrook.qbits.dataanalysis.result.Point; +import com.kingsrook.qbits.dataanalysis.result.ScatterRequest; +import com.kingsrook.qbits.dataanalysis.result.ScatterResult; +import com.kingsrook.qbits.dataanalysis.result.Series; +import com.kingsrook.qbits.dataanalysis.result.TrendLine; +import com.kingsrook.qbits.dataanalysis.result.ValueType; +import com.kingsrook.qbits.dataanalysis.stats.Statistics; +import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; + +/******************************************************************************* + ** Pairwise correlation, bounded scatter samples and guarded contingency tables. + *******************************************************************************/ +public class RelationshipExplorer extends RelationshipSupport +{ + /******************************************************************************* + ** Each operation has its own execution state and policy snapshot. + *******************************************************************************/ + public RelationshipExplorer(AnalysisSecurity security, EffectiveConfig config, String table) + { + super(security, config, table); + } + + /******************************************************************************* + ** All unique CORR pairs share one aggregate call, including streamed fallback. + ** Pair counts use core CountAction because Aggregate has no per-measure filter. + *******************************************************************************/ + public CorrelationResult correlation(CorrelationRequest request) throws QException + { + List fields = request.getFields(); + if(fields == null || fields.size() < 2 || fields.size() > 12 || new LinkedHashSet<>(fields).size() != fields.size()) + { + throw new QBadRequestException("Choose 2 to 12 distinct numeric fields."); + } + for(String field : fields) + { + requireNumeric(field); + } + List aggregates = new ArrayList<>(); + for(int i = 0; i < fields.size(); i++) + { + for(int j = i; j < fields.size(); j++) + { + aggregates.add(measure(fields.get(i), AggregateOperator.CORR).withOtherFieldName(fields.get(j))); + } + } + var values = summarize(request.getFilter(), aggregates).getAggregateValues(); + CorrelationResult result = new CorrelationResult().withFields(List.copyOf(fields)); + List series = new ArrayList<>(); + for(int i = 0; i < fields.size(); i++) + { + String x = fields.get(i); + Series row = new Series().withName(x); + for(int j = 0; j < fields.size(); j++) + { + String y = fields.get(j); + Aggregate key = measure(fields.get(Math.min(i, j)), AggregateOperator.CORR).withOtherFieldName(fields.get(Math.max(i, j))); + BigDecimal value = decimal(values.get(key)); + long n = size(paired(request.getFilter(), x, y)); + boolean hidden = n < config.getMinGroupSize(); + Double r = hidden || n < 2 || value == null ? null : value.doubleValue(); + result.getCells().add(new CorrelationCell().withXField(x).withYField(y).withN(hidden ? null : n).withR(r) + .withScatter(hidden ? null : new ScatterRequest().withTableName(table).withXField(x).withYField(y).withFilter(DrillFilters.copy(request.getFilter())))); + row.getPoints().add(new Point().withX(y).withY(r == null ? null : BigDecimal.valueOf(r)).withLabel(x + " / " + y)); + } + series.add(row); + } + return (result.withChart(chart(ChartKind.HEATMAP, series, "Pearson r; each cell includes its paired non-null count and scatter request."))); + } + + /******************************************************************************* + ** QueryInput has no sampling API. Fetch the first N paired records by primary + ** key as the user, with core-enforced record locks and a maximum of 2,000. + *******************************************************************************/ + public ScatterResult scatter(ScatterRequest request) throws QException + { + requireNumeric(request.getXField()); + requireNumeric(request.getYField()); + if(request.getLimit() == null || request.getLimit() < 1 || request.getLimit() > 2000) + { + throw new QBadRequestException("Scatter limit must be from 1 to 2000."); + } + String x = request.getXField(); + String y = request.getYField(); + String pk = primaryKey(table); + QQueryFilter filter = paired(request.getFilter(), x, y); + long n = size(filter); + ScatterResult result = new ScatterResult().withN(n).withSampling("First N paired records by primary key").withSampleSize(0); + if(n < config.getMinGroupSize()) + { + return (result.withN(null).withChart(chart(ChartKind.SCATTER, List.of(), "Hidden: fewer than " + config.getMinGroupSize() + " paired records."))); + } + Set selected = new LinkedHashSet<>(List.of(x, y)); + boolean drill = config.getMinGroupSize() == 0 && security.analyzableFields(table).stream().anyMatch(f -> f.getName().equals(pk)); + if(drill) + { + selected.add(pk); + } + var records = rows(table, filter, selected, request.getLimit()); + double[] xs = new double[records.size()]; + double[] ys = new double[records.size()]; + for(int i = 0; i < records.size(); i++) + { + var record = records.get(i); + BigDecimal xv = decimal(record.getValue(x)); + BigDecimal yv = decimal(record.getValue(y)); + xs[i] = xv.doubleValue(); + ys[i] = yv.doubleValue(); + result.getPoints().add(new Point().withX(xv).withY(yv).withDrill(drill ? DrillFilters.drill(table, null, + DrillFilters.condition(pk, QCriteriaOperator.EQUALS, record.getValue(pk))) : null)); + } + result.withSampleSize(records.size()).withTrend(trend(xs, ys)); + Series points = new Series().withName("Sample").withPoints(result.getPoints()); + Series line = new Series().withName("Least-squares trend"); + if(result.getTrend() != null) + { + double low = java.util.Arrays.stream(xs).min().orElseThrow(); + double high = java.util.Arrays.stream(xs).max().orElseThrow(); + for(double value : new double[] {low, high}) + { + line.getPoints().add(new Point().withX(BigDecimal.valueOf(value)).withY(BigDecimal.valueOf(result.getTrend().getIntercept() + result.getTrend().getSlope() * value))); + } + } + return (result.withChart(chart(ChartKind.SCATTER, List.of(points, line), "Trend describes the displayed sample, not the full population."))); + } + + /******************************************************************************* + ** Centered least squares remains stable when coordinates have large offsets. + *******************************************************************************/ + private TrendLine trend(double[] x, double[] y) + { + if(x.length < 2) + { + return (null); + } + double meanX = java.util.Arrays.stream(x).average().orElseThrow(); + double meanY = java.util.Arrays.stream(y).average().orElseThrow(); + double xx = 0; + double xy = 0; + for(int i = 0; i < x.length; i++) + { + xx += (x[i] - meanX) * (x[i] - meanX); + xy += (x[i] - meanX) * (y[i] - meanY); + } + return (xx == 0 ? null : new TrendLine().withSlope(xy / xx).withIntercept(meanY - xy / xx * meanX).withR(Statistics.pearson(x, y))); + } + + /******************************************************************************* + ** Cell counts include zero combinations. Suppression also removes shares + ** that could reconstruct a hidden cell from a known marginal total. + *******************************************************************************/ + public CrosstabResult crosstab(CrosstabRequest request) throws QException + { + full(request.getRowField()); + full(request.getColumnField()); + if(request.getRowField().equals(request.getColumnField()) || request.getLimit() == null || request.getLimit() < 1 || request.getLimit() > 20) + { + throw new QBadRequestException("Choose distinct crosstab fields and a limit from 1 to 20."); + } + List rows = categories(request.getRowField(), request.getFilter(), request.getLimit()); + List columns = categories(request.getColumnField(), request.getFilter(), request.getLimit()); + long[] rowTotals = new long[rows.size()]; + long[] columnTotals = new long[columns.size()]; + boolean[] hiddenRows = new boolean[rows.size()]; + boolean[] hiddenColumns = new boolean[columns.size()]; + double[][] counts = new double[rows.size()][columns.size()]; + boolean hidden = false; + for(int i = 0; i < rows.size(); i++) + { + for(int j = 0; j < columns.size(); j++) + { + long n = size(DrillFilters.and(request.getFilter(), DrillFilters.and(rows.get(i).condition(), columns.get(j).condition()))); + counts[i][j] = n; + rowTotals[i] += n; + columnTotals[j] += n; + if(n < config.getMinGroupSize()) + { + hidden = true; + hiddenRows[i] = true; + hiddenColumns[j] = true; + } + } + } + CrosstabResult result = new CrosstabResult().withRows(rows.stream().map(c -> new PivotHeader().withValue(c.value()).withLabel(c.label())).toList()) + .withColumns(columns.stream().map(c -> new PivotHeader().withValue(c.value()).withLabel(c.label())).toList()); + for(int i = 0; i < rows.size(); i++) + { + for(int j = 0; j < columns.size(); j++) + { + long n = (long) counts[i][j]; + QQueryFilter condition = DrillFilters.and(rows.get(i).condition(), columns.get(j).condition()); + Metric count = metric("count", n, ValueType.COUNT, request.getFilter(), condition); + Metric rowShare = metric("rowShare", ProfileSupport.share(n, rowTotals[i]), ValueType.PERCENT, request.getFilter(), condition); + Metric columnShare = metric("columnShare", ProfileSupport.share(n, columnTotals[j]), ValueType.PERCENT, request.getFilter(), condition); + List metrics = List.of(count, rowShare, columnShare); + if(rows.get(i).privateGroup() || columns.get(j).privateGroup()) + { + metrics.forEach(m -> m.withDrill(null)); + hidden = true; + } + guard(metrics, n); + if(hiddenRows[i]) + { + guard(List.of(rowShare), 0); + } + if(hiddenColumns[j]) + { + guard(List.of(columnShare), 0); + } + result.getCells().add(new CrosstabCell().withRowIndex(i).withColumnIndex(j).withCount(count).withRowShare(rowShare).withColumnShare(columnShare)); + } + } + return (result.withSignificance(significance(hidden ? Statistics.unavailable("Chi-square independence", "privacy guard") : Statistics.chiSquare(counts, false)))); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/RelationshipSupport.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/RelationshipSupport.java new file mode 100644 index 0000000..aa9def8 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/RelationshipSupport.java @@ -0,0 +1,241 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.io.Serializable; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import com.kingsrook.qbits.dataanalysis.result.Chart; +import com.kingsrook.qbits.dataanalysis.result.ChartKind; +import com.kingsrook.qbits.dataanalysis.result.Metric; +import com.kingsrook.qbits.dataanalysis.result.Series; +import com.kingsrook.qbits.dataanalysis.result.Significance; +import com.kingsrook.qbits.dataanalysis.result.ValueType; +import com.kingsrook.qbits.dataanalysis.stats.Statistics; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; +import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction; +import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; + +/******************************************************************************* + ** Shared core execution and semantic output for request-local relationships. + *******************************************************************************/ +public class RelationshipSupport extends StandInAnalysisBackend +{ + protected final AnalysisSecurity security; + protected final String table; + + /******************************************************************************* + ** A bucket retains exact membership; privacy buckets intentionally lack drills. + *******************************************************************************/ + protected record Category(Serializable value, String label, QQueryFilter condition, boolean privateGroup) + { + } + + /******************************************************************************* + ** Bind the caller's already-authorized table and current policy. + *******************************************************************************/ + public RelationshipSupport(AnalysisSecurity security, EffectiveConfig config, String table) + { + super(config); + this.security = security; + this.table = table; + } + + /******************************************************************************* + ** Core owns pushdown and bounded fallback. Never label a partial test exact. + *******************************************************************************/ + @Override + protected List aggregate(AggregateInput input) throws QException + { + input.withFilter(DrillFilters.restoreTypes(table, input.getFilter())).withAllowStreamedFallback(true).withStreamedRowLimit(config.getMaxRows()); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + var output = new AggregateAction().execute(input); + if(Boolean.TRUE.equals(output.getTruncated())) + { + throw new QBadRequestException("Analysis exceeds the streamed row limit; narrow the filter."); + } + return (output.getResults()); + } + + /******************************************************************************* + ** All named fields, including PVS numerics, must pass the FULL resolver first. + *******************************************************************************/ + protected QFieldMetaData full(String name) throws QException + { + return (security.field(table, name)); + } + + /******************************************************************************* + ** PVS identifiers are categorical even when their storage type is numeric. + *******************************************************************************/ + protected boolean numeric(QFieldMetaData field) + { + return (field.getPossibleValueSourceName() == null && List.of(QFieldType.INTEGER, QFieldType.LONG, QFieldType.DECIMAL).contains(field.getType())); + } + + /******************************************************************************* + ** Reject strings and categorical identifiers before computing relationships. + *******************************************************************************/ + protected void requireNumeric(String name) throws QException + { + if(!numeric(full(name))) + { + throw new QBadRequestException("Choose numeric analysis fields."); + } + } + + /******************************************************************************* + ** Typed aggregate keys must match keys returned by all backend implementations. + *******************************************************************************/ + protected Aggregate measure(String field, AggregateOperator operator) + { + Aggregate aggregate = new Aggregate(field, operator); + return (aggregate.withFieldType(com.kingsrook.qqq.backend.core.utils.aggregates.AggregateAnalysis.resultType(aggregate, field(table, field).getType()))); + } + + /******************************************************************************* + ** Scalar summaries have one result; empty populations have null numeric values. + *******************************************************************************/ + protected AggregateResult summarize(QQueryFilter filter, List measures) throws QException + { + AggregateInput input = new AggregateInput(table).withFilter(DrillFilters.copy(filter)); + measures.forEach(input::withAggregate); + List results = aggregate(input); + return (results.isEmpty() ? new AggregateResult() : results.getFirst()); + } + + /******************************************************************************* + ** Exact row count uses the core count action with the caller's record locks. + *******************************************************************************/ + protected long size(QQueryFilter filter) throws QException + { + return (count(table, filter)); + } + + /******************************************************************************* + ** Count and scalar drills preserve the caller's original OR predicates. + *******************************************************************************/ + protected Metric metric(String key, Serializable value, ValueType type, QQueryFilter filter, QQueryFilter condition) + { + return (ProfileSupport.metric(key.startsWith("category") ? type == ValueType.COUNT ? "count" : "share" : key, value, type).withKey(key).withDrill(DrillFilters.drill(table, filter, condition))); + } + + /******************************************************************************* + ** Apply the stricter cell guard to remove both values and membership drills. + *******************************************************************************/ + protected void guard(List metrics, long n) + { + new MinGroupSizeGuard(config.getMinGroupSize()).pivotMetrics(metrics, n); + } + + /******************************************************************************* + ** Group identities are typed; null and empty string remain distinct. + *******************************************************************************/ + protected List categories(String field, QQueryFilter filter, int limit) throws QException + { + QFieldMetaData metadata = full(field); + GroupBy group = new GroupBy(metadata); + Aggregate count = measure(primaryKey(table), AggregateOperator.COUNT); + List groups = new ArrayList<>(aggregate(new AggregateInput(table).withFilter(DrillFilters.copy(filter)).withGroupBy(group).withAggregate(count))); + groups.sort(Comparator.comparingLong(r -> ((Number) r.getAggregateValues().get(count)).longValue()).reversed() + .thenComparing(r -> String.valueOf(r.getGroupByValues().get(group)))); + List result = new ArrayList<>(); + List other = new ArrayList<>(); + boolean privacy = false; + for(AggregateResult row : groups) + { + Serializable value = row.getGroupByValues().get(group); + long n = ((Number) row.getAggregateValues().get(count)).longValue(); + boolean hidden = n < config.getMinGroupSize(); + Category category = new Category(value, hidden ? "Other" : value == null ? "(blank)" : security.label(metadata, value), + DrillFilters.value(field, value, metadata.getType() == QFieldType.STRING), hidden); + if(hidden || result.size() >= limit) + { + other.add(category); + privacy |= hidden; + } + else + { + result.add(category); + } + } + if(!other.isEmpty()) + { + result.add(merge(other, privacy)); + } + return (result); + } + + /******************************************************************************* + ** An explicit union avoids null-sensitive NOT IN semantics for Other. + *******************************************************************************/ + protected Category merge(List categories, boolean privacy) + { + QQueryFilter condition = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR); + categories.forEach(c -> condition.withSubFilter(c.condition())); + return (new Category(null, "Other", condition, privacy)); + } + + /******************************************************************************* + ** Keep the math package free of QQQ and JSON dependencies. + *******************************************************************************/ + protected Significance significance(Statistics.Result result) + { + return (new Significance().withTest(result.test()).withStatistic(result.statistic()).withDf(result.df()).withPValue(result.pValue()) + .withApplicable(result.applicable()).withSentence(result.sentence()).withCramersV(result.cramersV())); + } + + /******************************************************************************* + ** Scalar conversion preserves null instead of leaking non-finite JSON numbers. + *******************************************************************************/ + protected static BigDecimal decimal(Object value) + { + return (value == null ? null : new BigDecimal(value.toString())); + } + + /******************************************************************************* + ** Shared chart shape with a deliberate grouping/stacking annotation. + *******************************************************************************/ + protected Chart chart(ChartKind kind, List series, String annotation) + { + return (new Chart().withKind(kind).withSeries(series).withAnnotations(List.of(annotation))); + } + + /******************************************************************************* + ** Correlation and regression exclude rows missing either numeric coordinate. + *******************************************************************************/ + protected QQueryFilter paired(QQueryFilter filter, String x, String y) + { + return (DrillFilters.and(filter, DrillFilters.condition(x, QCriteriaOperator.IS_NOT_BLANK).withCriteria(y, QCriteriaOperator.IS_NOT_BLANK))); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ShiftExplorer.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ShiftExplorer.java new file mode 100644 index 0000000..7f57b6a --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ShiftExplorer.java @@ -0,0 +1,213 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.io.Serializable; +import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.dataanalysis.result.ChartKind; +import com.kingsrook.qbits.dataanalysis.result.Metric; +import com.kingsrook.qbits.dataanalysis.result.Point; +import com.kingsrook.qbits.dataanalysis.result.Series; +import com.kingsrook.qbits.dataanalysis.result.ShiftBucket; +import com.kingsrook.qbits.dataanalysis.result.ShiftRequest; +import com.kingsrook.qbits.dataanalysis.result.ShiftResult; +import com.kingsrook.qbits.dataanalysis.result.ValueType; +import com.kingsrook.qbits.dataanalysis.stats.Statistics; +import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.DateBucket; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.DateBucketUnit; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + +/******************************************************************************* + ** Distribution drift with stable categories or fixed baseline quartile bins. + *******************************************************************************/ +public class ShiftExplorer extends ComparisonExplorer +{ + /******************************************************************************* + ** Bind the same core and privacy policy as compare. + *******************************************************************************/ + public ShiftExplorer(AnalysisSecurity security, EffectiveConfig config, String table) + { + super(security, config, table); + } + + /******************************************************************************* + ** One calendar bucket with exact boundaries and whether its end has passed. + *******************************************************************************/ + private record Bucket(Serializable start, QQueryFilter condition, boolean complete) + { + } + + /******************************************************************************* + ** First and last complete nonempty buckets define the descriptive PSI. + *******************************************************************************/ + public ShiftResult shift(ShiftRequest request) throws QException + { + var field = full(request.getField()); + var date = full(request.getDateField()); + DateBucketUnit unit; + try + { + unit = DateBucketUnit.valueOf(request.getBucket()); + } + catch(RuntimeException e) + { + throw new QBadRequestException("Choose a calendar bucket."); + } + if(!List.of(QFieldType.DATE, QFieldType.DATE_TIME).contains(date.getType()) || unit == DateBucketUnit.DAY_OF_WEEK + || unit == DateBucketUnit.HOUR_OF_DAY || unit == DateBucketUnit.HOUR && date.getType() != QFieldType.DATE_TIME) + { + throw new QBadRequestException("Choose a date field and a calendar bucket from HOUR to YEAR."); + } + GroupBy group = new GroupBy(date).withBucket(new DateBucket(unit)); + var aggregate = measure(primaryKey(table), AggregateOperator.COUNT); + var grouped = aggregate(new AggregateInput(table).withFilter(DrillFilters.copy(request.getFilter())).withGroupBy(group).withAggregate(aggregate)); + List buckets = grouped.stream().map(r -> r.getGroupByValues().get(group)).filter(java.util.Objects::nonNull) + .sorted(Comparator.comparing(Object::toString)).map(start -> bucket(request.getDateField(), start, unit, date.getType())).toList(); + if(buckets.size() > 2000) + { + throw new QBadRequestException("Choose a larger time bucket; at most 2000 buckets are supported."); + } + ShiftResult result = new ShiftResult(); + List series = new ArrayList<>(); + List complete = new ArrayList<>(); + boolean numeric = numeric(field); + List categories = numeric ? List.of() : categories(request.getField(), request.getFilter(), 8); + Map baseline = null; + boolean privateData = false; + if(numeric) + { + for(String name : List.of("p25", "median", "p75")) + { + series.add(new Series().withName(name)); + } + } + else + { + for(Category category : categories) + { + series.add(new Series().withName(category.label())); + } + } + for(Bucket bucket : buckets) + { + QQueryFilter filter = DrillFilters.and(request.getFilter(), bucket.condition()); + long n = size(filter); + ShiftBucket output = new ShiftBucket().withValue(bucket.start()).withLabel(bucket.start().toString()).withComplete(bucket.complete()); + double[] counts; + if(numeric) + { + Map moments = moments(request.getField(), filter); + long nonNull = moments.get("count").longValue(); + if(baseline == null && bucket.complete()) + { + baseline = moments; + } + output.withMetrics(numericSide(request.getField(), output.getLabel(), filter, moments).getMetrics()); + counts = baseline == null || baseline.get("count").signum() == 0 ? new double[0] : numericCounts(request.getField(), filter, baseline); + privateData |= nonNull < config.getMinGroupSize(); + for(int i = 0; i < series.size(); i++) + { + String key = series.get(i).getName(); + Metric metric = output.getMetrics().stream().filter(m -> m.getKey().equals(key)).findFirst().orElseThrow(); + series.get(i).getPoints().add(new Point().withX(bucket.start()).withY(decimal(metric.getValue())).withDrill(metric.getDrill())); + } + } + else + { + counts = new double[categories.size()]; + for(int i = 0; i < categories.size(); i++) + { + Category category = categories.get(i); + long count = size(DrillFilters.and(filter, category.condition())); + counts[i] = count; + Metric countMetric = metric("category" + i + "Count", count, ValueType.COUNT, filter, category.condition()).withLabel(category.label()); + Metric share = metric("category" + i + "Share", ProfileSupport.share(count, n), ValueType.PERCENT, filter, category.condition()).withLabel(category.label()); + List metrics = List.of(countMetric, share); + guard(metrics, count); + if(category.privateGroup()) + { + metrics.forEach(m -> m.withDrill(null)); + } + privateData |= category.privateGroup() || count < config.getMinGroupSize(); + output.getMetrics().addAll(metrics); + series.get(i).getPoints().add(new Point().withX(bucket.start()).withY(decimal(share.getValue())).withLabel(category.label()).withDrill(share.getDrill())); + } + } + result.getBuckets().add(output); + if(bucket.complete()) + { + complete.add(counts); + } + } + Double psi = !privateData && complete.size() >= 2 && java.util.Arrays.stream(complete.getFirst()).sum() >= 2 + && java.util.Arrays.stream(complete.getLast()).sum() >= 2 ? Statistics.psi(complete.getFirst(), complete.getLast()) : null; + result.withPsi(ProfileSupport.metric("psi", psi, ValueType.DECIMAL)); + result.withSentence(psi == null ? "Not applicable: " + (privateData ? "privacy guard" : "two complete buckets with at least two values are required") + : "The first and last complete buckets have a population stability index of " + BigDecimal.valueOf(psi).setScale(3, java.math.RoundingMode.HALF_UP) + "."); + return (result.withChart(chart(ChartKind.TIME_SERIES, series, numeric ? "Numeric quartiles; PSI uses fixed first-complete-bucket quartile boundaries." + : "Stacked category shares. PSI floors zero shares at 0.000001; it is not a significance test.").withStacked(!numeric))); + } + + /******************************************************************************* + ** Calendar boundaries use the session zone, including daylight-saving days. + *******************************************************************************/ + private Bucket bucket(String field, Serializable value, DateBucketUnit unit, QFieldType type) + { + ZoneId zone = ValueUtils.getSessionOrInstanceZoneId(); + if(unit == DateBucketUnit.HOUR) + { + Instant start = ValueUtils.getValueAsInstant(value); + Instant end = start.plusSeconds(3600); + return (new Bucket(start, DrillFilters.range(field, start, end, false), !end.isAfter(Instant.now()))); + } + LocalDate start = ValueUtils.getValueAsLocalDate(value); + LocalDate end = next(start, DateUnit.valueOf(unit.name())); + return (new Bucket(start, DrillFilters.dateRange(field, start, end, type == QFieldType.DATE_TIME, zone), !end.atStartOfDay(zone).toInstant().isAfter(Instant.now()))); + } + + /******************************************************************************* + ** Both populations use identical, half-open quartile ranges with infinite tails. + *******************************************************************************/ + private double[] numericCounts(String field, QQueryFilter filter, Map baseline) throws QException + { + BigDecimal[] edges = {null, baseline.get("p25"), baseline.get("p50"), baseline.get("p75"), null}; + double[] counts = new double[4]; + for(int i = 0; i < 4; i++) + { + counts[i] = size(DrillFilters.and(filter, DrillFilters.range(field, edges[i], edges[i + 1], false))); + } + return (counts); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/Chart.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/Chart.java index e0dee58..3220a6a 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/result/Chart.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/Chart.java @@ -33,6 +33,7 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; public class Chart implements Serializable { private ChartKind kind; + private Boolean stacked; private List alternatives = List.of(); private Axis x; private Axis y; @@ -164,4 +165,21 @@ public class Chart implements Serializable this.annotations = annotations; return (this); } + + /******************************************************************************* + ** Whether category series should stack to a total share of one. + *******************************************************************************/ + public Boolean getStacked() + { + return (stacked); + } + + /******************************************************************************* + ** Declare stacked shares explicitly rather than relying on chart prose. + *******************************************************************************/ + public Chart withStacked(Boolean stacked) + { + this.stacked = stacked; + return (this); + } } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/CompareRequest.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/CompareRequest.java new file mode 100644 index 0000000..29775a2 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/CompareRequest.java @@ -0,0 +1,213 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + +import java.io.Serializable; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; + +/******************************************************************************* + ** Typed CompareRequest contract for comparison and relationship analysis. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE) +public class CompareRequest implements AnalysisRequest +{ + private String tableName; + private QQueryFilter filter; + private Boolean exact = true; + private QQueryFilter filterA; + private QQueryFilter filterB; + private String labelA = "A"; + private String labelB = "B"; + private List fields; + + + + /******************************************************************************* + ** Getter for tableName. + *******************************************************************************/ + @Override + public String getTableName() + { + return (tableName); + } + + + + /******************************************************************************* + ** Fluent setter for tableName. + *******************************************************************************/ + public CompareRequest withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for filter. + *******************************************************************************/ + @Override + public QQueryFilter getFilter() + { + return (filter); + } + + + + /******************************************************************************* + ** Fluent setter for filter. + *******************************************************************************/ + public CompareRequest withFilter(QQueryFilter filter) + { + this.filter = filter; + return (this); + } + + + + /******************************************************************************* + ** Getter for exact. + *******************************************************************************/ + @Override + public Boolean getExact() + { + return (exact); + } + + + + /******************************************************************************* + ** Fluent setter for exact. + *******************************************************************************/ + public CompareRequest withExact(Boolean exact) + { + this.exact = exact; + return (this); + } + + + + /******************************************************************************* + ** Getter for filterA. + *******************************************************************************/ + public QQueryFilter getFilterA() + { + return (filterA); + } + + + + /******************************************************************************* + ** Fluent setter for filterA. + *******************************************************************************/ + public CompareRequest withFilterA(QQueryFilter filterA) + { + this.filterA = filterA; + return (this); + } + + + + /******************************************************************************* + ** Getter for filterB. + *******************************************************************************/ + public QQueryFilter getFilterB() + { + return (filterB); + } + + + + /******************************************************************************* + ** Fluent setter for filterB. + *******************************************************************************/ + public CompareRequest withFilterB(QQueryFilter filterB) + { + this.filterB = filterB; + return (this); + } + + + + /******************************************************************************* + ** Getter for labelA. + *******************************************************************************/ + public String getLabelA() + { + return (labelA); + } + + + + /******************************************************************************* + ** Fluent setter for labelA. + *******************************************************************************/ + public CompareRequest withLabelA(String labelA) + { + this.labelA = labelA; + return (this); + } + + + + /******************************************************************************* + ** Getter for labelB. + *******************************************************************************/ + public String getLabelB() + { + return (labelB); + } + + + + /******************************************************************************* + ** Fluent setter for labelB. + *******************************************************************************/ + public CompareRequest withLabelB(String labelB) + { + this.labelB = labelB; + return (this); + } + + + + /******************************************************************************* + ** Getter for fields. + *******************************************************************************/ + public List getFields() + { + return (fields); + } + + + + /******************************************************************************* + ** Fluent setter for fields. + *******************************************************************************/ + public CompareRequest withFields(List fields) + { + this.fields = fields; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/CompareResult.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/CompareResult.java new file mode 100644 index 0000000..b816b2c --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/CompareResult.java @@ -0,0 +1,55 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + +import java.io.Serializable; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +/******************************************************************************* + ** Typed CompareResult contract for comparison and relationship analysis. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE) +public class CompareResult implements Serializable +{ + private List fields = new java.util.ArrayList<>(); + + + + /******************************************************************************* + ** Getter for fields. + *******************************************************************************/ + public List getFields() + { + return (fields); + } + + + + /******************************************************************************* + ** Fluent setter for fields. + *******************************************************************************/ + public CompareResult withFields(List fields) + { + this.fields = fields; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/ComparisonSide.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/ComparisonSide.java new file mode 100644 index 0000000..83b0a56 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/ComparisonSide.java @@ -0,0 +1,77 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + +import java.io.Serializable; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +/******************************************************************************* + ** Typed ComparisonSide contract for comparison and relationship analysis. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE) +public class ComparisonSide implements Serializable +{ + private String label; + private List metrics = new java.util.ArrayList<>(); + + + + /******************************************************************************* + ** Getter for label. + *******************************************************************************/ + public String getLabel() + { + return (label); + } + + + + /******************************************************************************* + ** Fluent setter for label. + *******************************************************************************/ + public ComparisonSide withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for metrics. + *******************************************************************************/ + public List getMetrics() + { + return (metrics); + } + + + + /******************************************************************************* + ** Fluent setter for metrics. + *******************************************************************************/ + public ComparisonSide withMetrics(List metrics) + { + this.metrics = metrics; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/ComparisonValue.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/ComparisonValue.java new file mode 100644 index 0000000..79916ee --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/ComparisonValue.java @@ -0,0 +1,186 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + +import java.io.Serializable; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +/******************************************************************************* + ** Typed ComparisonValue contract for comparison and relationship analysis. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE) +public class ComparisonValue implements Serializable +{ + private Serializable value; + private String label; + private ComparisonSide sideA; + private ComparisonSide sideB; + private Metric delta; + private Metric percentDelta; + private Significance significance; + + + + /******************************************************************************* + ** Getter for value. + *******************************************************************************/ + public Serializable getValue() + { + return (value); + } + + + + /******************************************************************************* + ** Fluent setter for value. + *******************************************************************************/ + public ComparisonValue withValue(Serializable value) + { + this.value = value; + return (this); + } + + + + /******************************************************************************* + ** Getter for label. + *******************************************************************************/ + public String getLabel() + { + return (label); + } + + + + /******************************************************************************* + ** Fluent setter for label. + *******************************************************************************/ + public ComparisonValue withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for sideA. + *******************************************************************************/ + public ComparisonSide getSideA() + { + return (sideA); + } + + + + /******************************************************************************* + ** Fluent setter for sideA. + *******************************************************************************/ + public ComparisonValue withSideA(ComparisonSide sideA) + { + this.sideA = sideA; + return (this); + } + + + + /******************************************************************************* + ** Getter for sideB. + *******************************************************************************/ + public ComparisonSide getSideB() + { + return (sideB); + } + + + + /******************************************************************************* + ** Fluent setter for sideB. + *******************************************************************************/ + public ComparisonValue withSideB(ComparisonSide sideB) + { + this.sideB = sideB; + return (this); + } + + + + /******************************************************************************* + ** Getter for delta. + *******************************************************************************/ + public Metric getDelta() + { + return (delta); + } + + + + /******************************************************************************* + ** Fluent setter for delta. + *******************************************************************************/ + public ComparisonValue withDelta(Metric delta) + { + this.delta = delta; + return (this); + } + + + + /******************************************************************************* + ** Getter for percentDelta. + *******************************************************************************/ + public Metric getPercentDelta() + { + return (percentDelta); + } + + + + /******************************************************************************* + ** Fluent setter for percentDelta. + *******************************************************************************/ + public ComparisonValue withPercentDelta(Metric percentDelta) + { + this.percentDelta = percentDelta; + return (this); + } + + + + /******************************************************************************* + ** Getter for significance. + *******************************************************************************/ + public Significance getSignificance() + { + return (significance); + } + + + + /******************************************************************************* + ** Fluent setter for significance. + *******************************************************************************/ + public ComparisonValue withSignificance(Significance significance) + { + this.significance = significance; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/CorrelationCell.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/CorrelationCell.java new file mode 100644 index 0000000..5bd965d --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/CorrelationCell.java @@ -0,0 +1,142 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + +import java.io.Serializable; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +/******************************************************************************* + ** Typed CorrelationCell contract for comparison and relationship analysis. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE) +public class CorrelationCell implements Serializable +{ + private String xField; + private String yField; + private Long n; + private Double r; + private ScatterRequest scatter; + + + + /******************************************************************************* + ** Getter for xField. + *******************************************************************************/ + public String getXField() + { + return (xField); + } + + + + /******************************************************************************* + ** Fluent setter for xField. + *******************************************************************************/ + public CorrelationCell withXField(String xField) + { + this.xField = xField; + return (this); + } + + + + /******************************************************************************* + ** Getter for yField. + *******************************************************************************/ + public String getYField() + { + return (yField); + } + + + + /******************************************************************************* + ** Fluent setter for yField. + *******************************************************************************/ + public CorrelationCell withYField(String yField) + { + this.yField = yField; + return (this); + } + + + + /******************************************************************************* + ** Getter for n. + *******************************************************************************/ + public Long getN() + { + return (n); + } + + + + /******************************************************************************* + ** Fluent setter for n. + *******************************************************************************/ + public CorrelationCell withN(Long n) + { + this.n = n; + return (this); + } + + + + /******************************************************************************* + ** Getter for r. + *******************************************************************************/ + public Double getR() + { + return (r); + } + + + + /******************************************************************************* + ** Fluent setter for r. + *******************************************************************************/ + public CorrelationCell withR(Double r) + { + this.r = r; + return (this); + } + + + + /******************************************************************************* + ** Getter for scatter. + *******************************************************************************/ + public ScatterRequest getScatter() + { + return (scatter); + } + + + + /******************************************************************************* + ** Fluent setter for scatter. + *******************************************************************************/ + public CorrelationCell withScatter(ScatterRequest scatter) + { + this.scatter = scatter; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/CorrelationRequest.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/CorrelationRequest.java new file mode 100644 index 0000000..e8d9320 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/CorrelationRequest.java @@ -0,0 +1,125 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + +import java.io.Serializable; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; + +/******************************************************************************* + ** Typed CorrelationRequest contract for comparison and relationship analysis. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE) +public class CorrelationRequest implements AnalysisRequest +{ + private String tableName; + private QQueryFilter filter; + private Boolean exact = true; + private List fields; + + + + /******************************************************************************* + ** Getter for tableName. + *******************************************************************************/ + @Override + public String getTableName() + { + return (tableName); + } + + + + /******************************************************************************* + ** Fluent setter for tableName. + *******************************************************************************/ + public CorrelationRequest withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for filter. + *******************************************************************************/ + @Override + public QQueryFilter getFilter() + { + return (filter); + } + + + + /******************************************************************************* + ** Fluent setter for filter. + *******************************************************************************/ + public CorrelationRequest withFilter(QQueryFilter filter) + { + this.filter = filter; + return (this); + } + + + + /******************************************************************************* + ** Getter for exact. + *******************************************************************************/ + @Override + public Boolean getExact() + { + return (exact); + } + + + + /******************************************************************************* + ** Fluent setter for exact. + *******************************************************************************/ + public CorrelationRequest withExact(Boolean exact) + { + this.exact = exact; + return (this); + } + + + + /******************************************************************************* + ** Getter for fields. + *******************************************************************************/ + public List getFields() + { + return (fields); + } + + + + /******************************************************************************* + ** Fluent setter for fields. + *******************************************************************************/ + public CorrelationRequest withFields(List fields) + { + this.fields = fields; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/CorrelationResult.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/CorrelationResult.java new file mode 100644 index 0000000..35e3ea0 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/CorrelationResult.java @@ -0,0 +1,99 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + +import java.io.Serializable; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +/******************************************************************************* + ** Typed CorrelationResult contract for comparison and relationship analysis. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE) +public class CorrelationResult implements Serializable +{ + private List fields; + private List cells = new java.util.ArrayList<>(); + private Chart chart; + + + + /******************************************************************************* + ** Getter for fields. + *******************************************************************************/ + public List getFields() + { + return (fields); + } + + + + /******************************************************************************* + ** Fluent setter for fields. + *******************************************************************************/ + public CorrelationResult withFields(List fields) + { + this.fields = fields; + return (this); + } + + + + /******************************************************************************* + ** Getter for cells. + *******************************************************************************/ + public List getCells() + { + return (cells); + } + + + + /******************************************************************************* + ** Fluent setter for cells. + *******************************************************************************/ + public CorrelationResult withCells(List cells) + { + this.cells = cells; + return (this); + } + + + + /******************************************************************************* + ** Getter for chart. + *******************************************************************************/ + public Chart getChart() + { + return (chart); + } + + + + /******************************************************************************* + ** Fluent setter for chart. + *******************************************************************************/ + public CorrelationResult withChart(Chart chart) + { + this.chart = chart; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/CrosstabCell.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/CrosstabCell.java new file mode 100644 index 0000000..1344944 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/CrosstabCell.java @@ -0,0 +1,142 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + +import java.io.Serializable; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +/******************************************************************************* + ** Typed CrosstabCell contract for comparison and relationship analysis. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE) +public class CrosstabCell implements Serializable +{ + private Integer rowIndex; + private Integer columnIndex; + private Metric count; + private Metric rowShare; + private Metric columnShare; + + + + /******************************************************************************* + ** Getter for rowIndex. + *******************************************************************************/ + public Integer getRowIndex() + { + return (rowIndex); + } + + + + /******************************************************************************* + ** Fluent setter for rowIndex. + *******************************************************************************/ + public CrosstabCell withRowIndex(Integer rowIndex) + { + this.rowIndex = rowIndex; + return (this); + } + + + + /******************************************************************************* + ** Getter for columnIndex. + *******************************************************************************/ + public Integer getColumnIndex() + { + return (columnIndex); + } + + + + /******************************************************************************* + ** Fluent setter for columnIndex. + *******************************************************************************/ + public CrosstabCell withColumnIndex(Integer columnIndex) + { + this.columnIndex = columnIndex; + return (this); + } + + + + /******************************************************************************* + ** Getter for count. + *******************************************************************************/ + public Metric getCount() + { + return (count); + } + + + + /******************************************************************************* + ** Fluent setter for count. + *******************************************************************************/ + public CrosstabCell withCount(Metric count) + { + this.count = count; + return (this); + } + + + + /******************************************************************************* + ** Getter for rowShare. + *******************************************************************************/ + public Metric getRowShare() + { + return (rowShare); + } + + + + /******************************************************************************* + ** Fluent setter for rowShare. + *******************************************************************************/ + public CrosstabCell withRowShare(Metric rowShare) + { + this.rowShare = rowShare; + return (this); + } + + + + /******************************************************************************* + ** Getter for columnShare. + *******************************************************************************/ + public Metric getColumnShare() + { + return (columnShare); + } + + + + /******************************************************************************* + ** Fluent setter for columnShare. + *******************************************************************************/ + public CrosstabCell withColumnShare(Metric columnShare) + { + this.columnShare = columnShare; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/CrosstabRequest.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/CrosstabRequest.java new file mode 100644 index 0000000..1d0ec3d --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/CrosstabRequest.java @@ -0,0 +1,168 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + +import java.io.Serializable; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; + +/******************************************************************************* + ** Typed CrosstabRequest contract for comparison and relationship analysis. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE) +public class CrosstabRequest implements AnalysisRequest +{ + private String tableName; + private QQueryFilter filter; + private Boolean exact = true; + private String rowField; + private String columnField; + private Integer limit = 20; + + + + /******************************************************************************* + ** Getter for tableName. + *******************************************************************************/ + @Override + public String getTableName() + { + return (tableName); + } + + + + /******************************************************************************* + ** Fluent setter for tableName. + *******************************************************************************/ + public CrosstabRequest withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for filter. + *******************************************************************************/ + @Override + public QQueryFilter getFilter() + { + return (filter); + } + + + + /******************************************************************************* + ** Fluent setter for filter. + *******************************************************************************/ + public CrosstabRequest withFilter(QQueryFilter filter) + { + this.filter = filter; + return (this); + } + + + + /******************************************************************************* + ** Getter for exact. + *******************************************************************************/ + @Override + public Boolean getExact() + { + return (exact); + } + + + + /******************************************************************************* + ** Fluent setter for exact. + *******************************************************************************/ + public CrosstabRequest withExact(Boolean exact) + { + this.exact = exact; + return (this); + } + + + + /******************************************************************************* + ** Getter for rowField. + *******************************************************************************/ + public String getRowField() + { + return (rowField); + } + + + + /******************************************************************************* + ** Fluent setter for rowField. + *******************************************************************************/ + public CrosstabRequest withRowField(String rowField) + { + this.rowField = rowField; + return (this); + } + + + + /******************************************************************************* + ** Getter for columnField. + *******************************************************************************/ + public String getColumnField() + { + return (columnField); + } + + + + /******************************************************************************* + ** Fluent setter for columnField. + *******************************************************************************/ + public CrosstabRequest withColumnField(String columnField) + { + this.columnField = columnField; + return (this); + } + + + + /******************************************************************************* + ** Getter for limit. + *******************************************************************************/ + public Integer getLimit() + { + return (limit); + } + + + + /******************************************************************************* + ** Fluent setter for limit. + *******************************************************************************/ + public CrosstabRequest withLimit(Integer limit) + { + this.limit = limit; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/CrosstabResult.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/CrosstabResult.java new file mode 100644 index 0000000..85826a4 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/CrosstabResult.java @@ -0,0 +1,121 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + +import java.io.Serializable; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +/******************************************************************************* + ** Typed CrosstabResult contract for comparison and relationship analysis. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE) +public class CrosstabResult implements Serializable +{ + private List rows = new java.util.ArrayList<>(); + private List columns = new java.util.ArrayList<>(); + private List cells = new java.util.ArrayList<>(); + private Significance significance; + + + + /******************************************************************************* + ** Getter for rows. + *******************************************************************************/ + public List getRows() + { + return (rows); + } + + + + /******************************************************************************* + ** Fluent setter for rows. + *******************************************************************************/ + public CrosstabResult withRows(List rows) + { + this.rows = rows; + return (this); + } + + + + /******************************************************************************* + ** Getter for columns. + *******************************************************************************/ + public List getColumns() + { + return (columns); + } + + + + /******************************************************************************* + ** Fluent setter for columns. + *******************************************************************************/ + public CrosstabResult withColumns(List columns) + { + this.columns = columns; + return (this); + } + + + + /******************************************************************************* + ** Getter for cells. + *******************************************************************************/ + public List getCells() + { + return (cells); + } + + + + /******************************************************************************* + ** Fluent setter for cells. + *******************************************************************************/ + public CrosstabResult withCells(List cells) + { + this.cells = cells; + return (this); + } + + + + /******************************************************************************* + ** Getter for significance. + *******************************************************************************/ + public Significance getSignificance() + { + return (significance); + } + + + + /******************************************************************************* + ** Fluent setter for significance. + *******************************************************************************/ + public CrosstabResult withSignificance(Significance significance) + { + this.significance = significance; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/FieldComparison.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/FieldComparison.java new file mode 100644 index 0000000..691e308 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/FieldComparison.java @@ -0,0 +1,121 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + +import java.io.Serializable; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +/******************************************************************************* + ** Typed FieldComparison contract for comparison and relationship analysis. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE) +public class FieldComparison implements Serializable +{ + private String field; + private List values = new java.util.ArrayList<>(); + private Significance significance; + private Chart chart; + + + + /******************************************************************************* + ** Getter for field. + *******************************************************************************/ + public String getField() + { + return (field); + } + + + + /******************************************************************************* + ** Fluent setter for field. + *******************************************************************************/ + public FieldComparison withField(String field) + { + this.field = field; + return (this); + } + + + + /******************************************************************************* + ** Getter for values. + *******************************************************************************/ + public List getValues() + { + return (values); + } + + + + /******************************************************************************* + ** Fluent setter for values. + *******************************************************************************/ + public FieldComparison withValues(List values) + { + this.values = values; + return (this); + } + + + + /******************************************************************************* + ** Getter for significance. + *******************************************************************************/ + public Significance getSignificance() + { + return (significance); + } + + + + /******************************************************************************* + ** Fluent setter for significance. + *******************************************************************************/ + public FieldComparison withSignificance(Significance significance) + { + this.significance = significance; + return (this); + } + + + + /******************************************************************************* + ** Getter for chart. + *******************************************************************************/ + public Chart getChart() + { + return (chart); + } + + + + /******************************************************************************* + ** Fluent setter for chart. + *******************************************************************************/ + public FieldComparison withChart(Chart chart) + { + this.chart = chart; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/ScatterRequest.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/ScatterRequest.java new file mode 100644 index 0000000..6c772e3 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/ScatterRequest.java @@ -0,0 +1,168 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + +import java.io.Serializable; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; + +/******************************************************************************* + ** Typed ScatterRequest contract for comparison and relationship analysis. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE) +public class ScatterRequest implements AnalysisRequest +{ + private String tableName; + private QQueryFilter filter; + private Boolean exact = true; + private String xField; + private String yField; + private Integer limit = 2000; + + + + /******************************************************************************* + ** Getter for tableName. + *******************************************************************************/ + @Override + public String getTableName() + { + return (tableName); + } + + + + /******************************************************************************* + ** Fluent setter for tableName. + *******************************************************************************/ + public ScatterRequest withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for filter. + *******************************************************************************/ + @Override + public QQueryFilter getFilter() + { + return (filter); + } + + + + /******************************************************************************* + ** Fluent setter for filter. + *******************************************************************************/ + public ScatterRequest withFilter(QQueryFilter filter) + { + this.filter = filter; + return (this); + } + + + + /******************************************************************************* + ** Getter for exact. + *******************************************************************************/ + @Override + public Boolean getExact() + { + return (exact); + } + + + + /******************************************************************************* + ** Fluent setter for exact. + *******************************************************************************/ + public ScatterRequest withExact(Boolean exact) + { + this.exact = exact; + return (this); + } + + + + /******************************************************************************* + ** Getter for xField. + *******************************************************************************/ + public String getXField() + { + return (xField); + } + + + + /******************************************************************************* + ** Fluent setter for xField. + *******************************************************************************/ + public ScatterRequest withXField(String xField) + { + this.xField = xField; + return (this); + } + + + + /******************************************************************************* + ** Getter for yField. + *******************************************************************************/ + public String getYField() + { + return (yField); + } + + + + /******************************************************************************* + ** Fluent setter for yField. + *******************************************************************************/ + public ScatterRequest withYField(String yField) + { + this.yField = yField; + return (this); + } + + + + /******************************************************************************* + ** Getter for limit. + *******************************************************************************/ + public Integer getLimit() + { + return (limit); + } + + + + /******************************************************************************* + ** Fluent setter for limit. + *******************************************************************************/ + public ScatterRequest withLimit(Integer limit) + { + this.limit = limit; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/ScatterResult.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/ScatterResult.java new file mode 100644 index 0000000..a69f625 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/ScatterResult.java @@ -0,0 +1,165 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + +import java.io.Serializable; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +/******************************************************************************* + ** Typed ScatterResult contract for comparison and relationship analysis. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE) +public class ScatterResult implements Serializable +{ + private Long n; + private Integer sampleSize; + private String sampling; + private List points = new java.util.ArrayList<>(); + private TrendLine trend; + private Chart chart; + + + + /******************************************************************************* + ** Getter for n. + *******************************************************************************/ + public Long getN() + { + return (n); + } + + + + /******************************************************************************* + ** Fluent setter for n. + *******************************************************************************/ + public ScatterResult withN(Long n) + { + this.n = n; + return (this); + } + + + + /******************************************************************************* + ** Getter for sampleSize. + *******************************************************************************/ + public Integer getSampleSize() + { + return (sampleSize); + } + + + + /******************************************************************************* + ** Fluent setter for sampleSize. + *******************************************************************************/ + public ScatterResult withSampleSize(Integer sampleSize) + { + this.sampleSize = sampleSize; + return (this); + } + + + + /******************************************************************************* + ** Getter for sampling. + *******************************************************************************/ + public String getSampling() + { + return (sampling); + } + + + + /******************************************************************************* + ** Fluent setter for sampling. + *******************************************************************************/ + public ScatterResult withSampling(String sampling) + { + this.sampling = sampling; + return (this); + } + + + + /******************************************************************************* + ** Getter for points. + *******************************************************************************/ + public List getPoints() + { + return (points); + } + + + + /******************************************************************************* + ** Fluent setter for points. + *******************************************************************************/ + public ScatterResult withPoints(List points) + { + this.points = points; + return (this); + } + + + + /******************************************************************************* + ** Getter for trend. + *******************************************************************************/ + public TrendLine getTrend() + { + return (trend); + } + + + + /******************************************************************************* + ** Fluent setter for trend. + *******************************************************************************/ + public ScatterResult withTrend(TrendLine trend) + { + this.trend = trend; + return (this); + } + + + + /******************************************************************************* + ** Getter for chart. + *******************************************************************************/ + public Chart getChart() + { + return (chart); + } + + + + /******************************************************************************* + ** Fluent setter for chart. + *******************************************************************************/ + public ScatterResult withChart(Chart chart) + { + this.chart = chart; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/ShiftBucket.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/ShiftBucket.java new file mode 100644 index 0000000..5a3e348 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/ShiftBucket.java @@ -0,0 +1,121 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + +import java.io.Serializable; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +/******************************************************************************* + ** Typed ShiftBucket contract for comparison and relationship analysis. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE) +public class ShiftBucket implements Serializable +{ + private Serializable value; + private String label; + private Boolean complete = false; + private List metrics = new java.util.ArrayList<>(); + + + + /******************************************************************************* + ** Getter for value. + *******************************************************************************/ + public Serializable getValue() + { + return (value); + } + + + + /******************************************************************************* + ** Fluent setter for value. + *******************************************************************************/ + public ShiftBucket withValue(Serializable value) + { + this.value = value; + return (this); + } + + + + /******************************************************************************* + ** Getter for label. + *******************************************************************************/ + public String getLabel() + { + return (label); + } + + + + /******************************************************************************* + ** Fluent setter for label. + *******************************************************************************/ + public ShiftBucket withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for complete. + *******************************************************************************/ + public Boolean getComplete() + { + return (complete); + } + + + + /******************************************************************************* + ** Fluent setter for complete. + *******************************************************************************/ + public ShiftBucket withComplete(Boolean complete) + { + this.complete = complete; + return (this); + } + + + + /******************************************************************************* + ** Getter for metrics. + *******************************************************************************/ + public List getMetrics() + { + return (metrics); + } + + + + /******************************************************************************* + ** Fluent setter for metrics. + *******************************************************************************/ + public ShiftBucket withMetrics(List metrics) + { + this.metrics = metrics; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/ShiftRequest.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/ShiftRequest.java new file mode 100644 index 0000000..d0e8336 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/ShiftRequest.java @@ -0,0 +1,168 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + +import java.io.Serializable; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; + +/******************************************************************************* + ** Typed ShiftRequest contract for comparison and relationship analysis. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE) +public class ShiftRequest implements AnalysisRequest +{ + private String tableName; + private QQueryFilter filter; + private Boolean exact = true; + private String field; + private String dateField; + private String bucket = "MONTH"; + + + + /******************************************************************************* + ** Getter for tableName. + *******************************************************************************/ + @Override + public String getTableName() + { + return (tableName); + } + + + + /******************************************************************************* + ** Fluent setter for tableName. + *******************************************************************************/ + public ShiftRequest withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for filter. + *******************************************************************************/ + @Override + public QQueryFilter getFilter() + { + return (filter); + } + + + + /******************************************************************************* + ** Fluent setter for filter. + *******************************************************************************/ + public ShiftRequest withFilter(QQueryFilter filter) + { + this.filter = filter; + return (this); + } + + + + /******************************************************************************* + ** Getter for exact. + *******************************************************************************/ + @Override + public Boolean getExact() + { + return (exact); + } + + + + /******************************************************************************* + ** Fluent setter for exact. + *******************************************************************************/ + public ShiftRequest withExact(Boolean exact) + { + this.exact = exact; + return (this); + } + + + + /******************************************************************************* + ** Getter for field. + *******************************************************************************/ + public String getField() + { + return (field); + } + + + + /******************************************************************************* + ** Fluent setter for field. + *******************************************************************************/ + public ShiftRequest withField(String field) + { + this.field = field; + return (this); + } + + + + /******************************************************************************* + ** Getter for dateField. + *******************************************************************************/ + public String getDateField() + { + return (dateField); + } + + + + /******************************************************************************* + ** Fluent setter for dateField. + *******************************************************************************/ + public ShiftRequest withDateField(String dateField) + { + this.dateField = dateField; + return (this); + } + + + + /******************************************************************************* + ** Getter for bucket. + *******************************************************************************/ + public String getBucket() + { + return (bucket); + } + + + + /******************************************************************************* + ** Fluent setter for bucket. + *******************************************************************************/ + public ShiftRequest withBucket(String bucket) + { + this.bucket = bucket; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/ShiftResult.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/ShiftResult.java new file mode 100644 index 0000000..63f17c6 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/ShiftResult.java @@ -0,0 +1,121 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + +import java.io.Serializable; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +/******************************************************************************* + ** Typed ShiftResult contract for comparison and relationship analysis. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE) +public class ShiftResult implements Serializable +{ + private List buckets = new java.util.ArrayList<>(); + private Metric psi; + private String sentence; + private Chart chart; + + + + /******************************************************************************* + ** Getter for buckets. + *******************************************************************************/ + public List getBuckets() + { + return (buckets); + } + + + + /******************************************************************************* + ** Fluent setter for buckets. + *******************************************************************************/ + public ShiftResult withBuckets(List buckets) + { + this.buckets = buckets; + return (this); + } + + + + /******************************************************************************* + ** Getter for psi. + *******************************************************************************/ + public Metric getPsi() + { + return (psi); + } + + + + /******************************************************************************* + ** Fluent setter for psi. + *******************************************************************************/ + public ShiftResult withPsi(Metric psi) + { + this.psi = psi; + return (this); + } + + + + /******************************************************************************* + ** Getter for sentence. + *******************************************************************************/ + public String getSentence() + { + return (sentence); + } + + + + /******************************************************************************* + ** Fluent setter for sentence. + *******************************************************************************/ + public ShiftResult withSentence(String sentence) + { + this.sentence = sentence; + return (this); + } + + + + /******************************************************************************* + ** Getter for chart. + *******************************************************************************/ + public Chart getChart() + { + return (chart); + } + + + + /******************************************************************************* + ** Fluent setter for chart. + *******************************************************************************/ + public ShiftResult withChart(Chart chart) + { + this.chart = chart; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/Significance.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/Significance.java new file mode 100644 index 0000000..b79accf --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/Significance.java @@ -0,0 +1,186 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + +import java.io.Serializable; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +/******************************************************************************* + ** Typed Significance contract for comparison and relationship analysis. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE) +public class Significance implements Serializable +{ + private String test; + private Double statistic; + private Double df; + private Double pValue; + private Boolean applicable = false; + private String sentence; + private Double cramersV; + + + + /******************************************************************************* + ** Getter for test. + *******************************************************************************/ + public String getTest() + { + return (test); + } + + + + /******************************************************************************* + ** Fluent setter for test. + *******************************************************************************/ + public Significance withTest(String test) + { + this.test = test; + return (this); + } + + + + /******************************************************************************* + ** Getter for statistic. + *******************************************************************************/ + public Double getStatistic() + { + return (statistic); + } + + + + /******************************************************************************* + ** Fluent setter for statistic. + *******************************************************************************/ + public Significance withStatistic(Double statistic) + { + this.statistic = statistic; + return (this); + } + + + + /******************************************************************************* + ** Getter for df. + *******************************************************************************/ + public Double getDf() + { + return (df); + } + + + + /******************************************************************************* + ** Fluent setter for df. + *******************************************************************************/ + public Significance withDf(Double df) + { + this.df = df; + return (this); + } + + + + /******************************************************************************* + ** Getter for pValue. + *******************************************************************************/ + public Double getPValue() + { + return (pValue); + } + + + + /******************************************************************************* + ** Fluent setter for pValue. + *******************************************************************************/ + public Significance withPValue(Double pValue) + { + this.pValue = pValue; + return (this); + } + + + + /******************************************************************************* + ** Getter for applicable. + *******************************************************************************/ + public Boolean getApplicable() + { + return (applicable); + } + + + + /******************************************************************************* + ** Fluent setter for applicable. + *******************************************************************************/ + public Significance withApplicable(Boolean applicable) + { + this.applicable = applicable; + return (this); + } + + + + /******************************************************************************* + ** Getter for sentence. + *******************************************************************************/ + public String getSentence() + { + return (sentence); + } + + + + /******************************************************************************* + ** Fluent setter for sentence. + *******************************************************************************/ + public Significance withSentence(String sentence) + { + this.sentence = sentence; + return (this); + } + + + + /******************************************************************************* + ** Getter for cramersV. + *******************************************************************************/ + public Double getCramersV() + { + return (cramersV); + } + + + + /******************************************************************************* + ** Fluent setter for cramersV. + *******************************************************************************/ + public Significance withCramersV(Double cramersV) + { + this.cramersV = cramersV; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/TrendLine.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/TrendLine.java new file mode 100644 index 0000000..62e5f32 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/TrendLine.java @@ -0,0 +1,98 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + +import java.io.Serializable; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +/******************************************************************************* + ** Typed TrendLine contract for comparison and relationship analysis. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE) +public class TrendLine implements Serializable +{ + private Double slope; + private Double intercept; + private Double r; + + + + /******************************************************************************* + ** Getter for slope. + *******************************************************************************/ + public Double getSlope() + { + return (slope); + } + + + + /******************************************************************************* + ** Fluent setter for slope. + *******************************************************************************/ + public TrendLine withSlope(Double slope) + { + this.slope = slope; + return (this); + } + + + + /******************************************************************************* + ** Getter for intercept. + *******************************************************************************/ + public Double getIntercept() + { + return (intercept); + } + + + + /******************************************************************************* + ** Fluent setter for intercept. + *******************************************************************************/ + public TrendLine withIntercept(Double intercept) + { + this.intercept = intercept; + return (this); + } + + + + /******************************************************************************* + ** Getter for r. + *******************************************************************************/ + public Double getR() + { + return (r); + } + + + + /******************************************************************************* + ** Fluent setter for r. + *******************************************************************************/ + public TrendLine withR(Double r) + { + this.r = r; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/stats/SpecialFunctions.java b/src/main/java/com/kingsrook/qbits/dataanalysis/stats/SpecialFunctions.java new file mode 100644 index 0000000..1afaca2 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/stats/SpecialFunctions.java @@ -0,0 +1,179 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.stats; + +/******************************************************************************* + ** Regularized beta and gamma using convergent series and continued fractions. + ** Lanczos log-gamma keeps products in log space; tails are evaluated directly. + *******************************************************************************/ +public final class SpecialFunctions +{ + private static final double EPSILON = 1e-14; + private static final double TINY = 1e-300; + private static final int ITERATIONS = 10000; + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private SpecialFunctions() + { + } + + /******************************************************************************* + ** Student t cumulative distribution, with symmetry around zero. + *******************************************************************************/ + public static double studentTCdf(double t, double df) + { + double tail = beta(df / (df + t * t), df / 2, 0.5) / 2; + return (t < 0 ? tail : 1 - tail); + } + + /******************************************************************************* + ** Chi-square CDF; significance tests use gammaQ directly to retain tiny tails. + *******************************************************************************/ + public static double chiSquareCdf(double x, double df) + { + return (1 - gammaQ(df / 2, x / 2)); + } + + /******************************************************************************* + ** Lanczos approximation, reflected below one half. + *******************************************************************************/ + public static double logGamma(double z) + { + double[] coefficients = {676.5203681218851, -1259.1392167224028, 771.3234287776531, -176.6150291621406, + 12.507343278686905, -0.13857109526572012, 9.984369578019572e-6, 1.5056327351493116e-7}; + if(z < 0.5) + { + return (Math.log(Math.PI) - Math.log(Math.sin(Math.PI * z)) - logGamma(1 - z)); + } + double x = 0.99999999999980993; + for(int i = 0; i < coefficients.length; i++) + { + x += coefficients[i] / (z + i); + } + double t = z + 6.5; + return (0.5 * Math.log(2 * Math.PI) + (z - 0.5) * Math.log(t) - t + Math.log(x)); + } + + /******************************************************************************* + ** Regularized incomplete beta I_x(a,b), with symmetry for stable tails. + *******************************************************************************/ + public static double beta(double x, double a, double b) + { + if(x <= 0 || x >= 1) + { + return (x <= 0 ? 0 : 1); + } + double factor = Math.exp(logGamma(a + b) - logGamma(a) - logGamma(b) + a * Math.log(x) + b * Math.log1p(-x)); + return (clamp(x < (a + 1) / (a + b + 2) ? factor * betaFraction(x, a, b) / a : 1 - factor * betaFraction(1 - x, b, a) / b)); + } + + /******************************************************************************* + ** Modified Lentz evaluation of the incomplete beta continued fraction. + *******************************************************************************/ + private static double betaFraction(double x, double a, double b) + { + double c = 1; + double d = 1 / nonzero(1 - (a + b) * x / (a + 1)); + double h = d; + for(int m = 1; m <= ITERATIONS; m++) + { + double aa = m * (b - m) * x / ((a + 2 * m - 1) * (a + 2 * m)); + d = 1 / nonzero(1 + aa * d); + c = nonzero(1 + aa / c); + h *= d * c; + aa = -(a + m) * (a + b + m) * x / ((a + 2 * m) * (a + 2 * m + 1)); + d = 1 / nonzero(1 + aa * d); + c = nonzero(1 + aa / c); + double delta = d * c; + h *= delta; + if(Math.abs(delta - 1) < EPSILON) + { + return (h); + } + } + throw new ArithmeticException("Incomplete beta did not converge"); + } + + /******************************************************************************* + ** Upper regularized gamma Q(a,x), the chi-square survival probability. + *******************************************************************************/ + public static double gammaQ(double a, double x) + { + if(x <= 0) + { + return (1); + } + double factor = Math.exp(-x + a * Math.log(x) - logGamma(a)); + if(x < a + 1) + { + double term = 1 / a; + double sum = term; + for(int n = 1; n <= ITERATIONS; n++) + { + term *= x / (a + n); + sum += term; + if(Math.abs(term) < Math.abs(sum) * EPSILON) + { + return (clamp(1 - sum * factor)); + } + } + } + else + { + double b = x + 1 - a; + double c = 1 / TINY; + double d = 1 / nonzero(b); + double h = d; + for(int n = 1; n <= ITERATIONS; n++) + { + double an = -n * (n - a); + b += 2; + d = 1 / nonzero(an * d + b); + c = nonzero(b + an / c); + double delta = d * c; + h *= delta; + if(Math.abs(delta - 1) < EPSILON) + { + return (clamp(factor * h)); + } + } + } + throw new ArithmeticException("Incomplete gamma did not converge"); + } + + /******************************************************************************* + ** Avoid divisions by zero without changing the sign of a fraction term. + *******************************************************************************/ + private static double nonzero(double value) + { + return (Math.abs(value) < TINY ? Math.copySign(TINY, value) : value); + } + + /******************************************************************************* + ** Roundoff cannot create a probability outside its domain. + *******************************************************************************/ + private static double clamp(double value) + { + return (Math.max(0, Math.min(1, value))); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/stats/Statistics.java b/src/main/java/com/kingsrook/qbits/dataanalysis/stats/Statistics.java new file mode 100644 index 0000000..253279e --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/stats/Statistics.java @@ -0,0 +1,286 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.stats; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; + +/******************************************************************************* + ** Dependency-free statistical tests. All tests use two-sided 5% significance. + *******************************************************************************/ +public final class Statistics +{ + /******************************************************************************* + ** Non-applicable results never contain NaN, infinity or an invented p-value. + *******************************************************************************/ + public record Result(String test, Double statistic, Double df, Double pValue, boolean applicable, String sentence, Double cramersV) + { + } + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private Statistics() + { + } + + /******************************************************************************* + ** Explain failed assumptions without claiming a negative test result. + *******************************************************************************/ + public static Result unavailable(String test, String reason) + { + return (new Result(test, null, null, null, false, "Not applicable: " + reason, null)); + } + + /******************************************************************************* + ** The strict comparison deliberately excludes p equal to 0.05. + *******************************************************************************/ + public static String sentence(double p) + { + String formatted = p < 0.001 ? String.format(Locale.ROOT, "%.3g", p) + : java.math.BigDecimal.valueOf(p).setScale(p < 0.01 ? 3 : 2, java.math.RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); + return ((p < 0.05 ? "A and B differ" : "No significant difference") + " (p = " + formatted + ")"); + } + + /******************************************************************************* + ** Pooled standard error under equal proportions; require expected counts >= 5. + *******************************************************************************/ + public static Result proportions(long successesA, long nA, long successesB, long nB) + { + String test = "Two-proportion z-test"; + if(nA < 2 || nB < 2) + { + return (unavailable(test, "too few records")); + } + if(successesA < 0 || successesB < 0 || successesA > nA || successesB > nB) + { + return (unavailable(test, "invalid counts")); + } + double pooled = ((double) successesA + successesB) / ((double) nA + nB); + if(Math.min(Math.min(nA * pooled, nA * (1 - pooled)), Math.min(nB * pooled, nB * (1 - pooled))) < 5) + { + return (unavailable(test, "expected counts below 5")); + } + double z = ((double) successesA / nA - (double) successesB / nB) / Math.sqrt(pooled * (1 - pooled) * (1.0 / nA + 1.0 / nB)); + double p = SpecialFunctions.gammaQ(0.5, z * z / 2); + return (new Result(test, z, null, p, true, sentence(p), null)); + } + + /******************************************************************************* + ** Welch-Satterthwaite degrees of freedom from sample (ddof=1) variances. + *******************************************************************************/ + public static Result welch(long nA, double meanA, double varianceA, long nB, double meanB, double varianceB) + { + String test = "Welch's t-test"; + if(nA < 2 || nB < 2) + { + return (unavailable(test, "too few records")); + } + if(!Double.isFinite(meanA) || !Double.isFinite(meanB) || !Double.isFinite(varianceA) || !Double.isFinite(varianceB) || varianceA < 0 || varianceB < 0) + { + return (unavailable(test, "invalid numeric summary")); + } + double a = varianceA / nA; + double b = varianceB / nB; + if(a + b == 0) + { + return (unavailable(test, "both groups have zero variance")); + } + double t = (meanA - meanB) / Math.sqrt(a + b); + double df = (a + b) * (a + b) / (a * a / (nA - 1) + b * b / (nB - 1)); + double p = SpecialFunctions.beta(df / (df + t * t), df / 2, 0.5); + return (new Result(test, t, df, p, true, sentence(p), null)); + } + + /******************************************************************************* + ** Pearson chi-square without Yates correction; low expected categories pool + ** into Other. Homogeneity preserves sample rows; independence pools both axes. + *******************************************************************************/ + public static Result chiSquare(double[][] observed, boolean homogeneity) + { + String test = homogeneity ? "Chi-square homogeneity" : "Chi-square independence"; + if(observed == null || observed.length < 2 || observed[0] == null || observed[0].length < 2) + { + return (unavailable(test, "a table of at least 2 by 2 is required")); + } + int columns = observed[0].length; + for(double[] row : observed) + { + if(row == null || row.length != columns || Arrays.stream(row).anyMatch(v -> !Double.isFinite(v) || v < 0)) + { + return (unavailable(test, "invalid counts")); + } + if(Arrays.stream(row).sum() < 2) + { + return (unavailable(test, "too few records")); + } + } + double[][] table = poolColumns(observed); + if(!homogeneity) + { + table = transpose(poolColumns(transpose(table))); + double[][] rowFirst = poolColumns(transpose(poolColumns(transpose(observed)))); + if(validExpected(rowFirst) && (!validExpected(table) || rowFirst.length * rowFirst[0].length > table.length * table[0].length)) + { + table = rowFirst; + } + } + if(table.length < 2 || table[0].length < 2) + { + return (unavailable(test, "too few categories after merging expected counts below 5 into Other")); + } + double[] rows = Arrays.stream(table).mapToDouble(r -> Arrays.stream(r).sum()).toArray(); + double[] cols = Arrays.stream(transpose(table)).mapToDouble(r -> Arrays.stream(r).sum()).toArray(); + double total = Arrays.stream(rows).sum(); + double statistic = 0; + for(int i = 0; i < rows.length; i++) + { + for(int j = 0; j < cols.length; j++) + { + double expected = rows[i] * cols[j] / total; + if(expected < 5) + { + return (unavailable(test, "expected counts below 5 after merging into Other")); + } + statistic += Math.pow(table[i][j] - expected, 2) / expected; + } + } + double df = (rows.length - 1.0) * (cols.length - 1.0); + double p = SpecialFunctions.gammaQ(df / 2, statistic / 2); + double v = Math.sqrt(statistic / (total * Math.min(rows.length - 1, cols.length - 1))); + return (new Result(test, statistic, df, p, true, sentence(p), v)); + } + + /******************************************************************************* + ** Rare rows must not force every column into Other when row pooling alone + ** would suffice. Prefer the valid pooling order retaining more categories. + *******************************************************************************/ + private static boolean validExpected(double[][] table) + { + double[] rows = Arrays.stream(table).mapToDouble(r -> Arrays.stream(r).sum()).toArray(); + double[] columns = Arrays.stream(transpose(table)).mapToDouble(r -> Arrays.stream(r).sum()).toArray(); + return (rows.length >= 2 && columns.length >= 2 + && Arrays.stream(rows).min().orElse(0) * Arrays.stream(columns).min().orElse(0) / Arrays.stream(rows).sum() >= 5); + } + + /******************************************************************************* + ** Pool all categories with any expected count below five into one final column. + *******************************************************************************/ + private static double[][] poolColumns(double[][] table) + { + double[] rows = Arrays.stream(table).mapToDouble(r -> Arrays.stream(r).sum()).toArray(); + double total = Arrays.stream(rows).sum(); + List kept = new ArrayList<>(); + List other = new ArrayList<>(); + for(int j = 0; j < table[0].length; j++) + { + double sum = 0; + for(double[] row : table) + { + sum += row[j]; + } + double column = sum; + (Arrays.stream(rows).anyMatch(r -> r * column / total < 5) ? other : kept).add(j); + } + double[][] result = new double[rows.length][kept.size() + (other.isEmpty() ? 0 : 1)]; + for(int i = 0; i < rows.length; i++) + { + for(int j = 0; j < kept.size(); j++) + { + result[i][j] = table[i][kept.get(j)]; + } + for(int j : other) + { + result[i][kept.size()] += table[i][j]; + } + } + return (result); + } + + /******************************************************************************* + ** Transposition lets independence apply the same pooling rule to both axes. + *******************************************************************************/ + private static double[][] transpose(double[][] input) + { + double[][] result = new double[input[0].length][input.length]; + for(int i = 0; i < input.length; i++) + { + for(int j = 0; j < input[i].length; j++) + { + result[j][i] = input[i][j]; + } + } + return (result); + } + + /******************************************************************************* + ** Normalize counts to shares; floor zero shares at 1e-6 for finite PSI. + ** This descriptive index is not a p-value or a significance test. + *******************************************************************************/ + public static Double psi(double[] first, double[] last) + { + if(first == null || last == null || first.length == 0 || first.length != last.length + || Arrays.stream(first).anyMatch(v -> !Double.isFinite(v) || v < 0) || Arrays.stream(last).anyMatch(v -> !Double.isFinite(v) || v < 0)) + { + return (null); + } + double a = Arrays.stream(first).sum(); + double b = Arrays.stream(last).sum(); + if(a <= 0 || b <= 0) + { + return (null); + } + double result = 0; + for(int i = 0; i < first.length; i++) + { + double x = Math.max(1e-6, first[i] / a); + double y = Math.max(1e-6, last[i] / b); + result += (y - x) * Math.log(y / x); + } + return (result); + } + + /******************************************************************************* + ** Centered covariance avoids cancellation for large offsets; null on constants. + *******************************************************************************/ + public static Double pearson(double[] x, double[] y) + { + if(x == null || y == null || x.length < 2 || x.length != y.length + || Arrays.stream(x).anyMatch(v -> !Double.isFinite(v)) || Arrays.stream(y).anyMatch(v -> !Double.isFinite(v))) + { + return (null); + } + double meanX = Arrays.stream(x).average().orElseThrow(); + double meanY = Arrays.stream(y).average().orElseThrow(); + double xx = 0; + double yy = 0; + double xy = 0; + for(int i = 0; i < x.length; i++) + { + xx += (x[i] - meanX) * (x[i] - meanX); + yy += (y[i] - meanY) * (y[i] - meanY); + xy += (x[i] - meanX) * (y[i] - meanY); + } + return (xx == 0 || yy == 0 ? null : Math.max(-1, Math.min(1, xy / Math.sqrt(xx * yy)))); + } +} diff --git a/src/main/resources/openapi/qqq-v1/data-analysis.yaml b/src/main/resources/openapi/qqq-v1/data-analysis.yaml index 5d53e6d..5f369a4 100644 --- a/src/main/resources/openapi/qqq-v1/data-analysis.yaml +++ b/src/main/resources/openapi/qqq-v1/data-analysis.yaml @@ -118,6 +118,10 @@ components: DataAnalysisChartV1: nullable: true properties: + stacked: + description: "Stack category series to a total share of one when true." + nullable: true + type: "boolean" kind: description: "Semantic field kind: NUMERIC, DATE, DATE_TIME, TEXT, BOOLEAN\ \ or CATEGORY." @@ -273,6 +277,221 @@ components: description: "Chart." nullable: true type: "object" + DataAnalysisCompareRequestV1: + properties: + filterA: + $ref: "#/components/schemas/QueryFilter" + description: "Population A filter; null selects the whole permitted table." + nullable: true + filterB: + $ref: "#/components/schemas/QueryFilter" + description: "Population B filter; null selects the whole permitted table." + nullable: true + labelA: + description: "Display name of population A; default A." + type: "string" + labelB: + description: "Display name of population B; default B." + type: "string" + fields: + description: "Distinct FULL fields to analyze; correlation requires 2 to\ + \ 12 numeric fields." + items: + type: "string" + type: "array" + required: + - "fields" + type: "object" + DataAnalysisCompareV1: + properties: + fields: + description: "Distinct FULL fields to analyze; correlation requires 2 to\ + \ 12 numeric fields." + items: + $ref: "#/components/schemas/DataAnalysisFieldComparisonV1" + nullable: true + type: "array" + type: "object" + DataAnalysisComparisonSideV1: + properties: + label: + description: "Label." + nullable: true + type: "string" + metrics: + description: "Typed metrics with exact population drills unless privacy\ + \ suppresses them." + items: + $ref: "#/components/schemas/DataAnalysisMetricV1" + nullable: true + type: "array" + type: "object" + DataAnalysisComparisonValueV1: + properties: + value: + description: "Value." + nullable: true + label: + description: "Label." + nullable: true + type: "string" + sideA: + $ref: "#/components/schemas/DataAnalysisComparisonSideV1" + description: "Side a." + nullable: true + sideB: + $ref: "#/components/schemas/DataAnalysisComparisonSideV1" + description: "Side b." + nullable: true + delta: + $ref: "#/components/schemas/DataAnalysisMetricV1" + description: "B minus A, in original units (share differences are fractions)." + nullable: true + percentDelta: + $ref: "#/components/schemas/DataAnalysisMetricV1" + description: "(B minus A) divided by absolute A, a fraction; null at a zero\ + \ baseline." + nullable: true + significance: + $ref: "#/components/schemas/DataAnalysisSignificanceV1" + description: "Test assumptions, two-sided p-value, degrees of freedom and\ + \ plain-language conclusion." + nullable: true + type: "object" + DataAnalysisCorrelationCellV1: + properties: + xField: + description: "FULL numeric field on the x axis." + nullable: true + type: "string" + yField: + description: "FULL numeric field on the y axis." + nullable: true + type: "string" + "n": + description: "Pairwise non-null population count; null when privacy suppresses\ + \ it." + nullable: true + type: "integer" + r: + description: "Pearson correlation; null for insufficient data, zero variance\ + \ or privacy suppression." + nullable: true + type: "number" + scatter: + $ref: "#/components/schemas/DataAnalysisScatterRequestV1" + description: "POST the included parameters to the table scatter endpoint." + nullable: true + type: "object" + DataAnalysisCorrelationRequestV1: + properties: + filter: + $ref: "#/components/schemas/QueryFilter" + description: "Current-user query filter; record locks and core field rules\ + \ apply." + nullable: true + fields: + description: "Distinct FULL fields to analyze; correlation requires 2 to\ + \ 12 numeric fields." + items: + type: "string" + type: "array" + required: + - "fields" + type: "object" + DataAnalysisCorrelationV1: + properties: + fields: + description: "Distinct FULL fields to analyze; correlation requires 2 to\ + \ 12 numeric fields." + items: + type: "string" + nullable: true + type: "array" + cells: + description: "Cells." + items: + $ref: "#/components/schemas/DataAnalysisCorrelationCellV1" + nullable: true + type: "array" + chart: + $ref: "#/components/schemas/DataAnalysisChartV1" + description: "Semantic series; category comparisons are grouped and category\ + \ time series are stacked." + nullable: true + type: "object" + DataAnalysisCrosstabCellV1: + properties: + rowIndex: + description: "Row index." + nullable: true + type: "integer" + columnIndex: + description: "Column index." + nullable: true + type: "integer" + count: + $ref: "#/components/schemas/DataAnalysisMetricV1" + description: "Cell count with its exact drill, or a suppressed metric." + nullable: true + rowShare: + $ref: "#/components/schemas/DataAnalysisMetricV1" + description: "Cell count divided by row total; suppressed if it could disclose\ + \ a hidden cell." + nullable: true + columnShare: + $ref: "#/components/schemas/DataAnalysisMetricV1" + description: "Cell count divided by column total; suppressed if it could\ + \ disclose a hidden cell." + nullable: true + type: "object" + DataAnalysisCrosstabRequestV1: + properties: + filter: + $ref: "#/components/schemas/QueryFilter" + description: "Current-user query filter; record locks and core field rules\ + \ apply." + nullable: true + rowField: + description: "FULL field defining crosstab rows." + type: "string" + columnField: + description: "FULL field defining crosstab columns." + type: "string" + limit: + description: "Maximum sample points (1 to 2000, default 2000) or kept crosstab\ + \ categories per axis (1 to 20, default 20)." + type: "integer" + required: + - "rowField" + - "columnField" + type: "object" + DataAnalysisCrosstabV1: + properties: + rows: + description: "Rows." + items: + $ref: "#/components/schemas/DataAnalysisPivotHeaderV1" + nullable: true + type: "array" + columns: + description: "Columns." + items: + $ref: "#/components/schemas/DataAnalysisPivotHeaderV1" + nullable: true + type: "array" + cells: + description: "Cells." + items: + $ref: "#/components/schemas/DataAnalysisCrosstabCellV1" + nullable: true + type: "array" + significance: + $ref: "#/components/schemas/DataAnalysisSignificanceV1" + description: "Test assumptions, two-sided p-value, degrees of freedom and\ + \ plain-language conclusion." + nullable: true + type: "object" DataAnalysisDateProfileV1: nullable: true properties: @@ -495,6 +714,29 @@ components: nullable: true type: "array" type: "object" + DataAnalysisFieldComparisonV1: + properties: + field: + description: "FULL field whose distribution is compared over time." + nullable: true + type: "string" + values: + description: "Values." + items: + $ref: "#/components/schemas/DataAnalysisComparisonValueV1" + nullable: true + type: "array" + significance: + $ref: "#/components/schemas/DataAnalysisSignificanceV1" + description: "Test assumptions, two-sided p-value, degrees of freedom and\ + \ plain-language conclusion." + nullable: true + chart: + $ref: "#/components/schemas/DataAnalysisChartV1" + description: "Semantic series; category comparisons are grouped and category\ + \ time series are stacked." + nullable: true + type: "object" DataAnalysisFieldSummaryV1: nullable: true properties: @@ -912,6 +1154,68 @@ components: nullable: true type: "string" type: "object" + DataAnalysisScatterRequestV1: + properties: + tableName: + description: "Table for the scatter link; route parameter takes precedence." + nullable: true + type: "string" + exact: + description: "Compatibility flag; scatter always returns the bounded sample." + nullable: true + type: "boolean" + filter: + $ref: "#/components/schemas/QueryFilter" + description: "Current-user query filter; record locks and core field rules\ + \ apply." + nullable: true + xField: + description: "FULL numeric field on the x axis." + type: "string" + yField: + description: "FULL numeric field on the y axis." + type: "string" + limit: + description: "Maximum sample points (1 to 2000, default 2000) or kept crosstab\ + \ categories per axis (1 to 20, default 20)." + type: "integer" + required: + - "xField" + - "yField" + type: "object" + DataAnalysisScatterV1: + properties: + "n": + description: "Pairwise non-null population count; null when privacy suppresses\ + \ it." + nullable: true + type: "integer" + sampleSize: + description: "Number of displayed paired records." + nullable: true + type: "integer" + sampling: + description: "Sampling method; core query currently uses a deterministic\ + \ primary-key prefix." + nullable: true + type: "string" + points: + description: "Points." + items: + $ref: "#/components/schemas/DataAnalysisPointV1" + nullable: true + type: "array" + trend: + $ref: "#/components/schemas/DataAnalysisTrendLineV1" + description: "Least-squares slope, intercept and r fitted to the displayed\ + \ sample." + nullable: true + chart: + $ref: "#/components/schemas/DataAnalysisChartV1" + description: "Semantic series; category comparisons are grouped and category\ + \ time series are stacked." + nullable: true + type: "object" DataAnalysisSeriesV1: nullable: true properties: @@ -926,6 +1230,105 @@ components: nullable: true type: "array" type: "object" + DataAnalysisShiftBucketV1: + properties: + value: + description: "Value." + nullable: true + label: + description: "Label." + nullable: true + type: "string" + complete: + description: "True when the calendar bucket end has passed in the session\ + \ timezone." + nullable: true + type: "boolean" + metrics: + description: "Typed metrics with exact population drills unless privacy\ + \ suppresses them." + items: + $ref: "#/components/schemas/DataAnalysisMetricV1" + nullable: true + type: "array" + type: "object" + DataAnalysisShiftRequestV1: + properties: + filter: + $ref: "#/components/schemas/QueryFilter" + description: "Current-user query filter; record locks and core field rules\ + \ apply." + nullable: true + field: + description: "FULL field whose distribution is compared over time." + type: "string" + dateField: + description: "FULL DATE or DATE_TIME field for session-timezone calendar\ + \ buckets." + type: "string" + bucket: + description: "Calendar unit: HOUR, DAY, WEEK, MONTH, QUARTER or YEAR; default\ + \ MONTH." + type: "string" + required: + - "field" + - "dateField" + type: "object" + DataAnalysisShiftV1: + properties: + buckets: + description: "Buckets." + items: + $ref: "#/components/schemas/DataAnalysisShiftBucketV1" + nullable: true + type: "array" + psi: + $ref: "#/components/schemas/DataAnalysisMetricV1" + description: "Population stability index, null when inapplicable; zero shares\ + \ are floored at 0.000001." + nullable: true + sentence: + description: "Plain-language result or reason the statistic is not applicable." + nullable: true + type: "string" + chart: + $ref: "#/components/schemas/DataAnalysisChartV1" + description: "Semantic series; category comparisons are grouped and category\ + \ time series are stacked." + nullable: true + type: "object" + DataAnalysisSignificanceV1: + properties: + test: + description: "Test." + nullable: true + type: "string" + statistic: + description: "Statistic." + nullable: true + type: "number" + df: + description: "Degrees of freedom; null when undefined." + nullable: true + type: "number" + pValue: + description: "Two-sided p-value; p >= 0.05 is never significant." + nullable: true + type: "number" + applicable: + description: "Whether all test validity and privacy requirements hold." + nullable: true + type: "boolean" + sentence: + description: "Plain-language result or reason the statistic is not applicable." + nullable: true + type: "string" + cramersV: + description: "Cramer association strength computed on the pooled contingency\ + \ table; null when invalid." + nullable: true + type: "number" + type: "object" DataAnalysisStartRunRequestV1: additionalProperties: false properties: @@ -1054,6 +1457,22 @@ components: nullable: true type: "integer" type: "object" + DataAnalysisTrendLineV1: + properties: + slope: + description: "Slope." + nullable: true + type: "number" + intercept: + description: "Intercept." + nullable: true + type: "number" + r: + description: "Pearson correlation; null for insufficient data, zero variance\ + \ or privacy suppression." + nullable: true + type: "number" + type: "object" DataAnalysisValueCountV1: nullable: true properties: @@ -1165,10 +1584,10 @@ components: info: contact: email: "contact@kingsrook.com" - description: "Profiles, pivots, distinct values, duplicate groups and exact analysis\ - \ runs as the signed-in user." + description: "Profiles, pivots, comparisons, distribution shifts, correlations,\ + \ scatter samples and crosstabs as the signed-in user." title: "QQQ Middleware API extension: data-analysis" - version: "1.1.0" + version: "1.2.0" openapi: "3.0.3" paths: /qqq/v1/qbit/data-analysis/tables/{table}/capabilities: @@ -1250,6 +1669,154 @@ paths: - sessionUuidCookie: - "N/A" summary: "List distinct column values" + /qqq/v1/qbit/data-analysis/tables/{table}/compare: + post: + description: "Compare filtered A with filtered B (null means the whole permitted\ + \ table). Return means, medians or category shares, B-minus-A deltas, valid\ + \ significance tests and count-equivalent drills. Only FULL fields are accepted." + parameters: + - description: "QQQ table name to analyze." + in: "path" + name: "table" + required: true + schema: + type: "string" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisCompareRequestV1" + required: true + responses: + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisCompareV1" + description: "Successful Response" + security: + - sessionUuidCookie: + - "N/A" + summary: "Compare two populations" + /qqq/v1/qbit/data-analysis/tables/{table}/shift: + post: + description: "Compare category shares or numeric quartiles across session-timezone\ + \ calendar buckets. Compute descriptive PSI between the first and last complete\ + \ nonempty buckets using fixed categories or baseline quartile bins." + parameters: + - description: "QQQ table name to analyze." + in: "path" + name: "table" + required: true + schema: + type: "string" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisShiftRequestV1" + required: true + responses: + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisShiftV1" + description: "Successful Response" + security: + - sessionUuidCookie: + - "N/A" + summary: "Measure distribution shift over time" + /qqq/v1/qbit/data-analysis/tables/{table}/correlation: + post: + description: "Compute a Pearson correlation matrix for 2 to 12 distinct FULL\ + \ numeric fields, using paired non-null records. Every cell includes n and\ + \ parameters for the scatter endpoint; core pushes down CORR or uses bounded\ + \ streaming." + parameters: + - description: "QQQ table name to analyze." + in: "path" + name: "table" + required: true + schema: + type: "string" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisCorrelationRequestV1" + required: true + responses: + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisCorrelationV1" + description: "Successful Response" + security: + - sessionUuidCookie: + - "N/A" + summary: "Find numeric relationships" + /qqq/v1/qbit/data-analysis/tables/{table}/scatter: + post: + description: "Return at most 2000 paired numeric points as the user, ordered\ + \ by primary key when core query sampling is unavailable, plus a least-squares\ + \ line fitted to the sample. Privacy disables individual drills and suppresses\ + \ small populations." + parameters: + - description: "QQQ table name to analyze." + in: "path" + name: "table" + required: true + schema: + type: "string" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisScatterRequestV1" + required: true + responses: + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisScatterV1" + description: "Successful Response" + security: + - sessionUuidCookie: + - "N/A" + summary: "Inspect a numeric relationship" + /qqq/v1/qbit/data-analysis/tables/{table}/crosstab: + post: + description: "Count combinations of two FULL fields with row and column shares,\ + \ up to 20 categories per axis plus Other. Test independence using chi-square\ + \ and Cramer V after pooling low expected counts; privacy suppresses small\ + \ cells and their drills." + parameters: + - description: "QQQ table name to analyze." + in: "path" + name: "table" + required: true + schema: + type: "string" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisCrosstabRequestV1" + required: true + responses: + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisCrosstabV1" + description: "Successful Response" + security: + - sessionUuidCookie: + - "N/A" + summary: "Cross-tabulate two fields" /qqq/v1/qbit/data-analysis/tables/{table}/pivot: post: description: "Group records by up to 3 row and 2 column fields (with date or\ diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisApiTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisApiTest.java index 5f12f49..86a1b6f 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisApiTest.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisApiTest.java @@ -169,7 +169,7 @@ class DataAnalysisApiTest extends EngineFixture assertThat(denied.getStatus()).isEqualTo(403); assertThat(run.path(operation.equals("COLUMN_PROFILE") ? "columnProfile" : "tableProfile").isObject()).isTrue(); } - assertThat(invoker.listOperations().stream().filter(o -> o.pathTemplate().startsWith(DataAnalysisApi.PREFIX)).toList()).hasSize(8); + assertThat(invoker.listOperations().stream().filter(o -> o.pathTemplate().startsWith(DataAnalysisApi.PREFIX)).toList()).hasSize(13); var client = java.net.http.HttpClient.newHttpClient(); var response = client.send(java.net.http.HttpRequest.newBuilder(java.net.URI.create("http://127.0.0.1:" + server.getBoundPort() + "/qqq/v1/openapi.json")).build(), java.net.http.HttpResponse.BodyHandlers.ofString()); assertThat(response.statusCode()).isEqualTo(200); @@ -269,7 +269,7 @@ class DataAnalysisApiTest extends EngineFixture } /******************************************************************************* - ** Generated fragment is committed verbatim and has only the eight operations. + ** Generated fragment is committed verbatim and has all thirteen operations. *******************************************************************************/ @Test void contract() throws Exception @@ -320,4 +320,99 @@ class DataAnalysisApiTest extends EngineFixture caller = "denied"; call(HttpMethod.POST, "/tables/{table}/pivot", body, 403); } + + /******************************************************************************* + ** Exercise all five relationship endpoints and count drills. + *******************************************************************************/ + @Test + void relationshipHttp() throws Exception + { + start(); + JsonNode compare = call(HttpMethod.POST, "/tables/{table}/compare", Map.of("fields", List.of("amount", "status"), "labelA", "Selected", "labelB", "All"), 200); + assertThat(compare.path("fields").size()).isEqualTo(2); + assertThat(compare.path("fields").get(0).path("values").get(0).path("sideA").path("label").asText()).isEqualTo("Selected"); + relationshipCountDrills(compare); + JsonNode shift = call(HttpMethod.POST, "/tables/{table}/shift", Map.of("field", "status", "dateField", "order_date", "bucket", "DAY"), 200); + assertThat(shift.path("psi").path("value").asDouble()).isZero(); + relationshipCountDrills(shift); + JsonNode correlation = call(HttpMethod.POST, "/tables/{table}/correlation", Map.of("fields", List.of("amount", "quantity")), 200); + assertThat(correlation.path("cells").size()).isEqualTo(4); + assertThat(correlation.path("cells").get(1).path("n").asLong()).isEqualTo(11); + JsonNode scatter = call(HttpMethod.POST, "/tables/{table}/scatter", Map.of("xField", "id", "yField", "amount", "limit", 3), 200); + assertThat(scatter.path("sampleSize").asInt()).isEqualTo(3); + assertThat(scatter.path("trend").path("slope").asDouble()).isEqualTo(10); + for(JsonNode point : scatter.path("points")) + { + JsonNode drill = point.path("drill"); + var response = invoker.invoke(new UiApiRequest().withMethod(HttpMethod.POST).withPath("/table/{tableName}/count") + .withPathParam("tableName", "orders").withJsonBody(Map.of("filter", drill.get("filter"))).withSessionUUID(caller)); + assertThat(response.getStatus()).isEqualTo(200); + assertThat(response.getJson().path("count").asLong()).isEqualTo(1); + } + relationshipCountDrills(call(HttpMethod.POST, "/tables/{table}/crosstab", Map.of("rowField", "status", "columnField", "customer"), 200)); + // All new schemas are discoverable from the same served OpenAPI catalog. + assertThat(com.kingsrook.qbits.dataanalysis.api.DataAnalysisApi.extension().getSemanticVersion()).isEqualTo("1.2.0"); + } + + /******************************************************************************* + ** Exercise all five relationship endpoints and count drills. + *******************************************************************************/ + /******************************************************************************* + ** Verify strict request validation, privacy and authorization over HTTP. + *******************************************************************************/ + @Test + void relationshipHttpErrorsAndPrivacy() throws Exception + { + config.withDefaultMinGroupSize(20); + instance.withSupplementalMetaData(new DataAnalysisRuntime(config)); + start(); + var requests = Map.of("compare", Map.of("fields", List.of("amount")), "shift", Map.of("field", "status", "dateField", "order_date"), + "correlation", Map.of("fields", List.of("amount", "quantity")), "scatter", Map.of("xField", "id", "yField", "amount"), + "crosstab", Map.of("rowField", "status", "columnField", "customer")); + assertThat(call(HttpMethod.POST, "/tables/{table}/scatter", requests.get("scatter"), 200).path("points").size()).isZero(); + for(String field : List.of("secret", "missing")) + { + call(HttpMethod.POST, "/tables/{table}/compare", Map.of("fields", List.of(field)), 400); + call(HttpMethod.POST, "/tables/{table}/shift", Map.of("field", "status", "dateField", field), 400); + call(HttpMethod.POST, "/tables/{table}/correlation", Map.of("fields", List.of("id", field)), 400); + call(HttpMethod.POST, "/tables/{table}/scatter", Map.of("xField", "id", "yField", field), 400); + call(HttpMethod.POST, "/tables/{table}/crosstab", Map.of("rowField", "status", "columnField", field), 400); + } + call(HttpMethod.POST, "/tables/{table}/compare", Map.of("fields", "amount"), 400); + call(HttpMethod.POST, "/tables/{table}/compare", Map.of("fields", List.of("amount"), "filterA", "bad"), 400); + call(HttpMethod.POST, "/tables/{table}/compare", Map.of("fields", List.of("amount"), "labelA", 7), 400); + call(HttpMethod.POST, "/tables/{table}/scatter", Map.of("xField", "id", "yField", "amount", "limit", 2.5), 400); + call(HttpMethod.POST, "/tables/{table}/scatter", Map.of("xField", "id", "yField", "amount", "limit", 2001), 400); + call(HttpMethod.POST, "/tables/{table}/correlation", Map.of("fields", List.of("id")), 400); + call(HttpMethod.POST, "/tables/{table}/correlation", Map.of("fields", List.of("id", "id")), 400); + call(HttpMethod.POST, "/tables/{table}/shift", Map.of("field", "amount", "dateField", "order_date", "bucket", "BAD"), 400); + call(HttpMethod.POST, "/tables/{table}/crosstab", Map.of("rowField", "status", "columnField", "customer", "limit", 21), 400); + for(String operation : requests.keySet()) + { + call(HttpMethod.POST, "/tables/{table}/" + operation, Map.of(), 400); + } + caller = "denied"; + for(var request : requests.entrySet()) + { + call(HttpMethod.POST, "/tables/{table}/" + request.getKey(), request.getValue(), 403); + } + } + + /******************************************************************************* + ** Post each released count drill to the actual core count endpoint. + *******************************************************************************/ + private void relationshipCountDrills(JsonNode node) throws Exception + { + if(node.path("valueType").asText().equals("COUNT")) + { + checkDrills(node); + } + else if(node.isContainerNode()) + { + for(JsonNode child : node) + { + relationshipCountDrills(child); + } + } + } } diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java index e16bbe4..2c714f1 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java @@ -61,7 +61,7 @@ import static org.assertj.core.api.Assertions.within; /******************************************************************************* ** Shared backend acceptance tests, including a generic drill equivalence walker. *******************************************************************************/ -abstract class EngineContractTest extends PivotContractTest +abstract class EngineContractTest extends RelationshipContractTest { /******************************************************************************* ** Every emitted key across all profile kinds has a label, on every backend. diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/RelationshipContractTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/RelationshipContractTest.java new file mode 100644 index 0000000..1554b01 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/RelationshipContractTest.java @@ -0,0 +1,370 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis; + +import java.math.BigDecimal; +import java.util.List; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.kingsrook.qbits.dataanalysis.engine.DrillFilters; +import com.kingsrook.qbits.dataanalysis.engine.FieldAnalysisAccess; +import com.kingsrook.qbits.dataanalysis.result.CompareRequest; +import com.kingsrook.qbits.dataanalysis.result.CorrelationRequest; +import com.kingsrook.qbits.dataanalysis.result.CrosstabRequest; +import com.kingsrook.qbits.dataanalysis.result.Drill; +import com.kingsrook.qbits.dataanalysis.result.Metric; +import com.kingsrook.qbits.dataanalysis.result.ScatterRequest; +import com.kingsrook.qbits.dataanalysis.result.ShiftRequest; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; +import com.kingsrook.qqq.backend.core.actions.tables.CountAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; +import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.within; + +/******************************************************************************* + ** Identical relationship contracts inherited by memory, H2 and PostgreSQL. + *******************************************************************************/ +abstract class RelationshipContractTest extends PivotContractTest +{ + /******************************************************************************* + ** Verify compare numeric and categorical. + *******************************************************************************/ + @Test + void compareNumericAndCategorical() throws Exception + { + var request = new CompareRequest().withTableName("orders").withFields(List.of("amount")) + .withFilterA(DrillFilters.condition("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, 6)) + .withFilterB(DrillFilters.condition("id", QCriteriaOperator.GREATER_THAN, 6)).withLabelA("Early").withLabelB("Late"); + var result = service.compare(request); + var field = result.getFields().getFirst(); + var value = field.getValues().getFirst(); + assertThat(value.getSideA().getLabel()).isEqualTo("Early"); + assertNumber(value.getSideA().getMetrics(), "count", 6); + assertNumber(value.getSideA().getMetrics(), "mean", 50.0 / 6); + assertNumber(value.getSideA().getMetrics(), "median", 10); + assertNumber(value.getSideB().getMetrics(), "count", 5); + assertNumber(value.getSideB().getMetrics(), "mean", 224); + assertNumber(value.getSideB().getMetrics(), "median", 40); + assertThat(((Number) value.getDelta().getValue()).doubleValue()).isCloseTo(224 - 50.0 / 6, within(1e-9)); + // scipy.stats.ttest_ind_from_stats(50/6, sqrt(136.66666666666666), 6, 224, sqrt(188530), 5, equal_var=False) + assertThat(field.getSignificance().getPValue()).isCloseTo(.3290396212758404, within(1e-10)); + assertThat(field.getChart().getKind().name()).isEqualTo("BOX"); + assertThat(field.getChart().getSeries()).hasSize(2).allSatisfy(s -> assertThat(s.getPoints()).hasSize(5)); + relationshipDrills(result); + var categorical = service.compare(new CompareRequest().withTableName("orders").withFields(List.of("status"))); + var category = categorical.getFields().getFirst(); + assertThat(category.getValues()).hasSize(2); + assertThat(category.getSignificance().getPValue()).isEqualTo(1); + assertThat(category.getSignificance().getDf()).isEqualTo(1); + assertThat(category.getChart().getKind().name()).isEqualTo("BAR"); + for(var v : category.getValues()) + { + assertNumber(v.getSideA().getMetrics(), "count", 6); + assertNumber(v.getSideB().getMetrics(), "share", .5); + assertThat(v.getSignificance().getApplicable()).isTrue(); + } + relationshipDrills(categorical); + var small = service.compare(request.withFields(List.of("status"))); + assertThat(small.getFields().getFirst().getValues()).singleElement().satisfies(v -> assertThat(v.getLabel()).isEqualTo("Other")); + assertThat(small.getFields().getFirst().getSignificance().getApplicable()).isFalse(); + relationshipDrills(small); + } + + /******************************************************************************* + ** Verify correlation matches independent java pearson. + *******************************************************************************/ + @Test + void correlationMatchesIndependentJavaPearson() throws Exception + { + var result = service.correlation(new CorrelationRequest().withTableName("orders").withFields(List.of("amount", "quantity", "id"))); + assertThat(result.getCells()).hasSize(9); + double[] x = {-10, 0, 10, 10, 20, 20, 30, 40, 50, 1000, 0}; + double[] y = {1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6}; + double meanX = java.util.Arrays.stream(x).average().orElseThrow(); + double meanY = java.util.Arrays.stream(y).average().orElseThrow(); + double xx = 0, yy = 0, xy = 0; + for(int i = 0; i < x.length; i++) + { + xx += Math.pow(x[i] - meanX, 2); + yy += Math.pow(y[i] - meanY, 2); + xy += (x[i] - meanX) * (y[i] - meanY); + } + var cell = result.getCells().stream().filter(c -> c.getXField().equals("amount") && c.getYField().equals("quantity")).findFirst().orElseThrow(); + assertThat(cell.getN()).isEqualTo(11); + assertThat(cell.getR()).isCloseTo(xy / Math.sqrt(xx * yy), within(1e-10)); + assertThat(service.scatter(cell.getScatter()).getSampleSize()).isEqualTo(11); + assertThat(result.getChart().getKind().name()).isEqualTo("HEATMAP"); + for(var c : result.getCells()) + { + if(c.getXField().equals(c.getYField())) + { + assertThat(c.getR()).isCloseTo(1, within(1e-12)); + } + } + } + + /******************************************************************************* + ** Verify scatter limits trend and record drills. + *******************************************************************************/ + @Test + void scatterLimitsTrendAndRecordDrills() throws Exception + { + var result = service.scatter(new ScatterRequest().withTableName("orders").withXField("id").withYField("amount").withLimit(3)); + assertThat(result.getN()).isEqualTo(11); + assertThat(result.getSampleSize()).isEqualTo(3); + assertThat(result.getPoints()).extracting(p -> p.getX().toString()).containsExactly("1", "2", "3"); + assertThat(result.getTrend().getSlope()).isCloseTo(10, within(1e-12)); + assertThat(result.getTrend().getIntercept()).isCloseTo(-20, within(1e-12)); + assertThat(result.getTrend().getR()).isCloseTo(1, within(1e-12)); + assertThat(result.getChart().getSeries()).hasSize(2); + for(var point : result.getPoints()) + { + assertThat(drillCount(point.getDrill())).isEqualTo(1); + assertThat(point.getDrill().getFilter().getSubFilters().getLast().getCriteria()).singleElement().satisfies(c -> + { + assertThat(c.getFieldName()).isEqualTo("id"); + assertThat(c.getOperator()).isEqualTo(QCriteriaOperator.EQUALS); + }); + } + assertThatThrownBy(() -> service.scatter(new ScatterRequest().withTableName("orders").withXField("id").withYField("amount").withLimit(2001))).isInstanceOf(QBadRequestException.class); + assertThat(service.scatter(new ScatterRequest().withTableName("orders").withXField("id").withYField("amount").withLimit(1)).getTrend()).isNull(); + var constant = service.scatter(new ScatterRequest().withTableName("orders").withXField("id").withYField("quantity").withLimit(2)); + assertThat(constant.getTrend().getSlope()).isZero(); + assertThat(constant.getTrend().getIntercept()).isEqualTo(1); + assertThat(constant.getTrend().getR()).isNull(); + assertThat(service.scatter(new ScatterRequest().withTableName("orders").withXField("quantity").withYField("id").withLimit(2)).getTrend()).isNull(); + } + + /******************************************************************************* + ** Verify crosstab counts shares chi square and other. + *******************************************************************************/ + @Test + void crosstabCountsSharesChiSquareAndOther() throws Exception + { + var request = new CrosstabRequest().withTableName("orders").withRowField("status").withColumnField("customer"); + var result = service.crosstab(request); + assertThat(result.getCells()).hasSize(4); + assertThat(result.getCells()).extracting(c -> ((Number) c.getCount().getValue()).longValue()).containsExactlyInAnyOrder(4L, 4L, 2L, 2L); + assertThat(result.getCells()).allSatisfy(c -> assertThat(((Number) c.getColumnShare().getValue()).doubleValue()).isEqualTo(.5)); + assertThat(result.getSignificance().getApplicable()).isFalse(); + relationshipDrills(result); + relationshipDrills(service.crosstab(request.withLimit(1))); + for(int i = 13; i <= 132; i++) + { + TestSupport.insert("orders", new QRecord().withValue("id", i).withValue("customer", i < 73 ? 1 : 2).withValue("status", i % 3 == 0 ? "NEW" : "PAID")); + } + var larger = service.crosstab(request.withLimit(20)); + assertThat(larger.getSignificance().getApplicable()).isTrue(); + assertThat(larger.getSignificance().getCramersV()).isNotNull(); + // scipy.stats.chi2_contingency([[24,22],[44,42]], correction=False) + assertThat(larger.getSignificance().getStatistic()).isCloseTo(.012267293166002618, within(1e-12)); + assertThat(larger.getSignificance().getPValue()).isCloseTo(.9118083997104913, within(1e-12)); + relationshipDrills(larger); + } + + /******************************************************************************* + ** Verify shift psi and quartiles. + *******************************************************************************/ + @Test + void shiftPsiAndQuartiles() throws Exception + { + var request = new ShiftRequest().withTableName("orders").withField("status").withDateField("order_date").withBucket("DAY"); + var result = service.shift(request); + assertThat(result.getBuckets()).hasSize(2).allSatisfy(b -> assertThat(b.getComplete()).isTrue()); + assertThat(((Number) result.getPsi().getValue()).doubleValue()).isZero(); + assertThat(result.getChart().getSeries()).hasSize(2); + relationshipDrills(result); + var shifted = service.shift(request.withField("customer")); + // scipy.special.rel_entr([.5,.5], [1,1e-6]).sum() + scipy.special.rel_entr([1,1e-6], [.5,.5]).sum() + assertThat(((Number) shifted.getPsi().getValue()).doubleValue()).isCloseTo(6.907742156618759, within(1e-10)); + var numeric = service.shift(request.withField("amount")); + assertThat(numeric.getPsi().getValue()).isNotNull(); + assertNumber(numeric.getBuckets().getFirst().getMetrics(), "median", 5); + relationshipDrills(numeric); + for(String unit : List.of("WEEK", "MONTH", "QUARTER", "YEAR", "HOUR")) + { + var bucketed = service.shift(request.withDateField("created").withBucket(unit)); + relationshipDrills(bucketed); + } + } + + /******************************************************************************* + ** Verify relationship privacy and full field security. + *******************************************************************************/ + @Test + void relationshipPrivacyAndFullFieldSecurity() throws Exception + { + config.withDefaultMinGroupSize(6); + service.getConfigResolver().invalidate(); + var scatter = new ScatterRequest().withTableName("orders").withXField("id").withYField("amount"); + assertThat(service.scatter(scatter).getPoints()).allSatisfy(p -> assertThat(p.getDrill()).isNull()); + assertThat(service.scatter(scatter.withFilter(DrillFilters.condition("id", QCriteriaOperator.LESS_THAN, 4))).getPoints()).isEmpty(); + var comparison = service.compare(new CompareRequest().withTableName("orders").withFields(List.of("amount", "customer")) + .withFilterA(DrillFilters.condition("id", QCriteriaOperator.LESS_THAN, 4))); + assertThat(comparison.getFields().getFirst().getValues().getFirst().getSideA().getMetrics()).allSatisfy(m -> + { + assertThat(m.getValue()).isNull(); + assertThat(m.getDrill()).isNull(); + }); + assertThat(comparison.getFields().getFirst().getSignificance().getApplicable()).isFalse(); + var cross = service.crosstab(new CrosstabRequest().withTableName("orders").withRowField("status").withColumnField("customer")); + assertThat(cross.getCells()).allSatisfy(c -> assertThat(c.getCount().getValue()).isNull()); + assertThat(cross.getSignificance().getApplicable()).isFalse(); + assertThat(service.shift(new ShiftRequest().withTableName("orders").withField("customer").withDateField("order_date").withBucket("DAY")).getPsi().getValue()).isNull(); + assertThat(service.correlation(new CorrelationRequest().withTableName("orders").withFields(List.of("id", "amount")).withFilter(scatter.getFilter())).getCells()) + .allSatisfy(c -> assertThat(c.getR()).isNull()); + for(String field : List.of("secret", "missing")) + { + assertThatThrownBy(() -> service.compare(new CompareRequest().withTableName("orders").withFields(List.of(field)))).isInstanceOf(QBadRequestException.class); + assertThatThrownBy(() -> service.correlation(new CorrelationRequest().withTableName("orders").withFields(List.of("id", field)))).isInstanceOf(QBadRequestException.class); + assertThatThrownBy(() -> service.scatter(new ScatterRequest().withTableName("orders").withXField("id").withYField(field))).isInstanceOf(QBadRequestException.class); + assertThatThrownBy(() -> service.crosstab(new CrosstabRequest().withTableName("orders").withRowField("status").withColumnField(field))).isInstanceOf(QBadRequestException.class); + assertThatThrownBy(() -> service.shift(new ShiftRequest().withTableName("orders").withField(field).withDateField("order_date"))).isInstanceOf(QBadRequestException.class); + } + QContext.init(instance, TestSupport.user("denied").withPermissions("orders.read")); + assertThatThrownBy(() -> service.scatter(scatter)).isInstanceOf(QPermissionDeniedException.class); + } + + /******************************************************************************* + ** Verify relationship locks and resolver. + *******************************************************************************/ + @Test + void relationshipLocksAndResolver() throws Exception + { + QContext.getQSession().getSecurityKeyValues().remove("orders.all"); + QContext.getQSession().withSecurityKeyValue("customer_scope", 1); + assertThat(service.scatter(new ScatterRequest().withTableName("orders").withXField("id").withYField("amount")).getN()).isEqualTo(8); + var restricted = new DataAnalysisQBitConfig().withBackendName("memory").withFieldAnalysisAccessResolver(() -> (t, f) -> f.getName().equals("amount") ? FieldAnalysisAccess.NONE : FieldAnalysisAccess.FULL); + service = newService(restricted, Runnable::run); + assertThatThrownBy(() -> service.compare(new CompareRequest().withTableName("orders").withFields(List.of("amount")))).isInstanceOf(QBadRequestException.class); + } + + /******************************************************************************* + ** Verify top categories and empty populations. + *******************************************************************************/ + @Test + void topCategoriesAndEmptyPopulations() throws Exception + { + for(int i = 0; i < 300; i++) + { + TestSupport.insert("orders", new QRecord().withValue("id", 1000 + i).withValue("customer", 1) + .withValue("note", "Category " + i % 15).withValue("paid", i % 2 == 0) + .withValue("order_date", java.time.LocalDate.of(2025, 1, i < 150 ? 1 : 2))); + } + var filter = DrillFilters.condition("id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, 1000); + var compared = service.compare(new CompareRequest().withTableName("orders").withFields(List.of("note")).withFilterA(filter).withFilterB(filter)); + assertThat(compared.getFields().getFirst().getValues()).hasSize(11); + assertThat(compared.getFields().getFirst().getValues().getLast().getLabel()).isEqualTo("Other"); + assertNumber(compared.getFields().getFirst().getValues().getLast().getSideA().getMetrics(), "count", 100); + relationshipDrills(compared); + var shift = service.shift(new ShiftRequest().withTableName("orders").withField("note").withDateField("order_date").withBucket("DAY").withFilter(filter)); + assertThat(shift.getChart().getSeries()).hasSize(9); + assertThat(shift.getChart().getStacked()).isTrue(); + assertThat(((Number) shift.getPsi().getValue()).doubleValue()).isZero(); + relationshipDrills(shift); + relationshipDrills(service.crosstab(new CrosstabRequest().withTableName("orders").withRowField("note").withColumnField("paid").withFilter(filter).withLimit(5))); + var empty = DrillFilters.condition("id", QCriteriaOperator.LESS_THAN, 0); + assertThat(service.compare(new CompareRequest().withTableName("orders").withFields(List.of("amount")).withFilterA(empty)).getFields().getFirst().getSignificance().getApplicable()).isFalse(); + assertThat(service.correlation(new CorrelationRequest().withTableName("orders").withFields(List.of("id", "amount")).withFilter(empty)).getCells()).allSatisfy(c -> assertThat(c.getR()).isNull()); + assertThat(service.crosstab(new CrosstabRequest().withTableName("orders").withRowField("note").withColumnField("paid").withFilter(empty)).getCells()).isEmpty(); + assertThat(service.shift(new ShiftRequest().withTableName("orders").withField("note").withDateField("order_date").withFilter(empty)).getPsi().getValue()).isNull(); + } + + /******************************************************************************* + ** Verify future buckets do not enter psi and point drills respect resolver. + *******************************************************************************/ + @Test + void futureBucketsDoNotEnterPsiAndPointDrillsRespectResolver() throws Exception + { + var future = java.time.LocalDate.now(java.time.ZoneId.of("America/Chicago")).plusYears(1); + TestSupport.insert("orders", new QRecord().withValue("id", 99).withValue("customer", 1).withValue("order_date", future).withValue("status", "NEW")); + var shift = service.shift(new ShiftRequest().withTableName("orders").withField("status").withDateField("order_date").withBucket("DAY")); + assertThat(shift.getBuckets().getLast().getComplete()).isFalse(); + assertThat(((Number) shift.getPsi().getValue()).doubleValue()).isZero(); + TestSupport.insert("orders", new QRecord().withValue("id", 98).withValue("customer", 1).withValue("amount", BigDecimal.ONE) + .withValue("order_date", java.time.LocalDate.of(2024, 1, 1)).withValue("status", "NEW")); + for(String field : List.of("amount", "status")) + { + var sparseFirst = service.shift(new ShiftRequest().withTableName("orders").withField(field).withDateField("order_date").withBucket("DAY")); + assertThat(sparseFirst.getPsi().getValue()).isNull(); + assertThat(sparseFirst.getSentence()).contains("at least two values"); + } + var restricted = new DataAnalysisQBitConfig().withBackendName("memory").withFieldAnalysisAccessResolver(() -> (t, f) -> f.getName().equals("id") ? FieldAnalysisAccess.NONE : FieldAnalysisAccess.FULL); + service = newService(restricted, Runnable::run); + assertThat(service.scatter(new ScatterRequest().withTableName("orders").withXField("amount").withYField("quantity")).getPoints()).hasSize(11).allSatisfy(p -> assertThat(p.getDrill()).isNull()); + restricted.withFieldAnalysisAccessResolver(() -> (t, f) -> f.getName().equals("amount") ? FieldAnalysisAccess.EXACT_MATCH_ONLY : FieldAnalysisAccess.FULL); + service = newService(restricted, Runnable::run); + assertThatThrownBy(() -> service.correlation(new CorrelationRequest().withTableName("orders").withFields(List.of("amount", "quantity")))).isInstanceOf(QBadRequestException.class); + } + + /******************************************************************************* + ** Verify assert number. + *******************************************************************************/ + private static void assertNumber(List metrics, String key, double expected) + { + assertThat(metrics.stream().filter(m -> m.getKey().equals(key)).findFirst().orElseThrow().getValue()).isNotNull(); + assertThat(new BigDecimal(metrics.stream().filter(m -> m.getKey().equals(key)).findFirst().orElseThrow().getValue().toString()).doubleValue()).isCloseTo(expected, within(1e-9)); + } + + /******************************************************************************* + ** Verify drill count. + *******************************************************************************/ + private static long drillCount(Drill drill) throws Exception + { + CountInput input = new CountInput(drill.getTableName()).withFilter(DrillFilters.restoreTypes(drill.getTableName(), drill.getFilter())); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + return (new CountAction().execute(input).getCount()); + } + + /******************************************************************************* + ** Verify relationship drills. + *******************************************************************************/ + private void relationshipDrills(Object result) throws Exception + { + walkRelationship(new ObjectMapper().readTree(JsonUtils.toJson(result))); + } + + /******************************************************************************* + ** Verify walk relationship. + *******************************************************************************/ + private void walkRelationship(JsonNode node) throws Exception + { + if(node.isObject() && node.path("valueType").asText().equals("COUNT") && node.hasNonNull("drill")) + { + assertThat(drillCount(JsonUtils.toObject(node.get("drill").toString(), Drill.class))).as(node.toString()).isEqualTo(node.path("value").asLong()); + } + if(node.isContainerNode()) + { + for(JsonNode child : node) + { + walkRelationship(child); + } + } + } +} diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/RelationshipFallbackTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/RelationshipFallbackTest.java new file mode 100644 index 0000000..0c485a6 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/RelationshipFallbackTest.java @@ -0,0 +1,133 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import com.kingsrook.qbits.dataanalysis.result.CorrelationRequest; +import com.kingsrook.qbits.dataanalysis.result.CompareRequest; +import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface; +import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateCapability; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryAggregateAction; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.within; + +/******************************************************************************* + ** Force unavailable capabilities without replacing the actual core execution. + *******************************************************************************/ +class RelationshipFallbackTest extends EngineFixture +{ + /******************************************************************************* + ** Verify restore. + *******************************************************************************/ + @AfterEach + void restore() + { + QBackendModuleDispatcher.registerBackendModule(new MemoryBackendModule()); + } + + /******************************************************************************* + ** Verify core streams unsupported correlation and rejects truncation. + *******************************************************************************/ + @Test + void coreStreamsUnsupportedCorrelationAndRejectsTruncation() throws Exception + { + var request = new CorrelationRequest().withTableName("orders").withFields(List.of("id", "amount", "quantity")); + var nativeResult = service.correlation(request); + TrackingModule.fallback = true; + QBackendModuleDispatcher.registerBackendModule(new TrackingModule()); + var streamed = service.correlation(request); + for(int i = 0; i < nativeResult.getCells().size(); i++) + { + assertThat(streamed.getCells().get(i).getR()).isCloseTo(nativeResult.getCells().get(i).getR(), within(1e-12)); + assertThat(streamed.getCells().get(i).getN()).isEqualTo(nativeResult.getCells().get(i).getN()); + } + assertThat(service.compare(new CompareRequest().withTableName("orders").withFields(List.of("amount"))).getFields().getFirst().getSignificance().getPValue()).isEqualTo(1); + config.withDefaultMaxRows(3); + service.getConfigResolver().invalidate(); + assertThatThrownBy(() -> service.correlation(request)).isInstanceOf(QBadRequestException.class).hasMessageContaining("streamed row limit"); + } + + /******************************************************************************* + ** Verify all correlations use one aggregate call. + *******************************************************************************/ + @Test + void allCorrelationsUseOneAggregateCall() throws Exception + { + TrackingModule.fallback = false; + TrackingModule.calls.set(0); + QBackendModuleDispatcher.registerBackendModule(new TrackingModule()); + service.correlation(new CorrelationRequest().withTableName("orders").withFields(List.of("id", "amount", "quantity"))); + assertThat(TrackingModule.calls.get()).isEqualTo(1); + } + + /******************************************************************************* + ** Verify backend capability execution. + *******************************************************************************/ + public static class TrackingModule extends MemoryBackendModule + { + static boolean fallback; + static final AtomicInteger calls = new AtomicInteger(); + + /******************************************************************************* + ** Verify get aggregate interface. + *******************************************************************************/ + @Override + public AggregateInterface getAggregateInterface() + { + return (new MemoryAggregateAction() + { + /******************************************************************************* + ** Verify get supported capabilities. + *******************************************************************************/ + @Override + public Set getSupportedCapabilities(AggregateInput input) + { + return (fallback ? Set.of() : super.getSupportedCapabilities(input)); + } + + /******************************************************************************* + ** Verify execute. + *******************************************************************************/ + @Override + public AggregateOutput execute(AggregateInput input) throws com.kingsrook.qqq.backend.core.exceptions.QException + { + if(!fallback && input.getTableName().equals("orders")) + { + calls.incrementAndGet(); + assertThat(input.getAggregates()).hasSize(6).allSatisfy(a -> assertThat(a.getOperator()).isEqualTo(AggregateOperator.CORR)); + } + return (super.execute(input)); + } + }); + } + } +} diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/StatisticsTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/StatisticsTest.java new file mode 100644 index 0000000..34594a9 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/StatisticsTest.java @@ -0,0 +1,148 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis; + +import com.kingsrook.qbits.dataanalysis.stats.SpecialFunctions; +import com.kingsrook.qbits.dataanalysis.stats.Statistics; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.within; + +/******************************************************************************* + ** Independent SciPy 1.18.1 references, never computed with production helpers. + *******************************************************************************/ +class StatisticsTest +{ + /******************************************************************************* + ** Verify welch references. + *******************************************************************************/ + @Test + void welchReferences() + { + // scipy.stats.ttest_ind_from_stats(15, sqrt(87.5), 13, 12, sqrt(39), 11, equal_var=False) + var result = Statistics.welch(13, 15, 87.5, 11, 12, 39); + assertThat(result.statistic()).isCloseTo(0.9358461935556048, within(1e-12)); + assertThat(result.pValue()).isCloseTo(0.35999818693244234, within(1e-12)); + // scipy.stats.ttest_ind([1,3,4,6,11,13,15,19,22,24,25,26,26], [2,4,6,9,11,13,14,15,18,19,21], equal_var=False).df + assertThat(result.df()).isCloseTo(20.984611233429924, within(1e-12)); + assertThat(result.sentence()).isEqualTo("No significant difference (p = 0.36)"); + // scipy.stats.ttest_ind_from_stats(100, 2, 30, 104, 3, 25, equal_var=False) + assertThat(Statistics.welch(30, 100, 4, 25, 104, 9).pValue()).isCloseTo(1.2223199525471597e-6, within(1e-16)); + assertThat(Statistics.welch(1, 0, 1, 2, 0, 1).sentence()).isEqualTo("Not applicable: too few records"); + assertThat(Statistics.welch(2, 0, 0, 2, 1, 0).applicable()).isFalse(); + assertThat(Statistics.welch(2, 0, -1, 2, 1, 1).applicable()).isFalse(); + assertThat(Statistics.welch(2, Double.NaN, 1, 2, 1, 1).applicable()).isFalse(); + assertThat(Statistics.welch(20, 1, 1, 20, 1, 1).pValue()).isEqualTo(1); + } + + /******************************************************************************* + ** Verify proportion reference and significance boundary. + *******************************************************************************/ + @Test + void proportionReferenceAndSignificanceBoundary() + { + // p=(30+45)/(150+200); z=(30/150-45/200)/sqrt(p*(1-p)*(1/150+1/200)); 2*scipy.stats.norm.sf(abs(z)) + var result = Statistics.proportions(30, 150, 45, 200); + assertThat(result.statistic()).isCloseTo(-0.564076074817766, within(1e-12)); + assertThat(result.pValue()).isCloseTo(0.5727023597783104, within(1e-12)); + assertThat(Statistics.sentence(0.05)).isEqualTo("No significant difference (p = 0.05)"); + assertThat(Statistics.sentence(0.003)).isEqualTo("A and B differ (p = 0.003)"); + assertThat(Statistics.sentence(0.41)).isEqualTo("No significant difference (p = 0.41)"); + assertThat(Statistics.proportions(1, 1, 1, 2).applicable()).isFalse(); + assertThat(Statistics.proportions(0, 100, 0, 100).applicable()).isFalse(); + assertThat(Statistics.proportions(-1, 20, 10, 20).applicable()).isFalse(); + assertThat(Statistics.proportions(21, 20, 10, 20).applicable()).isFalse(); + } + + /******************************************************************************* + ** Verify chi square reference and pooling. + *******************************************************************************/ + @Test + void chiSquareReferenceAndPooling() + { + // scipy.stats.chi2_contingency([[30,20],[10,40]], correction=False) + var result = Statistics.chiSquare(new double[][] {{30, 20}, {10, 40}}, false); + assertThat(result.statistic()).isCloseTo(16.666666666666668, within(1e-12)); + assertThat(result.df()).isEqualTo(1); + assertThat(result.pValue()).isCloseTo(4.455709060405612e-5, within(1e-14)); + // scipy.stats.contingency.association([[30,20],[10,40]], method='cramer', correction=False) + assertThat(result.cramersV()).isCloseTo(0.408248290463863, within(1e-12)); + // scipy.stats.chi2_contingency([[20,30,50],[30,40,30]], correction=False) + assertThat(Statistics.chiSquare(new double[][] {{20, 30, 50}, {30, 40, 30}}, true).pValue()).isCloseTo(0.01478287719483942, within(1e-12)); + // Ten rare columns pool into Other, yielding [[40,10],[20,30]]; scipy.stats.chi2_contingency(..., correction=False). + double[][] rare = {{40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, {20, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3}}; + assertThat(Statistics.chiSquare(rare, true).pValue()).isCloseTo(4.455709060405612e-5, within(1e-14)); + // scipy.stats.chi2_contingency([[40,10],[20,30],[4,8]], correction=False); four rare rows merge into Other. + assertThat(Statistics.chiSquare(new double[][] {{40, 10}, {20, 30}, {1, 2}, {1, 2}, {1, 2}, {1, 2}}, false).pValue()) + .isCloseTo(5.993665986784811e-5, within(1e-14)); + assertThat(Statistics.chiSquare(new double[][] {{3, 3}, {3, 3}}, true).applicable()).isFalse(); + assertThat(Statistics.chiSquare(new double[][] {{20, 1}, {20, 1}}, true).applicable()).isFalse(); + assertThat(Statistics.chiSquare(new double[][] {{1, 0}, {10, 10}}, true).applicable()).isFalse(); + assertThat(Statistics.chiSquare(new double[][] {{10}, {10}}, true).applicable()).isFalse(); + assertThat(Statistics.chiSquare(new double[][] {{10, 10}, {10, -1}}, true).applicable()).isFalse(); + assertThat(Statistics.chiSquare(new double[][] {{10, 10}, {10}}, true).applicable()).isFalse(); + } + + /******************************************************************************* + ** Verify descriptive references. + *******************************************************************************/ + @Test + void descriptiveReferences() + { + // scipy.stats.pearsonr([1,2,3,4,5], [2,1,4,3,6]).statistic + assertThat(Statistics.pearson(new double[] {1, 2, 3, 4, 5}, new double[] {2, 1, 4, 3, 6})).isCloseTo(0.8219949365267865, within(1e-14)); + // scipy.special.rel_entr(q,p).sum() + scipy.special.rel_entr(p,q).sum(), p=[.2,.3,.5], q=[.3,.3,.4] + assertThat(Statistics.psi(new double[] {20, 30, 50}, new double[] {30, 30, 40})).isCloseTo(0.06286086594223739, within(1e-14)); + assertThat(Statistics.psi(new double[] {0, 1}, new double[] {1, 0})).isFinite(); + assertThat(Statistics.psi(new double[] {0, 0}, new double[] {1, 1})).isNull(); + assertThat(Statistics.psi(new double[] {-1, 1}, new double[] {1, 1})).isNull(); + assertThat(Statistics.psi(new double[] {1}, new double[] {1, 1})).isNull(); + assertThat(Statistics.pearson(new double[] {1, 1}, new double[] {2, 3})).isNull(); + assertThat(Statistics.pearson(new double[] {1}, new double[] {2})).isNull(); + assertThat(Statistics.pearson(new double[] {1, Double.NaN}, new double[] {2, 3})).isNull(); + } + + /******************************************************************************* + ** Verify distribution tails. + *******************************************************************************/ + @Test + void distributionTails() + { + // scipy.stats.t.cdf(0, 10), t.cdf(1, 1), t.cdf(-1, 1); scipy.stats.chi2.cdf(2, 4) + assertThat(SpecialFunctions.studentTCdf(0, 10)).isEqualTo(.5); + assertThat(SpecialFunctions.studentTCdf(1, 1)).isCloseTo(.75, within(1e-13)); + assertThat(SpecialFunctions.studentTCdf(-1, 1)).isCloseTo(.25, within(1e-13)); + assertThat(SpecialFunctions.chiSquareCdf(2, 4)).isCloseTo(.2642411176571153, within(1e-13)); + // scipy.special.betainc(2,5,.2), betainc(2,5,.8), betainc(.5,.5,.5), betainc(100,10,.02) + assertThat(SpecialFunctions.beta(.2, 2, 5)).isCloseTo(.3446400000000002, within(1e-13)); + assertThat(SpecialFunctions.beta(.8, 2, 5)).isCloseTo(.9984, within(1e-13)); + assertThat(SpecialFunctions.beta(.5, .5, .5)).isCloseTo(.5, within(1e-13)); + assertThat(SpecialFunctions.beta(.02, 100, 10)).isCloseTo(4.514221366229434e-158, within(1e-169)); + assertThat(SpecialFunctions.beta(0, 2, 5)).isZero(); + assertThat(SpecialFunctions.beta(1, 2, 5)).isEqualTo(1); + // scipy.special.gammaincc(2,1), gammaincc(2,8), gammaincc(.5,2), gammaincc(20,18) + assertThat(SpecialFunctions.gammaQ(2, 1)).isCloseTo(.7357588823428847, within(1e-13)); + assertThat(SpecialFunctions.gammaQ(2, 8)).isCloseTo(.003019163651122605, within(1e-13)); + assertThat(SpecialFunctions.gammaQ(.5, 2)).isCloseTo(.04550026389635857, within(1e-13)); + assertThat(SpecialFunctions.gammaQ(20, 18)).isCloseTo(.6509161279807018, within(1e-13)); + assertThat(SpecialFunctions.gammaQ(2, 0)).isEqualTo(1); + } +} -- 2.54.0 From 43c757186472f47569eefe18363453ea85958b3d Mon Sep 17 00:00:00 2001 From: James Maes Date: Tue, 13 Oct 2026 20:34:19 -0500 Subject: [PATCH 06/10] feat(watches): anomaly watches through the alerts SPI, contingent pickers Watches evaluate a metric per time bucket as their owner (SPIKE, DROP, OUT_OF_RANGE, NEW_CATEGORY, NULL_RATE_CREEP with robust baselines) on a hidden scheduled process, raise and resolve data.analysis.anomaly alerts through QAlerts, and offer Evaluate now / Enable / Disable record actions and a preview endpoint (extension 1.3.0). Pivot and crosstab labels are human (Yes/No, Average Annual Salary). Contingent fields: a watch's saved analysis depends on its table, table names use the core tables PVS, and analyzable/excluded fields pick from the chosen table's fields. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 9 + README.md | 39 +- .../dataanalysis/DataAnalysisQBitConfig.java | 18 + .../DataAnalysisQBitProducer.java | 12 + .../dataanalysis/DataAnalysisRuntime.java | 10 + .../dataanalysis/DataAnalysisSchema.java | 1 + .../dataanalysis/DataAnalysisService.java | 13 + .../dataanalysis/api/DataAnalysisApi.java | 43 +- .../dataanalysis/api/DataAnalysisSchemas.java | 1 + .../qbits/dataanalysis/api/WatchSchemas.java | 126 +++++ .../dataanalysis/app/DataAnalysisHome.java | 21 + .../dataanalysis/engine/AnalysisLabels.java | 4 + .../dataanalysis/engine/PivotExplorer.java | 37 +- .../engine/RelationshipSupport.java | 2 +- .../dataanalysis/metadata/WatchProcesses.java | 93 ++++ ...ataAnalysisFieldPossibleValueProvider.java | 112 +++++ .../model/DataAnalysisTables.java | 46 +- .../dataanalysis/processes/WatchActions.java | 186 +++++++ .../qbits/dataanalysis/watch/WatchBucket.java | 32 ++ .../dataanalysis/watch/WatchCustomizer.java | 97 ++++ .../dataanalysis/watch/WatchDefinition.java | 152 ++++++ .../dataanalysis/watch/WatchDetectors.java | 131 +++++ .../dataanalysis/watch/WatchEvaluator.java | 236 +++++++++ .../dataanalysis/watch/WatchPreview.java | 31 ++ .../dataanalysis/watch/WatchRecords.java | 141 ++++++ .../qbits/dataanalysis/watch/WatchSeries.java | 278 +++++++++++ .../dataanalysis/watch/WatchVerdict.java | 38 ++ .../openapi/qqq-v1/data-analysis.yaml | 83 +++- .../dataanalysis/DataAnalysisApiTest.java | 43 +- .../dataanalysis/DataAnalysisHomeTest.java | 6 +- .../DataAnalysisMetadataTest.java | 4 +- .../DataAnalysisPossibleValuesTest.java | 167 +++++++ .../qbits/dataanalysis/PivotContractTest.java | 18 + .../dataanalysis/WatchDetectorsTest.java | 174 +++++++ .../qbits/dataanalysis/WatchesTest.java | 460 ++++++++++++++++++ 35 files changed, 2830 insertions(+), 34 deletions(-) create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/api/WatchSchemas.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/metadata/WatchProcesses.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/model/DataAnalysisFieldPossibleValueProvider.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/processes/WatchActions.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchBucket.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchCustomizer.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchDefinition.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchDetectors.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchEvaluator.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchPreview.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchRecords.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchSeries.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchVerdict.java create mode 100644 src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisPossibleValuesTest.java create mode 100644 src/test/java/com/kingsrook/qbits/dataanalysis/WatchDetectorsTest.java create mode 100644 src/test/java/com/kingsrook/qbits/dataanalysis/WatchesTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index f831652..744d857 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ Following [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and [Semantic ### Added +- Phase e anomaly watches: owner-locked configuration, validated metrics/detector parameters, complete calendar buckets, five pure detectors, and scheduled evaluation through owner sessions. +- Core alerts SPI type `data.analysis.anomaly` (core disallows the spec's underscore), stable watch keys, relative drill links, raise/recovery transitions, and persisted outcomes without an alerts implementation. +- Standard streamed ETL Evaluate now / Enable / Disable record actions and firing-first linked watches on the Analyses home. +- Read-only watch preview HTTP endpoint, semantic chart/baseline annotations, extension 1.3.0 and regenerated OpenAPI fragment. +- Detector edge cases, real-row owner/security/alert transition tests, calendar boundary and action tests, and HTTP preview coverage. + - Phase d compare, shift, correlation, scatter and crosstab service operations and HTTP specs; extension version 1.2.0 with typed `DataAnalysis*V1` schemas and regenerated OpenAPI. - Pure-Java two-proportion, Welch, chi-square/Cramer V, PSI and Pearson statistics, including regularized incomplete beta/gamma and independent SciPy references. - Core-pushed numeric moments and batched correlations, sample trend lines, calendar distribution shifts, category pooling, privacy suppression and exact count drills. @@ -23,6 +29,9 @@ Following [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and [Semantic ### Changed +- Boolean pivot/crosstab headers now say Yes / No / (Blank). Pivot measures use human labels such as Count, Average Amount, Sum of Amount, Distinct Email, Median Amount and 90th percentile of Amount. +- Watch DDL includes owner/date fields and last value/message; existing databases need an owner/date backfill before enabling watches. + - Scalable estimates sample each native aggregate without primary-key IN lists, scale counts using observed sample sizes, retain exact population totals and unscaled ratios, and drill over the full population. Portable estimates use one primary-key boundary and report unscaled prefix counts. - Target QQQ 4.1.0-RC.6; local verification uses `-Dqqq.snapshot.version=4.1.0-DATAANALYSIS-SNAPSHOT`. diff --git a/README.md b/README.md index fe664f0..411a05c 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Statistical analysis and data dives on any QQQ table, filter or saved view: prof The host API provides column and table profiles, pivots, comparisons, distribution shifts, correlations, scatter samples, crosstabs, distinct-value pages, duplicate groups, drill-through filters, privacy suppression, and asynchronous exact profile runs with a persistent per-user cache. Metadata, six visible tables, owner locks, PostgreSQL DDL generation and the Analyses home are also available. -Thirteen HTTP operations are available under `/qqq/v1/qbit/data-analysis`; the Analyze UI is a later phase. Pivot and relationship backends are available. Anomalies, saved-analysis execution, scheduling and exports remain later phases. +Fourteen HTTP operations are available under `/qqq/v1/qbit/data-analysis`; the Analyze UI is a later phase. Pivot and relationship backends are available. Anomaly watches, their scheduled evaluator and alert transitions are available. Saved-analysis execution, scheduled delivery and exports remain later phases. ## Requirements @@ -47,6 +47,8 @@ QQQ names and physical SQL table names are identical. `backendName` in configura Only Saved Analyses, Watches, Analysis Settings and Analysis Runs appear in navigation. Schedules, stars and linked watches appear on saved records. Enum PVS cover analysis kind, chart kind, detector, watch state, run status, schedule frequency and format. JSON definitions and results are stored as text. Profile, distinct-value and duplicate requests have typed Java contracts and are validated by the facade. +The four `table_name` fields use core's `tables` possible-value source; an existing host source is preserved. A watch's `saved_id` options are filtered by its selected `table_name`. Analysis Settings' `analyzable_fields` and `excluded_fields` use a custom field picker scoped to the selected table. Core currently has no multi-select PVS field, so these settings use a single PVS with comma-separated TEXT storage (for example, `amount,status`); the configuration resolver continues to parse those lists. Explicit dependency metadata awaits core's `possibleValueSourceDependsOn` support. + ## Permissions | Permission | Grants | @@ -138,7 +140,7 @@ Count drills combine an untouched copy of the request filter with their producin ## HTTP endpoints -Extension `data-analysis` version **1.2.0**, prefix `/qbit/data-analysis` inside `/qqq/v1`: +Extension `data-analysis` version **1.3.0**, prefix `/qbit/data-analysis` inside `/qqq/v1`: | Method | Path after `/qqq/v1` | Body or purpose | | --- | --- | --- | @@ -155,10 +157,11 @@ Extension `data-analysis` version **1.2.0**, prefix `/qbit/data-analysis` inside | POST | `/qbit/data-analysis/tables/{table}/crosstab` | `{rowField, columnField, filter, limit}`; default/maximum 20 categories per axis plus Other | | POST | `/qbit/data-analysis/runs` | `{operation: COLUMN_PROFILE or TABLE_PROFILE, tableName, request}` returns `{runId, status}` | | GET | `/qbit/data-analysis/runs/{runId}` | Status, timestamps, error, typed `columnProfile` or `tableProfile` | +| POST | `/qbit/data-analysis/watches/{id}/preview` | No body; complete bucket series, `TIME_SERIES` chart with baseline annotations and detector verdict; no writes or alerts | Every operation uses the request session through `DataAnalysisService`. Capabilities returns 200 with `canAnalyze: false` and no fields when denied. Other permission denials return 403, bad bodies or unavailable fields return 400, and unknown tables return 404. Core middleware supplies plain messages in its standard `{error: message}` JSON envelope. Run rows remain owner-locked. -Requests and drills reuse core's `QueryFilter` JSON shape and schema reference. Versioned `DataAnalysis...V1` schemas cover profiles, metrics, drills, charts, series, points, field summaries, distinct and duplicate results, capabilities and runs. The [OpenAPI fragment](src/main/resources/openapi/qqq-v1/data-analysis.yaml) is checked against regeneration. The deployed `/qqq/v1/openapi.json` and `UiApiInvoker.listOperations()` include all eight operations. +Requests and drills reuse core's `QueryFilter` JSON shape and schema reference. Versioned `DataAnalysis...V1` schemas cover profiles, metrics, drills, charts, series, points, field summaries, distinct and duplicate results, capabilities and runs. The [OpenAPI fragment](src/main/resources/openapi/qqq-v1/data-analysis.yaml) is checked against regeneration. The deployed `/qqq/v1/openapi.json` and `UiApiInvoker.listOperations()` include all fourteen operations. Regenerate deliberately after an API change: @@ -202,6 +205,34 @@ Shift uses session-timezone calendar buckets. Only buckets whose end has passed The minimum-group-size guard removes small numeric summaries, suppresses crosstab cells and revealing marginal shares, disables drills on privacy-merged Other groups, and suppresses scatter points below the minimum. Statistical tests and PSI are unavailable when protected values would contribute. No runtime statistics dependency is added; beta/gamma, Welch, chi-square, proportions, PSI and Pearson are pure Java, checked against independently computed SciPy references. +## Anomaly watches + +Watches are ordinary owner-locked `data_analysis_watch` rows. Create and edit them through core table CRUD with `data_analysis.watch.read`, `.insert`, `.edit`, and `.delete`. Standard table/record actions **Evaluate now**, **Enable**, and **Disable** use streamed ETL Review and Result summaries. They never appear in navigation. The home lists firing watches first and links to their records. + +A watch has `owner_user_id`, `date_field`, `bucket` (`HOUR`, `DAY`, `WEEK`, `MONTH`), `detector`, `parameters_json`, `severity` (`INFO`, `WARNING`, `ERROR`, `CRITICAL`) and `enabled`. It targets either: + +- `table_name`, optional core `filter_json`, and `metric_json`, such as `{"operator":"COUNT"}`, `{"operator":"SUM","field":"amount"}`, `{"operator":"AVG","field":"amount"}`, `{"operator":"NULL_SHARE","field":"email"}`, or `{"operator":"CATEGORY_SET","field":"status"}`. +- `saved_id`: the owner's saved table and filter, with an explicit watch `metric_json` or one saved definition `metric` / single pivot `values` measure. The date field and bucket remain explicit on the watch. Ambiguous saved measures are rejected; saved filters cannot be overridden by `filter_json` on the watch. + +Detector parameters are validated by core pre-insert/update customizers, including partial updates. Unknown parameters and invalid metric/detector combinations return clear record errors. + +| Detector | Parameters and behavior | +| --- | --- | +| SPIKE / DROP | `k=14`, `threshold=3.5`, `relativeChange=0.5`; robust z = `0.6745 * (latest - median) / MAD`, strict directional threshold. With zero MAD, compare relative change against the absolute median. A zero baseline fires only for a nonzero change in the requested direction. | +| OUT_OF_RANGE | Required `min` and `max`; fire outside the inclusive range. Needs only the latest bucket. | +| NEW_CATEGORY | `k=14`; requires `CATEGORY_SET`; compare latest categories with the union of previous buckets. Null is excluded from the category set. | +| NULL_RATE_CREEP | `k=14`, `delta=0.05`, `n=3`; requires `NULL_SHARE`; fire for a rise above the baseline median greater than delta, or `n` consecutive increases (`n+1` values). | + +`k` and `n` are bounded to 1–1000. The trailing window contains at most `k+1` complete buckets, excludes the current bucket, and follows the owner's time zone (Monday-based weeks). Leading buckets with no observed rows do not invent history. After the first observed bucket, missing COUNT/SUM/category buckets are zero/empty; AVG and NULL_SHARE with no population are unavailable. Too little history, incomplete streamed results, unavailable fields, and privacy-suppressed buckets never cause a recovery. No suppressed series, categories, values or drills leave the computation. All source reads enforce READ, Analyze, record locks and FULL field access, including filter fields. + +`data_analysis_evaluate_watches` is hidden and repeats every 900 seconds when `schedulerName` is configured. Give the scheduler session its standard process permission, `data_analysis_evaluate_watches.hasAccess`. Inventory and status writes use grants restricted to watch rows; source aggregation always switches to the owner. The default integration calls the host authentication module's `createAutomatedSessionForUser`. Hosts can instead configure `withOwnerSessionSupplier(userId -> session)`. The session must identify that owner and include the `data_analysis_user` owner key plus their current permissions. Missing/mismatched sessions record **Owner session unavailable**; there is no system-session fallback. Manual Evaluate now can also evaluate a disabled watch. + +A transition from OK to FIRING raises through core `QAlerts`, with the watch record reference and a relative table-query `AlertLink`; recovery resolves the same `data_analysis_watch:` key. The declared type is **`data.analysis.anomaly`**, labelled **Data anomaly**. This spelling is necessary because the supplied core SPI validator rejects the design's `data_analysis.anomaly` underscore. No qbit-alerts dependency is required. With no sink installed, core logs the signal and the watch row still records the outcome. Repeated FIRING/OK evaluations do not send duplicate transitions. Calls are serialized within each runtime; hosts should run this scheduler on one node (cross-node transactional coordination is not provided). + +`state`, `alert_key`, `last_evaluated_at`, `last_value` and `last_message` describe the most recent attempt. Skips clear the last value and leave the existing OK/FIRING state intact. Preview uses the requesting viewer's permissions and never updates these fields. + +`DataAnalysisSchema.postgresScript(config)` generates the new fields and owner requirement. Existing installations must add `date_field VARCHAR(250)`, `owner_user_id VARCHAR(250)`, `last_value NUMERIC`, and `last_message TEXT`, then backfill each watch's actual owner/date field before making the first two columns NOT NULL and enabling evaluation. The CREATE-IF-NOT-EXISTS generator does not migrate existing tables. + ## Adopting in another app 1. Add a backend, install the producer and run the usual `QInstanceValidator`. @@ -210,7 +241,7 @@ The minimum-group-size guard removes small numeric summaries, suppresses crossta 4. Review per-table include/exclude sets, limits and minimum group sizes before enabling analysis. Integrate Saved Analyses and Watches with host ESB/Quick Search when those optional host features are installed; this scaffold does not add those dependencies. 5. Use the Analyses home as the owner of analysis data. Other dashboards should link there until saved-analysis widgets are implemented. -Phase **b** profiles, drill-through, runs/cache and HTTP endpoints, phase **c** pivot backend, and phase **d** compare/relationships backend are implemented. Remaining work: **b** analysis UI; **c** pivot UI; **d** comparison/relationship UI; **e** watch evaluation and alerts; **f** saved-analysis execution, sharing, stars integration, schedules, chart widgets, full home and exports. The current home summarizes stored rows; the host facade executes profiles. The home does not send deliveries. +Phase **b** profiles, drill-through, runs/cache and HTTP endpoints, phase **c** pivot backend, phase **d** compare/relationships backend, and phase **e** anomaly watches are implemented. Remaining work: **b** analysis UI; **c** pivot UI; **d** comparison/relationship UI; **f** saved-analysis execution, sharing, stars integration, schedules, chart widgets, full home and exports. The current home summarizes stored rows; the host facade executes profiles. The home does not send deliveries. ## Build diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitConfig.java b/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitConfig.java index 6120fa9..ea24926 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitConfig.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitConfig.java @@ -39,6 +39,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig; public class DataAnalysisQBitConfig implements QBitConfig { private Supplier fieldAnalysisAccessResolver = CoreTierFieldAccessResolver::defaultResolver; + private transient java.util.function.Function ownerSessionSupplier; private String backendName; private String schedulerName; private String appName = "dataAnalysis"; @@ -408,4 +409,21 @@ public class DataAnalysisQBitConfig implements QBitConfig this.fieldAnalysisAccessResolver = java.util.Objects.requireNonNull(resolver); return (this); } + + /******************************************************************************* + ** Optional host adapter for scheduled owner permissions and record-lock keys. + *******************************************************************************/ + public java.util.function.Function getOwnerSessionSupplier() + { + return (ownerSessionSupplier); + } + + /******************************************************************************* + ** Without an adapter, use the host authentication module's automated session. + *******************************************************************************/ + public DataAnalysisQBitConfig withOwnerSessionSupplier(java.util.function.Function supplier) + { + ownerSessionSupplier = supplier; + return (this); + } } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitProducer.java b/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitProducer.java index 24e95aa..4e38760 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitProducer.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitProducer.java @@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.TableFeaturePer import com.kingsrook.qqq.backend.core.model.metadata.permissions.TableFeaturePermissionsMetaData; import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData; import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.tables.TablesPossibleValueSourceMetaDataProvider; /******************************************************************************* @@ -129,9 +130,20 @@ public class DataAnalysisQBitProducer implements QBitMetaDataProducer "INTEGER"; case LONG -> "BIGINT"; + case DECIMAL -> "NUMERIC"; case BOOLEAN -> "BOOLEAN"; case DATE_TIME -> "TIMESTAMP WITHOUT TIME ZONE"; case TEXT -> "TEXT"; diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisService.java b/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisService.java index 6929f11..9d2575c 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisService.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisService.java @@ -338,4 +338,17 @@ public class DataAnalysisService validate(request); return (new ShiftExplorer(security, configs.resolve(request.getTableName()), request.getTableName()).shift(request)); } + + /******************************************************************************* + ** Watch aggregates use the same current-user table and FULL-field policy. + *******************************************************************************/ + public com.kingsrook.qbits.dataanalysis.watch.WatchPreview watchPreview(com.kingsrook.qqq.backend.core.model.data.QRecord row, java.time.Instant now) throws QException + { + var resolved = com.kingsrook.qbits.dataanalysis.watch.WatchRecords.resolve(row); + String table = resolved.getValueString("table_name"); + security.table(table); + var definition = com.kingsrook.qbits.dataanalysis.watch.WatchDefinition.parse(resolved); + return (new com.kingsrook.qbits.dataanalysis.watch.WatchSeries(security, configs.resolve(table), table) + .preview(definition, com.kingsrook.qbits.dataanalysis.watch.WatchRecords.filter(resolved), now)); + } } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisApi.java b/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisApi.java index b84e8c2..ebd7532 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisApi.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisApi.java @@ -76,11 +76,11 @@ public class DataAnalysisApi *******************************************************************************/ public static MiddlewareSpecExtension extension() { - return (new MiddlewareSpecExtension().withName("data-analysis").withSemanticVersion("1.2.0") - .withDescription("Profiles, pivots, comparisons, distribution shifts, correlations, scatter samples and crosstabs as the signed-in user.") + return (new MiddlewareSpecExtension().withName("data-analysis").withSemanticVersion("1.3.0") + .withDescription("Profiles, pivots, comparisons, distribution shifts, correlations, scatter samples, crosstabs and anomaly watch previews as the signed-in user.") .withSpec(CapabilitiesSpec.class).withSpec(ColumnProfileSpec.class).withSpec(DistinctValuesSpec.class) .withSpec(CompareSpec.class).withSpec(ShiftSpec.class).withSpec(CorrelationSpec.class).withSpec(ScatterSpec.class).withSpec(CrosstabSpec.class) - .withSpec(PivotSpec.class).withSpec(TableProfileSpec.class).withSpec(DuplicatesSpec.class).withSpec(StartRunSpec.class).withSpec(GetRunSpec.class)); + .withSpec(WatchPreviewSpec.class).withSpec(PivotSpec.class).withSpec(TableProfileSpec.class).withSpec(DuplicatesSpec.class).withSpec(StartRunSpec.class).withSpec(GetRunSpec.class)); } /******************************************************************************* @@ -123,9 +123,9 @@ public class DataAnalysisApi @Override public List defineRequestParameters() { - String name = path.contains("{table}") ? "table" : path.contains("{runId}") ? "runId" : null; + String name = path.contains("{table}") ? "table" : path.contains("{runId}") ? "runId" : path.contains("{id}") ? "id" : null; return (name == null ? List.of() : List.of(new Parameter().withName(name).withIn(In.PATH).withRequired(true) - .withDescription(name.equals("table") ? "QQQ table name to analyze." : "Owner-scoped analysis run id.").withSchema(new Schema().withType(Type.STRING)))); + .withDescription(name.equals("table") ? "QQQ table name to analyze." : "Owner-scoped analysis run or watch id.").withSchema(new Schema().withType(Type.STRING)))); } /******************************************************************************* @@ -757,4 +757,37 @@ public class DataAnalysisApi return (transport(DataAnalysisRuntime.service().crosstab(request), RelationshipSchemas.DataAnalysisCrosstabV1.class)); } } + + /******************************************************************************* + ** Read-only watch preview; standard table endpoints own watch CRUD. + *******************************************************************************/ + public static class WatchPreviewSpec extends ProfileSpec + { + /******************************************************************************* + ** Publish a side-effect-free detector preview for a saved watch. + *******************************************************************************/ + public WatchPreviewSpec() + { + super("/watches/{id}/preview", HttpMethod.POST, "Preview an anomaly watch", + "Compute complete time buckets and a detector verdict under the signed-in user's READ, Analyze and FULL-field permissions. Includes a baseline band annotation. Does not update the watch or raise alerts.", null, "WatchPreview"); + } + + /******************************************************************************* + ** Missing or inaccessible watch ids return the same not-found response. + *******************************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + Integer id; + try + { + id = Integer.valueOf(input.getPathParam("id")); + } + catch(NumberFormatException e) + { + throw new QBadRequestException("Watch id must be an integer."); + } + return (transport(DataAnalysisRuntime.watches().preview(id, java.time.Instant.now()), WatchSchemas.DataAnalysisWatchPreviewV1.class)); + } + } } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java b/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java index 8c9c5b8..2a0d845 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java @@ -115,6 +115,7 @@ public class DataAnalysisSchemas result.put("DataAnalysisPivotCellV1", new DataAnalysisPivotCellV1().toSchema()); result.put("DataAnalysisPivotV1", new DataAnalysisPivotV1().toSchema()); result.putAll(RelationshipSchemas.components()); + result.putAll(WatchSchemas.components()); return (result); } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/api/WatchSchemas.java b/src/main/java/com/kingsrook/qbits/dataanalysis/api/WatchSchemas.java new file mode 100644 index 0000000..8142911 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/api/WatchSchemas.java @@ -0,0 +1,126 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.api; + +import java.io.Serializable; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.middleware.javalin.schemabuilder.ToSchema; +import com.kingsrook.qqq.openapi.model.Schema; +import com.kingsrook.qqq.openapi.model.Type; + +/******************************************************************************* + ** Standalone watch preview contracts referencing the existing chart and drill. + *******************************************************************************/ +public class WatchSchemas +{ + /******************************************************************************* + ** Register every referenced version-one watch response component. + *******************************************************************************/ + public static Map components() + { + return (Map.of("DataAnalysisWatchPreviewV1", new DataAnalysisWatchPreviewV1().toSchema(), + "DataAnalysisWatchBucketV1", new DataAnalysisWatchBucketV1().toSchema(), + "DataAnalysisWatchVerdictV1", new DataAnalysisWatchVerdictV1().toSchema())); + } + + /******************************************************************************* + ** One complete bucket; suppressed series never expose categories or drills. + *******************************************************************************/ + public static class DataAnalysisWatchBucketV1 implements ToSchema + { + public Serializable start; + public Double value; + public List categories; + public DataAnalysisSchemas.DataAnalysisDrillV1 drill; + + /******************************************************************************* + ** Explicit nullable types and component references. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("start", new Schema().withType(Type.STRING).withDescription("Calendar date or hour-start instant in the user's time zone.")); + properties.put("value", new Schema().withType(Type.NUMBER).withNullable(true)); + properties.put("categories", new Schema().withType(Type.ARRAY).withItems(new Schema().withType(Type.STRING))); + properties.put("drill", new Schema().withRef("#/components/schemas/DataAnalysisDrillV1").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withProperties(properties)); + } + } + + /******************************************************************************* + ** Unevaluated means no state change; baseline bounds describe the annotation. + *******************************************************************************/ + public static class DataAnalysisWatchVerdictV1 implements ToSchema + { + public Boolean evaluated; + public Boolean firing; + public Double value; + public Double baseline; + public Double baselineLow; + public Double baselineHigh; + public String message; + public DataAnalysisSchemas.DataAnalysisDrillV1 drill; + + /******************************************************************************* + ** Carry numeric baseline bounds as well as the plain-English explanation. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("evaluated", new Schema().withType(Type.BOOLEAN).withDescription("False for insufficient history or privacy suppression; must not resolve an alert.")); + properties.put("firing", new Schema().withType(Type.BOOLEAN)); + for(String field : List.of("value", "baseline", "baselineLow", "baselineHigh")) + { + properties.put(field, new Schema().withType(Type.NUMBER).withNullable(true)); + } + properties.put("message", new Schema().withType(Type.STRING)); + properties.put("drill", new Schema().withRef("#/components/schemas/DataAnalysisDrillV1").withNullable(true)); + return (new Schema().withType(Type.OBJECT).withProperties(properties)); + } + } + + /******************************************************************************* + ** Complete preview, using the same semantic chart as other analysis endpoints. + *******************************************************************************/ + public static class DataAnalysisWatchPreviewV1 implements ToSchema + { + public List series; + public DataAnalysisSchemas.DataAnalysisChartV1 chart; + public DataAnalysisWatchVerdictV1 verdict; + + /******************************************************************************* + ** Stable references let clients vendor this fragment without middleware. + *******************************************************************************/ + @Override + public Schema toSchema() + { + Map properties = new LinkedHashMap<>(); + properties.put("series", new Schema().withType(Type.ARRAY).withItems(new Schema().withRef("#/components/schemas/DataAnalysisWatchBucketV1"))); + properties.put("chart", new Schema().withRef("#/components/schemas/DataAnalysisChartV1").withDescription("TIME_SERIES with baseline band annotations.")); + properties.put("verdict", new Schema().withRef("#/components/schemas/DataAnalysisWatchVerdictV1")); + return (new Schema().withType(Type.OBJECT).withProperties(properties)); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/app/DataAnalysisHome.java b/src/main/java/com/kingsrook/qbits/dataanalysis/app/DataAnalysisHome.java index fa345a9..e76f51f 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/app/DataAnalysisHome.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/app/DataAnalysisHome.java @@ -158,6 +158,27 @@ public class DataAnalysisHome extends AbstractWidgetRenderer { html.append("

    No watches yet.

    "); } + QueryInput watches = new QueryInput(DataAnalysisTables.WATCH).withShouldFetchHeavyFields(true) + .withFilter(new QQueryFilter().withOrderBy(new QFilterOrderBy("state", true)).withOrderBy(new QFilterOrderBy("id", false)).withLimit(5)); + watches.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + List recent = new QueryAction().execute(watches).getRecords(); + if(!recent.isEmpty()) + { + html.append(""); + } link(html, DataAnalysisTables.WATCH); } else diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisLabels.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisLabels.java index 3cf2229..6e39fe5 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisLabels.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisLabels.java @@ -85,6 +85,10 @@ public final class AnalysisLabels { return (null); } + if(field.getType() == com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType.BOOLEAN) + { + return (com.kingsrook.qqq.backend.core.utils.ValueUtils.getValueAsBoolean(value) ? "Yes" : "No"); + } String fallback = value.toString(); QPossibleValueSource source = QContext.getQInstance().getPossibleValueSource(field.getPossibleValueSourceName()); if(source == null || visited.size() >= 8 || !visited.add(source.getName())) diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/PivotExplorer.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/PivotExplorer.java index b1c23f5..b436fc2 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/PivotExplorer.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/PivotExplorer.java @@ -326,7 +326,42 @@ public class PivotExplorer measures.add(aggregate); ValueType type = operator == PivotMeasure.Operator.COUNT || operator == PivotMeasure.Operator.COUNT_DISTINCT ? ValueType.COUNT : numeric(field) ? ValueType.DECIMAL : field.getType() == QFieldType.DATE ? ValueType.DATE : field.getType() == QFieldType.DATE_TIME ? ValueType.DATE_TIME : ValueType.TEXT; - measureInfo.add(new PivotMeasureInfo().withKey("value" + index).withLabel(operator.name() + (field == null ? " records" : " " + field.getLabel())).withValueType(type)); + measureInfo.add(new PivotMeasureInfo().withKey("value" + index).withLabel(measureLabel(measure, field)).withValueType(type)); + } + + /******************************************************************************* + ** Human measure labels shared by headers, totals and cells. + *******************************************************************************/ + private static String measureLabel(PivotMeasure measure, QFieldMetaData field) + { + String label = field == null ? "" : field.getLabel(); + return (switch(measure.getOperator()) + { + case COUNT -> "Count"; + case AVG -> "Average " + label; + case SUM -> "Sum of " + label; + case COUNT_DISTINCT -> "Distinct " + label; + case MIN -> "Minimum " + label; + case MAX -> "Maximum " + label; + case PERCENTILE -> measure.getPercentile().compareTo(new BigDecimal("0.5")) == 0 ? "Median " + label + : ordinal(measure.getPercentile().multiply(BigDecimal.valueOf(100))) + " percentile of " + label; + }); + } + + /******************************************************************************* + ** English ordinal suffixes, including the eleventh through thirteenth. + *******************************************************************************/ + private static String ordinal(BigDecimal value) + { + int number = value.intValue(); + String suffix = number % 100 >= 11 && number % 100 <= 13 ? "th" : switch(number % 10) + { + case 1 -> "st"; + case 2 -> "nd"; + case 3 -> "rd"; + default -> "th"; + }; + return (value.stripTrailingZeros().toPlainString() + suffix); } /******************************************************************************* diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/RelationshipSupport.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/RelationshipSupport.java index aa9def8..aad7842 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/RelationshipSupport.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/RelationshipSupport.java @@ -177,7 +177,7 @@ public class RelationshipSupport extends StandInAnalysisBackend Serializable value = row.getGroupByValues().get(group); long n = ((Number) row.getAggregateValues().get(count)).longValue(); boolean hidden = n < config.getMinGroupSize(); - Category category = new Category(value, hidden ? "Other" : value == null ? "(blank)" : security.label(metadata, value), + Category category = new Category(value, hidden ? "Other" : value == null ? "(Blank)" : security.label(metadata, value), DrillFilters.value(field, value, metadata.getType() == QFieldType.STRING), hidden); if(hidden || result.size() >= limit) { diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/metadata/WatchProcesses.java b/src/main/java/com/kingsrook/qbits/dataanalysis/metadata/WatchProcesses.java new file mode 100644 index 0000000..4b127c7 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/metadata/WatchProcesses.java @@ -0,0 +1,93 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.metadata; + +import java.util.List; +import com.kingsrook.qbits.dataanalysis.DataAnalysisQBitConfig; +import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables; +import com.kingsrook.qbits.dataanalysis.processes.WatchActions; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; + +/******************************************************************************* + ** Watch record actions and one hidden scheduled process, never app children. + *******************************************************************************/ +public class WatchProcesses extends QBitComponentMetaDataProducer +{ + /******************************************************************************* + ** Build after the watch table has been registered. + *******************************************************************************/ + @Override + public int getSortOrder() + { + return (200); + } + + /******************************************************************************* + ** Component scanning also encounters unconfigured producers. + *******************************************************************************/ + @Override + public boolean isEnabled() + { + return (getQBitConfig() != null); + } + + /******************************************************************************* + ** Ordinary core actions provide selected records, Review, and Result screens. + *******************************************************************************/ + @Override + public MetaDataProducerMultiOutput produce(QInstance instance) + { + MetaDataProducerMultiOutput output = new MetaDataProducerMultiOutput(); + for(String name : List.of(WatchActions.EVALUATE, WatchActions.ENABLE, WatchActions.DISABLE)) + { + String label = WatchActions.EVALUATE.equals(name) ? "Evaluate now" : WatchActions.ENABLE.equals(name) ? "Enable" : "Disable"; + QProcessMetaData process = StreamedETLWithFrontendProcess.processMetaDataBuilder() + .withName(name).withLabel(label).withIcon(new QIcon("notifications_active")) + .withTableName(DataAnalysisTables.WATCH).withSourceTable(DataAnalysisTables.WATCH).withDestinationTable(DataAnalysisTables.WATCH) + .withExtractStepClass(ExtractViaQueryStep.class).withTransformStepClass(WatchActions.Review.class).withLoadStepClass(WatchActions.Result.class) + .withSupportsFullValidation(true).withDoFullValidation(true).withMinInputRecords(1) + .withPreviewMessage("Review the selected watches.").getProcessMetaData(); + // Table READ/EDIT and owner locks are enforced by extraction and both steps. + process.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED)); + output.add(process); + } + QProcessMetaData scheduled = new QProcessMetaData().withName(WatchActions.SCHEDULED).withLabel("Evaluate watches").withIsHidden(true) + .withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION).withPermissionBaseName(WatchActions.SCHEDULED)) + .withStep(new QBackendStepMetaData().withName("run").withCode(new QCodeReference(WatchActions.Scheduled.class))); + if(getQBitConfig().getSchedulerName() != null) + { + scheduled.withSchedule(new QScheduleMetaData().withSchedulerName(getQBitConfig().getSchedulerName()).withRepeatSeconds(900).withInitialDelaySeconds(900)); + } + output.add(scheduled); + return (output); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/model/DataAnalysisFieldPossibleValueProvider.java b/src/main/java/com/kingsrook/qbits/dataanalysis/model/DataAnalysisFieldPossibleValueProvider.java new file mode 100644 index 0000000..983474d --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/model/DataAnalysisFieldPossibleValueProvider.java @@ -0,0 +1,112 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.kingsrook.qbits.dataanalysis.model; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** Fields of the table selected in the settings form. Core has no multi-select + ** PVS field yet; settings retain their comma-separated TEXT representation. + *******************************************************************************/ +public class DataAnalysisFieldPossibleValueProvider implements QCustomPossibleValueProvider +{ + public static final String NAME = "data_analysis_fields"; + + + + /******************************************************************************* + ** Without form values the table is unknown; preserve names, including CSV lists. + *******************************************************************************/ + @Override + public QPossibleValue getPossibleValue(Serializable idValue) + { + String name = ValueUtils.getValueAsString(idValue); + return (name == null ? null : new QPossibleValue<>(name, name)); + } + + + + /******************************************************************************* + ** Offer only fields of the selected table, honoring search text and id lookups. + *******************************************************************************/ + @Override + public List> search(SearchPossibleValueSourceInput input) + { + List> values = new ArrayList<>(); + QTableMetaData table = resolveTable(input.getOtherValues()); + if(table == null) + { + return (values); + } + for(QFieldMetaData field : table.getFields().values()) + { + String label = field.getLabel() == null ? field.getName() : field.getLabel(); + QPossibleValue value = new QPossibleValue<>(field.getName(), label + " (" + field.getName() + ")"); + if(CollectionUtils.nullSafeHasContents(input.getIdList())) + { + if(input.getIdList().stream().anyMatch(id -> value.getId().equals(ValueUtils.getValueAsString(id)))) + { + values.add(value); + } + } + else + { + String term = input.getSearchTerm(); + if(!StringUtils.hasContent(term) || value.getLabel().toLowerCase(Locale.ROOT).contains(term.toLowerCase(Locale.ROOT))) + { + values.add(value); + } + } + } + return (values); + } + + + + /******************************************************************************* + ** Resolve the table from the frontend form values, never from the config table. + *******************************************************************************/ + private QTableMetaData resolveTable(Map otherValues) + { + if(otherValues == null || QContext.getQInstance() == null) + { + return (null); + } + String tableName = ValueUtils.getValueAsString(otherValues.get("table_name")); + return (StringUtils.hasContent(tableName) ? QContext.getQInstance().getTable(tableName) : null); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/model/DataAnalysisTables.java b/src/main/java/com/kingsrook/qbits/dataanalysis/model/DataAnalysisTables.java index f57b481..1e77895 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/model/DataAnalysisTables.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/model/DataAnalysisTables.java @@ -29,8 +29,12 @@ import com.kingsrook.qbits.dataanalysis.DataAnalysisQBitConfig; import com.kingsrook.qbits.dataanalysis.permissions.DataAnalysisPermissions; import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.ChildRecordListRenderer; import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel; import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; @@ -49,6 +53,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.SectionFactory; +import com.kingsrook.qqq.backend.core.model.metadata.tables.TablesPossibleValueSourceMetaDataProvider; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; @@ -111,10 +116,15 @@ public final class DataAnalysisTables public static List possibleValueSources() { List sources = new ArrayList<>(); + sources.add(new QPossibleValueSource().withName(DataAnalysisFieldPossibleValueProvider.NAME) + .withType(QPossibleValueSourceType.CUSTOM).withIdType(QFieldType.STRING) + .withCustomCodeReference(new QCodeReference(DataAnalysisFieldPossibleValueProvider.class))); sources.add(enumSource("kind", "COLUMN_PROFILE", "TABLE_PROFILE", "PIVOT", "COMPARE", "CORRELATION", "CROSSTAB", "SHIFT")); sources.add(enumSource("chart_kind", "HISTOGRAM", "BOX", "BAR", "PARETO", "TIME_SERIES", "SCATTER", "HEATMAP")); sources.add(enumSource("detector", "SPIKE", "DROP", "OUT_OF_RANGE", "NEW_CATEGORY", "NULL_RATE_CREEP")); sources.add(enumSource("watch_state", "OK", "FIRING")); + sources.add(enumSource("watch_bucket", "HOUR", "DAY", "WEEK", "MONTH")); + sources.add(enumSource("watch_severity", "INFO", "WARNING", "ERROR", "CRITICAL")); sources.add(enumSource("run_status", "RUNNING", "COMPLETE", "FAILED")); sources.add(enumSource("schedule_frequency", "DAILY", "WEEKLY", "MONTHLY")); sources.add(enumSource("schedule_format", "HTML_SUMMARY", "CSV", "XLSX")); @@ -205,14 +215,16 @@ public final class DataAnalysisTables List tables = new ArrayList<>(); tables.add(base(config, TABLE_CONFIG, "Analysis Settings", "settings", "data_analysis.settings") .withRecordLabelFormatAndFields("%s", "table_name") - .withField(field("table_name", QFieldType.STRING).withIsRequired(true)) + .withField(field("table_name", QFieldType.STRING).withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME).withIsRequired(true)) .withField(field("enabled", QFieldType.BOOLEAN).withDefaultValue(true)) .withField(field("max_rows", QFieldType.INTEGER)) .withField(field("sample_threshold_rows", QFieldType.INTEGER)) .withField(field("sample_rows", QFieldType.INTEGER)) .withField(field("min_group_size", QFieldType.INTEGER)) - .withField(field("analyzable_fields", QFieldType.TEXT)) - .withField(field("excluded_fields", QFieldType.TEXT)) + // add possibleValueSourceDependsOn("table_name") once core has it + .withField(field("analyzable_fields", QFieldType.TEXT).withPossibleValueSourceName(DataAnalysisFieldPossibleValueProvider.NAME)) + // add possibleValueSourceDependsOn("table_name") once core has it + .withField(field("excluded_fields", QFieldType.TEXT).withPossibleValueSourceName(DataAnalysisFieldPossibleValueProvider.NAME)) .withField(field("cache_ttl_minutes", QFieldType.INTEGER)) .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD)) .withUniqueKey(new UniqueKey("table_name"))); @@ -220,7 +232,7 @@ public final class DataAnalysisTables .withRecordLabelFormatAndFields("%s", "name") .withField(field("name", QFieldType.STRING).withIsRequired(true)) .withField(field("description", QFieldType.TEXT)) - .withField(field("table_name", QFieldType.STRING).withIsRequired(true)) + .withField(field("table_name", QFieldType.STRING).withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME).withIsRequired(true)) .withField(field("filter_json", QFieldType.TEXT)) .withField(field("kind", QFieldType.STRING).withPossibleValueSourceName("data_analysis_kind").withIsRequired(true)) .withField(field("definition_json", QFieldType.TEXT).withIsRequired(true)) @@ -252,21 +264,31 @@ public final class DataAnalysisTables .withRecordSecurityLock(ownerLock(SAVED + ".owner_user_id").withJoinNameChain(List.of(joinName(SCHEDULE))))); tables.add(base(config, WATCH, "Watches", "notifications_active", "data_analysis.watch") .withRecordLabelFormatAndFields("%s", "detector") - .withField(field("saved_id", QFieldType.INTEGER).withPossibleValueSourceName("data_analysis_saved")) - .withField(field("table_name", QFieldType.STRING)) + .withField(field("saved_id", QFieldType.INTEGER).withPossibleValueSourceName("data_analysis_saved") + .withPossibleValueSourceFilter(new QQueryFilter(new QFilterCriteria("table_name", QCriteriaOperator.EQUALS, "${input.table_name}")))) + .withField(field("table_name", QFieldType.STRING).withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME)) .withField(field("filter_json", QFieldType.TEXT)) .withField(field("metric_json", QFieldType.TEXT)) .withField(field("detector", QFieldType.STRING).withPossibleValueSourceName("data_analysis_detector").withIsRequired(true)) .withField(field("parameters_json", QFieldType.TEXT)) - .withField(field("bucket", QFieldType.STRING)) - .withField(field("severity", QFieldType.STRING)) + .withField(field("date_field", QFieldType.STRING).withIsRequired(true)) + .withField(field("bucket", QFieldType.STRING).withDefaultValue("DAY").withIsRequired(true).withPossibleValueSourceName("data_analysis_watch_bucket")) + .withField(field("severity", QFieldType.STRING).withDefaultValue("WARNING").withIsRequired(true).withPossibleValueSourceName("data_analysis_watch_severity")) + .withField(field("owner_user_id", QFieldType.STRING).withIsRequired(true)) .withField(field("enabled", QFieldType.BOOLEAN).withDefaultValue(false)) - .withField(field("last_evaluated_at", QFieldType.DATE_TIME)) - .withField(field("state", QFieldType.STRING).withDefaultValue("OK").withPossibleValueSourceName("data_analysis_watch_state")) - .withField(field("alert_key", QFieldType.STRING))); + .withField(field("last_evaluated_at", QFieldType.DATE_TIME).withIsEditable(false)) + .withField(field("last_value", QFieldType.DECIMAL).withIsEditable(false)) + .withField(field("last_message", QFieldType.TEXT).withIsEditable(false)) + .withField(field("state", QFieldType.STRING).withDefaultValue("OK").withIsEditable(false).withPossibleValueSourceName("data_analysis_watch_state")) + .withField(field("alert_key", QFieldType.STRING).withIsEditable(false)) + .withRecordSecurityLock(ownerLock("owner_user_id")) + .withCustomizer(com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers.PRE_INSERT_RECORD, + new com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference(com.kingsrook.qbits.dataanalysis.watch.WatchCustomizer.class)) + .withCustomizer(com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers.PRE_UPDATE_RECORD, + new com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference(com.kingsrook.qbits.dataanalysis.watch.WatchCustomizer.class))); tables.add(base(config, RUN, "Analysis Runs", "history", "data_analysis.run") .withRecordLabelFormatAndFields("%s", "operation") - .withField(field("table_name", QFieldType.STRING).withIsRequired(true)) + .withField(field("table_name", QFieldType.STRING).withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME).withIsRequired(true)) .withField(field("operation", QFieldType.STRING).withIsRequired(true)) .withField(field("request_hash", QFieldType.STRING)) .withField(field("filter_hash", QFieldType.STRING)) diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/processes/WatchActions.java b/src/main/java/com/kingsrook/qbits/dataanalysis/processes/WatchActions.java new file mode 100644 index 0000000..8ed63b4 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/processes/WatchActions.java @@ -0,0 +1,186 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.processes; + +import java.time.Instant; +import java.util.ArrayList; +import com.kingsrook.qbits.dataanalysis.DataAnalysisRuntime; +import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; +import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface; + +/******************************************************************************* + ** Ordinary streamed ETL Review/Result steps; no custom forms or navigation. + *******************************************************************************/ +public final class WatchActions +{ + public static final String EVALUATE = "data_analysis_watch_evaluate"; + public static final String ENABLE = "data_analysis_watch_enable"; + public static final String DISABLE = "data_analysis_watch_disable"; + public static final String SCHEDULED = "data_analysis_evaluate_watches"; + + /******************************************************************************* + ** Steps only. + *******************************************************************************/ + private WatchActions() + { + } + + /******************************************************************************* + ** Preview selected records without evaluating or mutating them. + *******************************************************************************/ + public static class Review extends AbstractTransformStep implements ProcessSummaryProviderInterface + { + private final ArrayList lines = new ArrayList<>(); + + /******************************************************************************* + ** Extract runs as caller; enforce EDIT before promising any action. + *******************************************************************************/ + @Override + public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + if(!PermissionsHelper.hasTablePermission(new UpdateInput(DataAnalysisTables.WATCH), DataAnalysisTables.WATCH, TablePermissionSubType.EDIT)) + { + throw new QPermissionDeniedException("Watch editing is unavailable."); + } + String verb = verb(input.getProcessName()); + for(QRecord record : input.getRecords()) + { + lines.add(new ProcessSummaryLine(Status.INFO, null, "Watch " + record.getValueString("id") + " (" + record.getValueString("detector") + ") will be " + verb + ".")); + output.addRecord(record); + } + } + + /******************************************************************************* + ** Standard summary lines are rendered by the core Review screen. + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(RunBackendStepOutput output, boolean result) + { + return (result ? new ArrayList<>() : lines); + } + } + + /******************************************************************************* + ** Apply selected records and report exact evaluated/firing/OK/skipped counts. + *******************************************************************************/ + public static class Result extends AbstractLoadStep implements ProcessSummaryProviderInterface + { + private int done; + private int firing; + private int ok; + private String process; + private final ArrayList failures = new ArrayList<>(); + + /******************************************************************************* + ** Re-read and authorize each id when executing after Review. + *******************************************************************************/ + @Override + public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + process = input.getProcessName(); + verb(process); + for(QRecord record : input.getRecords()) + { + try + { + if(EVALUATE.equals(process)) + { + var verdict = DataAnalysisRuntime.watches().evaluate(record.getValue("id"), Instant.now()); + if(!verdict.evaluated()) + { + failures.add(new ProcessSummaryLine(Status.WARNING, null, "Watch " + record.getValueString("id") + " skipped: " + verdict.message())); + continue; + } + firing += verdict.firing() ? 1 : 0; + ok += verdict.firing() ? 0 : 1; + } + else + { + DataAnalysisRuntime.watches().setEnabled(record.getValue("id"), ENABLE.equals(process)); + } + done++; + output.addRecord(record); + } + catch(QException e) + { + failures.add(new ProcessSummaryLine(Status.ERROR, null, "Watch " + record.getValueString("id") + " failed: " + e.getMessage())); + } + } + } + + /******************************************************************************* + ** Counts span all streamed pages, so the Result screen is a true total. + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(RunBackendStepOutput output, boolean result) + { + ArrayList lines = new ArrayList<>(); + String message = done + (done == 1 ? " watch " : " watches ") + (EVALUATE.equals(process) ? "evaluated: " + firing + " firing, " + ok + " OK" + : ENABLE.equals(process) ? "enabled" : "disabled"); + lines.add(new ProcessSummaryLine(Status.OK, null, message)); + lines.addAll(failures); + return (lines); + } + } + + /******************************************************************************* + ** Scheduler invokes a hidden backend-only process every fifteen minutes. + *******************************************************************************/ + public static class Scheduled implements BackendStep + { + /******************************************************************************* + ** Store the real count of successfully evaluated enabled watches. + *******************************************************************************/ + @Override + public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + output.addValue("count", DataAnalysisRuntime.watches().evaluateEnabled(Instant.now())); + } + } + + /******************************************************************************* + ** Reject unknown processes instead of silently choosing a mutation. + *******************************************************************************/ + private static String verb(String process) throws QException + { + return (switch(process) + { + case EVALUATE -> "evaluated"; + case ENABLE -> "enabled"; + case DISABLE -> "disabled"; + default -> throw new QException("Unknown watch action."); + }); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchBucket.java b/src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchBucket.java new file mode 100644 index 0000000..cd045bd --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchBucket.java @@ -0,0 +1,32 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.watch; + +import java.io.Serializable; +import java.util.Set; +import com.kingsrook.qbits.dataanalysis.result.Drill; + +/******************************************************************************* + ** One complete calendar bucket, ordered oldest first for detector input. + *******************************************************************************/ +public record WatchBucket(Serializable start, Double value, Set categories, Drill drill) +{ +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchCustomizer.java b/src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchCustomizer.java new file mode 100644 index 0000000..06020e1 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchCustomizer.java @@ -0,0 +1,97 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.watch; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables; +import com.kingsrook.qbits.dataanalysis.permissions.DataAnalysisPermissions; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; + +/******************************************************************************* + ** Validate standard table CRUD, including partial updates and owner changes. + *******************************************************************************/ +public class WatchCustomizer implements TableCustomizerInterface +{ + private static final Set CONFIGURATION = Set.of("saved_id", "table_name", "filter_json", "metric_json", "date_field", "bucket", + "detector", "parameters_json", "severity", "owner_user_id", "enabled"); + + /******************************************************************************* + ** Validate every inserted watch, including disabled drafts. + *******************************************************************************/ + @Override + public List preInsert(InsertInput input, List records, boolean preview) throws QException + { + for(QRecord record : records) + { + validate(record, record); + } + return (records); + } + + /******************************************************************************* + ** Merge partial patches with the stored row before checking detector parameters. + *******************************************************************************/ + @Override + public List preUpdate(UpdateInput input, List records, boolean preview, Optional> oldRecords) throws QException + { + for(QRecord record : records) + { + if(record.getValues().keySet().stream().noneMatch(CONFIGURATION::contains)) + { + continue; + } + QRecord old = WatchRecords.get(DataAnalysisTables.WATCH, record.getValue("id")); + QRecord merged = new QRecord(old); + merged.getValues().putAll(record.getValues()); + validate(record, merged); + } + return (records); + } + + /******************************************************************************* + ** Record-level messages are shown by normal core insert/update screens. + *******************************************************************************/ + private void validate(QRecord output, QRecord merged) + { + try + { + String owner = merged.getValueString("owner_user_id"); + var session = QContext.getQSession(); + WatchDefinition.require(owner != null && !owner.isBlank(), "owner_user_id is required."); + WatchDefinition.require(session != null && (session.hasPermission(DataAnalysisPermissions.ADMIN) + || session.getUser() != null && owner.equals(session.getUser().getIdReference())), "Only an administrator may assign another watch owner."); + WatchDefinition.parse(WatchRecords.resolve(merged)); + } + catch(QException e) + { + output.addError(new BadInputStatusMessage(e.getMessage())); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchDefinition.java b/src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchDefinition.java new file mode 100644 index 0000000..1b51f54 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchDefinition.java @@ -0,0 +1,152 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.watch; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qqq.backend.core.alerts.AlertSeverity; +import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; + +/******************************************************************************* + ** Validated watch configuration; JSON accepts only documented, typed parameters. + *******************************************************************************/ +public record WatchDefinition(String detector, String metric, String field, String dateField, String bucket, + Integer k, Double threshold, Double relativeChange, Double min, Double max, Double delta, Integer n) +{ + /******************************************************************************* + ** Parse a definition after resolving an optional saved analysis. + *******************************************************************************/ + public static WatchDefinition parse(QRecord row) throws QBadRequestException + { + String detector = row.getValueString("detector"); + require(List.of("SPIKE", "DROP", "OUT_OF_RANGE", "NEW_CATEGORY", "NULL_RATE_CREEP").contains(detector == null ? "" : detector), "Choose a supported detector."); + Map metric = object(row.getValueString("metric_json"), "metric_json"); + require(metric.keySet().stream().allMatch(Set.of("operator", "field")::contains), "metric_json accepts operator and field only."); + String operator = string(metric.get("operator")); + String field = string(metric.get("field")); + require(List.of("COUNT", "SUM", "AVG", "NULL_SHARE", "CATEGORY_SET").contains(operator == null ? "" : operator), "Metric operator must be COUNT, SUM, AVG, NULL_SHARE or CATEGORY_SET."); + require("COUNT".equals(operator) || field != null && !field.isBlank(), "This metric requires a field."); + require("NEW_CATEGORY".equals(detector) == "CATEGORY_SET".equals(operator), "NEW_CATEGORY requires CATEGORY_SET; other detectors require a numeric metric."); + require(!"NULL_RATE_CREEP".equals(detector) || "NULL_SHARE".equals(operator), "NULL_RATE_CREEP requires NULL_SHARE."); + String date = row.getValueString("date_field"); + String bucket = row.getValueString("bucket"); + require(date != null && !date.isBlank(), "Choose a date field."); + require(List.of("HOUR", "DAY", "WEEK", "MONTH").contains(bucket == null ? "" : bucket), "Bucket must be HOUR, DAY, WEEK or MONTH."); + try + { + AlertSeverity.valueOf(row.getValueString("severity")); + } + catch(RuntimeException e) + { + throw new QBadRequestException("Choose a valid alert severity."); + } + Map parameters = object(row.getValueString("parameters_json"), "parameters_json"); + Set allowed = switch(detector) + { + case "SPIKE", "DROP" -> Set.of("k", "threshold", "relativeChange"); + case "OUT_OF_RANGE" -> Set.of("k", "min", "max"); + case "NULL_RATE_CREEP" -> Set.of("k", "delta", "n"); + default -> Set.of("k"); + }; + require(allowed.containsAll(parameters.keySet()), "Unsupported parameter for " + detector + ". Allowed: " + String.join(", ", allowed.stream().sorted().toList()) + "."); + int k = integer(parameters, "k", 14, 1, 1000); + int n = integer(parameters, "n", 3, 1, 1000); + double threshold = number(parameters, "threshold", 3.5); + double relative = number(parameters, "relativeChange", 0.5); + double delta = number(parameters, "delta", 0.05); + require(threshold > 0, "threshold must be positive."); + require(relative >= 0, "relativeChange must be non-negative (0.5 means 50%)."); + require(delta >= 0 && delta <= 1, "delta must be between 0 and 1."); + Double min = parameters.containsKey("min") ? number(parameters, "min", 0) : null; + Double max = parameters.containsKey("max") ? number(parameters, "max", 0) : null; + require(!"OUT_OF_RANGE".equals(detector) || min != null && max != null && min <= max, "OUT_OF_RANGE requires min and max with min <= max."); + return (new WatchDefinition(detector, operator, field, date, bucket, k, threshold, relative, min, max, delta, n)); + } + + /******************************************************************************* + ** Strict JSON objects keep malformed definitions out of standard CRUD. + *******************************************************************************/ + @SuppressWarnings("unchecked") + public static Map object(String json, String name) throws QBadRequestException + { + try + { + Object value = JsonUtils.toObject(json == null || json.isBlank() ? "{}" : json, Object.class); + if(value instanceof Map map) + { + return ((Map) map); + } + } + catch(Exception expected) + { + // Report a stable validation error, without echoing stored JSON. + } + throw new QBadRequestException(name + " must be a JSON object."); + } + + /******************************************************************************* + ** Numeric values must be finite JSON numbers, never coerced strings. + *******************************************************************************/ + private static double number(Map values, String key, double fallback) throws QBadRequestException + { + if(!values.containsKey(key)) + { + return (fallback); + } + require(values.get(key) instanceof Number, key + " must be a number."); + double value = ((Number) values.get(key)).doubleValue(); + require(Double.isFinite(value), key + " must be finite."); + return (value); + } + + /******************************************************************************* + ** Bound work and reject fractional window sizes. + *******************************************************************************/ + private static int integer(Map values, String key, int fallback, int min, int max) throws QBadRequestException + { + double value = number(values, key, fallback); + require(value == Math.rint(value) && value >= min && value <= max, key + " must be an integer from " + min + " to " + max + "."); + return ((int) value); + } + + /******************************************************************************* + ** Optional strings still reject incompatible scalar types. + *******************************************************************************/ + private static String string(Object value) throws QBadRequestException + { + require(value == null || value instanceof String, "Metric operator and field must be strings."); + return ((String) value); + } + + /******************************************************************************* + ** Clear errors shared by table validation and preview. + *******************************************************************************/ + public static void require(boolean valid, String message) throws QBadRequestException + { + if(!valid) + { + throw new QBadRequestException(message); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchDetectors.java b/src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchDetectors.java new file mode 100644 index 0000000..ba246c3 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchDetectors.java @@ -0,0 +1,131 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.watch; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import com.kingsrook.qbits.dataanalysis.engine.AnalysisFormatting; + +/******************************************************************************* + ** Pure detectors over complete buckets, with no context, queries or side effects. + *******************************************************************************/ +public final class WatchDetectors +{ + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private WatchDetectors() + { + } + + /******************************************************************************* + ** Compare the latest value to the preceding k complete buckets. + *******************************************************************************/ + public static WatchVerdict evaluate(WatchDefinition definition, List series, String label) + { + int k = definition.k(); + if(series.isEmpty() || !"OUT_OF_RANGE".equals(definition.detector()) && series.size() < k + 1) + { + return (WatchVerdict.skipped("Not enough history.")); + } + WatchBucket latest = series.getLast(); + List history = series.subList(Math.max(0, series.size() - k - 1), series.size() - 1); + if("NEW_CATEGORY".equals(definition.detector())) + { + Set previous = new HashSet<>(); + history.forEach(bucket -> previous.addAll(bucket.categories())); + long added = latest.categories().stream().filter(value -> !previous.contains(value)).count(); + return (new WatchVerdict(true, added > 0, (double) latest.categories().size(), (double) previous.size(), null, null, + label + (added > 0 ? " has " + added + " new " + (added == 1 ? "category." : "categories.") : " has no new categories."), latest.drill())); + } + if(latest.value() == null || !Double.isFinite(latest.value()) || !"OUT_OF_RANGE".equals(definition.detector()) && history.stream().anyMatch(bucket -> bucket.value() == null || !Double.isFinite(bucket.value()))) + { + return (WatchVerdict.skipped("Not enough history: a bucket has no numeric value.")); + } + double x = latest.value(); + double[] values = history.stream().filter(bucket -> bucket.value() != null && Double.isFinite(bucket.value())).mapToDouble(WatchBucket::value).sorted().toArray(); + Double baseline = values.length == 0 ? null : median(values); + Double low = values.length == 0 ? null : values[0]; + Double high = values.length == 0 ? null : values[values.length - 1]; + boolean firing; + switch(definition.detector()) + { + case "SPIKE", "DROP" -> + { + double mad = median(java.util.Arrays.stream(values).map(value -> Math.abs(value - baseline)).sorted().toArray()); + double change = x - baseline; + double score = mad == 0 ? baseline == 0 ? Math.signum(change) * (change == 0 ? 0 : Double.POSITIVE_INFINITY) + : change / Math.abs(baseline) : 0.6745 * change / mad; + double threshold = mad == 0 ? definition.relativeChange() : definition.threshold(); + firing = "SPIKE".equals(definition.detector()) ? score > threshold : score < -threshold; + } + case "OUT_OF_RANGE" -> + { + low = definition.min(); + high = definition.max(); + firing = x < low || x > high; + } + case "NULL_RATE_CREEP" -> firing = x - baseline > definition.delta() || rising(series, definition.n()); + default -> throw new IllegalArgumentException("Unsupported numeric detector."); + } + String verb = firing ? "DROP".equals(definition.detector()) ? " dropped to " : "SPIKE".equals(definition.detector()) + || "NULL_RATE_CREEP".equals(definition.detector()) ? " rose to " : " is out of range at " : " is "; + String band = low == null ? "" : " (" + ("OUT_OF_RANGE".equals(definition.detector()) ? "expected " : "usual ") + format(low) + "–" + format(high) + ")"; + return (new WatchVerdict(true, firing, x, baseline, low, high, label + verb + format(x) + band + ".", latest.drill())); + } + + /******************************************************************************* + ** n consecutive rises require n+1 values, including the latest bucket. + *******************************************************************************/ + private static boolean rising(List series, int n) + { + if(series.size() <= n) + { + return (false); + } + for(int i = series.size() - n; i < series.size(); i++) + { + if(series.get(i).value() <= series.get(i - 1).value()) + { + return (false); + } + } + return (true); + } + + /******************************************************************************* + ** Median of a sorted non-empty array, including even-sized baselines. + *******************************************************************************/ + private static double median(double[] values) + { + int middle = values.length / 2; + return (values.length % 2 == 0 ? values[middle - 1] / 2 + values[middle] / 2 : values[middle]); + } + + /******************************************************************************* + ** Compact English numeric labels with no scientific notation. + *******************************************************************************/ + private static String format(double value) + { + return (AnalysisFormatting.number(value, 3)); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchEvaluator.java b/src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchEvaluator.java new file mode 100644 index 0000000..5a814c7 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchEvaluator.java @@ -0,0 +1,236 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.watch; + +import java.io.Serializable; +import java.time.Instant; +import java.util.List; +import java.util.Objects; +import com.kingsrook.qbits.dataanalysis.DataAnalysisQBitConfig; +import com.kingsrook.qbits.dataanalysis.DataAnalysisService; +import com.kingsrook.qbits.dataanalysis.engine.DrillFilters; +import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables; +import com.kingsrook.qbits.dataanalysis.permissions.DataAnalysisPermissions; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; +import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.alerts.AlertLink; +import com.kingsrook.qqq.backend.core.alerts.AlertRecordRef; +import com.kingsrook.qqq.backend.core.alerts.AlertSeverity; +import com.kingsrook.qqq.backend.core.alerts.AlertSignal; +import com.kingsrook.qqq.backend.core.alerts.QAlerts; +import com.kingsrook.qqq.backend.core.context.CapturedContext; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.model.session.QUser; +import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleDispatcher; + +/******************************************************************************* + ** Transition watches through the core alerts SPI, computing exclusively as owner. + *******************************************************************************/ +public class WatchEvaluator +{ + public static final String ALERT_TYPE = "data.analysis.anomaly"; + private final DataAnalysisQBitConfig config; + private final DataAnalysisService service; + + /******************************************************************************* + ** One evaluator per instance serializes scheduled and manual transitions. + *******************************************************************************/ + public WatchEvaluator(DataAnalysisQBitConfig config, DataAnalysisService service) + { + this.config = config; + this.service = service; + } + + /******************************************************************************* + ** Preview uses the viewer's rights, never the saved owner's cached data. + *******************************************************************************/ + public WatchPreview preview(Serializable id, Instant now) throws QException + { + return (service.watchPreview(WatchRecords.get(DataAnalysisTables.WATCH, id), now)); + } + + /******************************************************************************* + ** Manual evaluation first requires the caller's read and edit rights. + *******************************************************************************/ + public synchronized WatchVerdict evaluate(Serializable id, Instant now) throws QException + { + requireEdit(); + QRecord row = WatchRecords.get(DataAnalysisTables.WATCH, id); + WatchVerdict verdict = asOwner(row, now); + store(row, verdict, now); + return (verdict); + } + + /******************************************************************************* + ** Only watch inventory/status use maintenance grants, never source data. + *******************************************************************************/ + public synchronized Integer evaluateEnabled(Instant now) throws QException + { + QSession maintenance = new QSession().withUser(new QUser().withIdReference("data-analysis-scheduler")) + .withPermissions("data_analysis.watch.read", "data_analysis.watch.edit") + .withSecurityKeyValue(DataAnalysisPermissions.ADMIN, true); + if(QContext.getQSession().getBackendVariants() != null) + { + maintenance.setBackendVariants(new java.util.HashMap<>(QContext.getQSession().getBackendVariants())); + } + int[] count = {0}; + QContext.withTemporaryContext(new CapturedContext(QContext.getQInstance(), maintenance), () -> + { + QueryInput input = new QueryInput(DataAnalysisTables.WATCH).withShouldFetchHeavyFields(true) + .withFilter(DrillFilters.condition("enabled", QCriteriaOperator.EQUALS, true)); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + for(QRecord row : new QueryAction().execute(input).getRecords()) + { + WatchVerdict verdict = evaluate(row.getValue("id"), now); + if(verdict.evaluated()) + { + count[0]++; + } + } + }); + return (count[0]); + } + + /******************************************************************************* + ** An unavailable or mismatched host session must never fall back to system. + *******************************************************************************/ + private WatchVerdict asOwner(QRecord row, Instant now) + { + String owner = row.getValueString("owner_user_id"); + QSession session; + try + { + session = config.getOwnerSessionSupplier() == null + ? new QAuthenticationModuleDispatcher().getQModule(QContext.getQInstance().getAuthentication()).createAutomatedSessionForUser(QContext.getQInstance(), owner) + : config.getOwnerSessionSupplier().apply(owner); + } + catch(Exception e) + { + return (WatchVerdict.skipped("Owner session unavailable")); + } + if(owner == null || session == null || session.getUser() == null || !Objects.equals(owner, session.getUser().getIdReference()) + || session.getSecurityKeyValues() == null || !session.getSecurityKeyValues().getOrDefault(DataAnalysisTables.USER_KEY, List.of()).contains(owner)) + { + return (WatchVerdict.skipped("Owner session unavailable")); + } + WatchVerdict[] verdict = new WatchVerdict[1]; + try + { + QContext.withTemporaryContext(new CapturedContext(QContext.getQInstance(), session), () -> + { + QRecord owned = WatchRecords.get(DataAnalysisTables.WATCH, row.getValue("id")); + verdict[0] = service.watchPreview(owned, now).verdict(); + }); + return (verdict[0]); + } + catch(QPermissionDeniedException e) + { + return (WatchVerdict.skipped("Owner lacks READ or Analyze access to the watched table.")); + } + catch(com.kingsrook.qqq.backend.core.exceptions.QBadRequestException e) + { + return (WatchVerdict.skipped("Watch evaluation unavailable: " + e.getMessage())); + } + catch(Exception e) + { + return (WatchVerdict.skipped("Watch evaluation unavailable; check the saved analysis and source configuration.")); + } + } + + /******************************************************************************* + ** Persist first; only successful evaluated transitions notify the alerts SPI. + *******************************************************************************/ + private void store(QRecord row, WatchVerdict verdict, Instant now) throws QException + { + String key = "data_analysis_watch:" + row.getValue("id"); + boolean wasFiring = "FIRING".equals(row.getValueString("state")); + QRecord patch = new QRecord().withValue("id", row.getValue("id")).withValue("last_evaluated_at", now) + .withValue("last_value", verdict.value()).withValue("last_message", verdict.message()); + if(verdict.evaluated()) + { + patch.withValue("state", verdict.firing() ? "FIRING" : "OK").withValue("alert_key", key); + } + update(patch); + if(!verdict.evaluated() || wasFiring == verdict.firing()) + { + return; + } + if(verdict.firing()) + { + AlertSignal signal = new AlertSignal(ALERT_TYPE).withDedupKey(key).withSeverity(AlertSeverity.valueOf(row.getValueString("severity"))) + .withTitle("Data anomaly").withSummary(verdict.message()).withRecordRef(AlertRecordRef.of(DataAnalysisTables.WATCH, row.getValue("id"))); + if(verdict.drill() != null) + { + signal.withLink(AlertLink.toTable(verdict.drill().getTableName(), verdict.drill().getFilter(), "View matching records")); + } + QAlerts.raise(signal); + } + else + { + QAlerts.resolve(ALERT_TYPE, key, verdict.message()); + } + } + + /******************************************************************************* + ** Enable and disable selected owner-locked rows through standard core updates. + *******************************************************************************/ + public void setEnabled(Serializable id, Boolean enabled) throws QException + { + requireEdit(); + WatchRecords.get(DataAnalysisTables.WATCH, id); + update(new QRecord().withValue("id", id).withValue("enabled", enabled)); + } + + /******************************************************************************* + ** Never count an unsuccessful core update as an evaluated watch. + *******************************************************************************/ + private static void update(QRecord record) throws QException + { + UpdateInput input = new UpdateInput(DataAnalysisTables.WATCH).withRecords(List.of(record)); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + QRecord stored = new UpdateAction().execute(input).getRecords().getFirst(); + if(stored.getErrors() != null && !stored.getErrors().isEmpty()) + { + throw new QException("Watch could not be updated: " + stored.getErrors().getFirst().getMessage()); + } + } + + /******************************************************************************* + ** Public mutation entry points enforce table permission before source queries. + *******************************************************************************/ + private static void requireEdit() throws QPermissionDeniedException + { + if(!PermissionsHelper.hasTablePermission(new UpdateInput(DataAnalysisTables.WATCH), DataAnalysisTables.WATCH, TablePermissionSubType.EDIT)) + { + throw new QPermissionDeniedException("Watch editing is unavailable."); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchPreview.java b/src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchPreview.java new file mode 100644 index 0000000..0e584f7 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchPreview.java @@ -0,0 +1,31 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.watch; + +import java.util.List; +import com.kingsrook.qbits.dataanalysis.result.Chart; + +/******************************************************************************* + ** Read-only preview with a semantic time series and detector explanation. + *******************************************************************************/ +public record WatchPreview(List series, Chart chart, WatchVerdict verdict) +{ +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchRecords.java b/src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchRecords.java new file mode 100644 index 0000000..ab54ba2 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchRecords.java @@ -0,0 +1,141 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.watch; + +import java.io.Serializable; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.dataanalysis.engine.DrillFilters; +import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; + +/******************************************************************************* + ** Watch and saved-definition reads always use the active user's core permissions. + *******************************************************************************/ +public final class WatchRecords +{ + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private WatchRecords() + { + } + + /******************************************************************************* + ** Fetch heavy JSON with record locks; missing and inaccessible ids look alike. + *******************************************************************************/ + public static QRecord get(String table, Serializable id) throws QException + { + QueryInput input = new QueryInput(table).withShouldFetchHeavyFields(true) + .withFilter(DrillFilters.condition("id", QCriteriaOperator.EQUALS, id).withLimit(1)); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + List records = new QueryAction().execute(input).getRecords(); + if(records.isEmpty()) + { + throw new QNotFoundException("Watch or saved analysis is unavailable."); + } + return (records.getFirst()); + } + + /******************************************************************************* + ** Saved pivots provide one measure; explicit watch metrics can also be used. + *******************************************************************************/ + public static QRecord resolve(QRecord row) throws QException + { + boolean saved = row.getValue("saved_id") != null; + WatchDefinition.require(saved != (row.getValueString("table_name") != null && !row.getValueString("table_name").isBlank()), "Choose either saved_id or table_name."); + QRecord result = new QRecord(row); + if(saved) + { + WatchDefinition.require(row.getValueString("filter_json") == null || row.getValueString("filter_json").isBlank(), "Saved watches use the saved analysis filter; omit filter_json."); + QRecord analysis = get(DataAnalysisTables.SAVED, row.getValue("saved_id")); + result.withValue("table_name", analysis.getValue("table_name")).withValue("filter_json", analysis.getValue("filter_json")); + Map definition = WatchDefinition.object(analysis.getValueString("definition_json"), "Saved definition"); + if(result.getValueString("metric_json") == null || result.getValueString("metric_json").isBlank()) + { + Object metric = definition.get("metric"); + if(metric == null && definition.get("values") instanceof List values && values.size() == 1) + { + metric = values.getFirst(); + } + WatchDefinition.require(metric != null, "Saved analysis needs one COUNT, SUM or AVG measure, or an explicit watch metric_json."); + Map selected = new LinkedHashMap<>(WatchDefinition.object(JsonUtils.toJson(metric), "Saved metric")); + if(selected.get("percentile") == null) + { + selected.remove("percentile"); + } + result.withValue("metric_json", JsonUtils.toJson(selected)); + } + if(result.getValueString("filter_json") == null && definition.get("filter") != null) + { + result.withValue("filter_json", JsonUtils.toJson(definition.get("filter"))); + } + } + filter(result); + return (result); + } + + /******************************************************************************* + ** Reject invalid filter JSON rather than silently broadening the watch. + *******************************************************************************/ + public static QQueryFilter filter(QRecord row) throws QBadRequestException + { + String json = row.getValueString("filter_json"); + if(json == null || json.isBlank()) + { + return (new QQueryFilter()); + } + WatchDefinition.object(json, "filter_json"); + try + { + QQueryFilter filter = JsonUtils.toObject(json, QQueryFilter.class); + validateFilter(filter); + return (filter); + } + catch(Exception e) + { + throw new QBadRequestException("filter_json must contain a valid core query filter."); + } + } + + /******************************************************************************* + ** Invalid operators and null criteria are never interpreted as empty filters. + *******************************************************************************/ + private static void validateFilter(QQueryFilter filter) + { + if(filter == null || filter.getCriteria() == null || filter.getSubFilters() == null + || filter.getCriteria().stream().anyMatch(c -> c == null || c.getOperator() == null || c.getFieldName() == null)) + { + throw new IllegalArgumentException("Invalid filter."); + } + filter.getSubFilters().forEach(WatchRecords::validateFilter); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchSeries.java b/src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchSeries.java new file mode 100644 index 0000000..ba730f1 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchSeries.java @@ -0,0 +1,278 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.watch; + +import java.io.Serializable; +import java.math.BigDecimal; +import java.time.Instant; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalAdjusters; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qbits.dataanalysis.engine.AnalysisSecurity; +import com.kingsrook.qbits.dataanalysis.engine.DrillFilters; +import com.kingsrook.qbits.dataanalysis.engine.EffectiveConfig; +import com.kingsrook.qbits.dataanalysis.engine.RelationshipSupport; +import com.kingsrook.qbits.dataanalysis.result.Axis; +import com.kingsrook.qbits.dataanalysis.result.Chart; +import com.kingsrook.qbits.dataanalysis.result.ChartKind; +import com.kingsrook.qbits.dataanalysis.result.Point; +import com.kingsrook.qbits.dataanalysis.result.Series; +import com.kingsrook.qbits.dataanalysis.result.ValueType; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.DateBucket; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.DateBucketUnit; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + +/******************************************************************************* + ** Exact core date-bucket aggregates under the active user's FULL-field policy. + *******************************************************************************/ +public class WatchSeries extends RelationshipSupport +{ + /******************************************************************************* + ** Bind request-local policy and core streamed limits. + *******************************************************************************/ + public WatchSeries(AnalysisSecurity security, EffectiveConfig config, String table) + { + super(security, config, table); + } + + /******************************************************************************* + ** Evaluate only complete buckets; reject partial or privacy-suppressed data. + *******************************************************************************/ + public WatchPreview preview(WatchDefinition definition, QQueryFilter filter, Instant now) throws QException + { + var date = full(definition.dateField()); + WatchDefinition.require(date.getType() == QFieldType.DATE || date.getType() == QFieldType.DATE_TIME, "Choose a DATE or DATE_TIME field."); + WatchDefinition.require(!"HOUR".equals(definition.bucket()) || date.getType() == QFieldType.DATE_TIME, "HOUR requires a DATE_TIME field."); + if(definition.field() != null) + { + full(definition.field()); + } + if(List.of("SUM", "AVG").contains(definition.metric())) + { + requireNumeric(definition.field()); + } + validateFilter(filter); + ZonedDateTime end = boundary(now, definition.bucket()); + ZonedDateTime start = move(end, definition.bucket(), -definition.k() - 1); + QQueryFilter window = DrillFilters.drill(table, filter, range(definition.dateField(), date.getType(), start, end)).getFilter(); + GroupBy dateGroup = new GroupBy(date).withBucket(new DateBucket(DateBucketUnit.valueOf(definition.bucket()))); + Aggregate count = measure(primaryKey(table), AggregateOperator.COUNT); + Aggregate metric = switch(definition.metric()) + { + case "SUM" -> measure(definition.field(), AggregateOperator.SUM); + case "AVG" -> measure(definition.field(), AggregateOperator.AVG); + case "NULL_SHARE" -> measure(definition.field(), AggregateOperator.COUNT_NULL); + default -> count; + }; + Aggregate nonNull = List.of("SUM", "AVG").contains(definition.metric()) ? measure(definition.field(), AggregateOperator.COUNT) : count; + AggregateInput input = new AggregateInput(table).withFilter(window).withGroupBy(dateGroup).withAggregate(count); + if(metric != count) + { + input.withAggregate(metric); + } + if(nonNull != count) + { + input.withAggregate(nonNull); + } + GroupBy category = "CATEGORY_SET".equals(definition.metric()) ? new GroupBy(full(definition.field())) : null; + if(category != null) + { + input.withGroupBy(category); + } + Map> grouped = new LinkedHashMap<>(); + for(AggregateResult row : aggregate(input)) + { + grouped.computeIfAbsent(key(row.getGroupByValues().get(dateGroup), definition.bucket()), ignored -> new ArrayList<>()).add(row); + } + List buckets = new ArrayList<>(); + boolean seen = false; + for(ZonedDateTime at = start; at.isBefore(end); at = move(at, definition.bucket(), 1)) + { + Serializable value = "HOUR".equals(definition.bucket()) ? at.toInstant() : at.toLocalDate(); + List rows = grouped.getOrDefault(key(value, definition.bucket()), List.of()); + seen |= !rows.isEmpty(); + if(!seen) + { + continue; + } + long size = rows.stream().mapToLong(row -> ((Number) row.getAggregateValues().get(count)).longValue()).sum(); + if(size < config.getMinGroupSize()) + { + return (skipped("Privacy guard: a bucket is below the minimum group size.")); + } + Set categories = new LinkedHashSet<>(); + if(category != null) + { + for(AggregateResult row : rows) + { + if(((Number) row.getAggregateValues().get(count)).longValue() < config.getMinGroupSize()) + { + return (skipped("Privacy guard: a category is below the minimum group size.")); + } + Serializable member = row.getGroupByValues().get(category); + if(member != null) + { + categories.add(member.toString()); + } + } + } + if(nonNull != count && !rows.isEmpty() && ((Number) rows.getFirst().getAggregateValues().get(nonNull)).longValue() < config.getMinGroupSize()) + { + return (skipped("Privacy guard: too few non-null values.")); + } + Object raw = rows.isEmpty() ? null : rows.getFirst().getAggregateValues().get(metric); + if("NULL_SHARE".equals(definition.metric()) && raw != null) + { + long nulls = ((Number) raw).longValue(); + if(nulls > 0 && nulls < config.getMinGroupSize() || size - nulls > 0 && size - nulls < config.getMinGroupSize()) + { + return (skipped("Privacy guard: too few null or non-null values.")); + } + } + Double number = switch(definition.metric()) + { + case "COUNT" -> (double) size; + case "CATEGORY_SET" -> (double) categories.size(); + case "NULL_SHARE" -> size == 0 ? null : ((Number) raw).doubleValue() / size; + case "SUM" -> rows.isEmpty() ? 0.0 : raw == null ? null : ((Number) raw).doubleValue(); + default -> raw == null ? null : ((Number) raw).doubleValue(); + }; + buckets.add(new WatchBucket(value, number, categories, DrillFilters.drill(table, filter, + range(definition.dateField(), date.getType(), at, move(at, definition.bucket(), 1))))); + } + String label = switch(definition.metric()) + { + case "COUNT" -> security.table(table).getLabel(); + case "NULL_SHARE" -> "Null share of " + full(definition.field()).getLabel(); + case "CATEGORY_SET" -> full(definition.field()).getLabel(); + default -> ("SUM".equals(definition.metric()) ? "Sum of " : "Average ") + full(definition.field()).getLabel(); + }; + label += " per " + definition.bucket().toLowerCase(java.util.Locale.ROOT); + WatchVerdict verdict = WatchDetectors.evaluate(definition, buckets, label); + List points = buckets.stream().map(bucket -> new Point().withX(bucket.start()) + .withY(bucket.value() == null ? null : BigDecimal.valueOf(bucket.value())).withDrill(bucket.drill())).toList(); + List annotations = new ArrayList<>(); + if(verdict.baselineLow() != null) + { + annotations.add("Baseline band: " + verdict.baselineLow() + " to " + verdict.baselineHigh() + "."); + } + annotations.add(verdict.message()); + Chart chart = new Chart().withKind(ChartKind.TIME_SERIES) + .withX(new Axis().withField(definition.dateField()).withLabel(date.getLabel()).withValueType(date.getType() == QFieldType.DATE ? ValueType.DATE : ValueType.DATE_TIME)) + .withY(new Axis().withLabel(label).withValueType("NULL_SHARE".equals(definition.metric()) ? ValueType.PERCENT : ValueType.DECIMAL)) + .withSeries(List.of(new Series().withName(label).withPoints(points))).withAnnotations(annotations); + return (new WatchPreview(buckets, chart, verdict)); + } + + /******************************************************************************* + ** No suppressed values, categories or drill predicates leave this computation. + *******************************************************************************/ + public static WatchPreview skipped(String message) + { + return (new WatchPreview(List.of(), new Chart().withKind(ChartKind.TIME_SERIES).withAnnotations(List.of(message)), WatchVerdict.skipped(message))); + } + + /******************************************************************************* + ** User-authored watch filters must also use FULL fields only. + *******************************************************************************/ + private void validateFilter(QQueryFilter filter) throws QException + { + if(filter == null) + { + return; + } + for(var criterion : filter.getCriteria()) + { + if(criterion.getFieldName() != null) + { + full(criterion.getFieldName()); + } + if(criterion.getFieldFunction() != null) + { + full(criterion.getFieldFunction().getFieldName()); + } + } + for(QQueryFilter child : filter.getSubFilters()) + { + validateFilter(child); + } + } + + /******************************************************************************* + ** Calendar boundaries follow the owner's time zone, including DST changes. + *******************************************************************************/ + private static ZonedDateTime boundary(Instant now, String bucket) + { + ZonedDateTime local = now.atZone(ValueUtils.getSessionOrInstanceZoneId()); + return (switch(bucket) + { + case "HOUR" -> local.truncatedTo(ChronoUnit.HOURS); + case "WEEK" -> local.toLocalDate().with(TemporalAdjusters.previousOrSame(java.time.DayOfWeek.MONDAY)).atStartOfDay(local.getZone()); + case "MONTH" -> local.toLocalDate().withDayOfMonth(1).atStartOfDay(local.getZone()); + default -> local.toLocalDate().atStartOfDay(local.getZone()); + }); + } + + /******************************************************************************* + ** Advance on the calendar, rather than treating every day as 24 hours. + *******************************************************************************/ + private static ZonedDateTime move(ZonedDateTime at, String bucket, int count) + { + return (switch(bucket) + { + case "HOUR" -> at.plusHours(count); + case "WEEK" -> at.plusWeeks(count); + case "MONTH" -> at.plusMonths(count); + default -> at.plusDays(count); + }); + } + + /******************************************************************************* + ** Keep DATE filters as dates and DATE_TIME filters as instants. + *******************************************************************************/ + private static QQueryFilter range(String field, QFieldType type, ZonedDateTime start, ZonedDateTime end) + { + return (DrillFilters.range(field, type == QFieldType.DATE ? start.toLocalDate() : start.toInstant(), + type == QFieldType.DATE ? end.toLocalDate() : end.toInstant(), false)); + } + + /******************************************************************************* + ** Normalize core date-bucket keys independently of JDBC value representation. + *******************************************************************************/ + private static String key(Serializable value, String bucket) + { + return ("HOUR".equals(bucket) ? ValueUtils.getValueAsInstant(value).toString() : ValueUtils.getValueAsLocalDate(value).toString()); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchVerdict.java b/src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchVerdict.java new file mode 100644 index 0000000..1267706 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/watch/WatchVerdict.java @@ -0,0 +1,38 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.watch; + +import com.kingsrook.qbits.dataanalysis.result.Drill; + +/******************************************************************************* + ** An unevaluated verdict must never resolve an existing alert. + *******************************************************************************/ +public record WatchVerdict(Boolean evaluated, Boolean firing, Double value, Double baseline, + Double baselineLow, Double baselineHigh, String message, Drill drill) +{ + /******************************************************************************* + ** Fail closed without retaining data or a drill. + *******************************************************************************/ + public static WatchVerdict skipped(String message) + { + return (new WatchVerdict(false, false, null, null, null, null, message, null)); + } +} diff --git a/src/main/resources/openapi/qqq-v1/data-analysis.yaml b/src/main/resources/openapi/qqq-v1/data-analysis.yaml index 5f369a4..0541bf6 100644 --- a/src/main/resources/openapi/qqq-v1/data-analysis.yaml +++ b/src/main/resources/openapi/qqq-v1/data-analysis.yaml @@ -1503,6 +1503,60 @@ components: \ null when suppressed." nullable: true type: "object" + DataAnalysisWatchBucketV1: + properties: + start: + description: "Calendar date or hour-start instant in the user's time zone." + type: "string" + value: + nullable: true + type: "number" + categories: + items: + type: "string" + type: "array" + drill: + $ref: "#/components/schemas/DataAnalysisDrillV1" + nullable: true + type: "object" + DataAnalysisWatchPreviewV1: + properties: + series: + items: + $ref: "#/components/schemas/DataAnalysisWatchBucketV1" + type: "array" + chart: + $ref: "#/components/schemas/DataAnalysisChartV1" + description: "TIME_SERIES with baseline band annotations." + verdict: + $ref: "#/components/schemas/DataAnalysisWatchVerdictV1" + type: "object" + DataAnalysisWatchVerdictV1: + properties: + evaluated: + description: "False for insufficient history or privacy suppression; must\ + \ not resolve an alert." + type: "boolean" + firing: + type: "boolean" + value: + nullable: true + type: "number" + baseline: + nullable: true + type: "number" + baselineLow: + nullable: true + type: "number" + baselineHigh: + nullable: true + type: "number" + message: + type: "string" + drill: + $ref: "#/components/schemas/DataAnalysisDrillV1" + nullable: true + type: "object" FilterCriteria: properties: fieldName: @@ -1585,9 +1639,9 @@ info: contact: email: "contact@kingsrook.com" description: "Profiles, pivots, comparisons, distribution shifts, correlations,\ - \ scatter samples and crosstabs as the signed-in user." + \ scatter samples, crosstabs and anomaly watch previews as the signed-in user." title: "QQQ Middleware API extension: data-analysis" - version: "1.2.0" + version: "1.3.0" openapi: "3.0.3" paths: /qqq/v1/qbit/data-analysis/tables/{table}/capabilities: @@ -1817,6 +1871,29 @@ paths: - sessionUuidCookie: - "N/A" summary: "Cross-tabulate two fields" + /qqq/v1/qbit/data-analysis/watches/{id}/preview: + post: + description: "Compute complete time buckets and a detector verdict under the\ + \ signed-in user's READ, Analyze and FULL-field permissions. Includes a baseline\ + \ band annotation. Does not update the watch or raise alerts." + parameters: + - description: "Owner-scoped analysis run or watch id." + in: "path" + name: "id" + required: true + schema: + type: "string" + responses: + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisWatchPreviewV1" + description: "Successful Response" + security: + - sessionUuidCookie: + - "N/A" + summary: "Preview an anomaly watch" /qqq/v1/qbit/data-analysis/tables/{table}/pivot: post: description: "Group records by up to 3 row and 2 column fields (with date or\ @@ -1930,7 +2007,7 @@ paths: \ result of an owner-scoped run. Completed results are rechecked against current\ \ permissions." parameters: - - description: "Owner-scoped analysis run id." + - description: "Owner-scoped analysis run or watch id." in: "path" name: "runId" required: true diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisApiTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisApiTest.java index 86a1b6f..b5c8256 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisApiTest.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisApiTest.java @@ -73,6 +73,10 @@ class DataAnalysisApiTest extends EngineFixture session.withPermission("orders.analyze"); } session.withSecurityKeyValue("data_analysis_user", session.getUser().getIdReference()); + if(session.getUuid().startsWith("watch-")) + { + session.getSecurityKeyValues().remove("data_analysis.admin"); + } } } @@ -169,7 +173,7 @@ class DataAnalysisApiTest extends EngineFixture assertThat(denied.getStatus()).isEqualTo(403); assertThat(run.path(operation.equals("COLUMN_PROFILE") ? "columnProfile" : "tableProfile").isObject()).isTrue(); } - assertThat(invoker.listOperations().stream().filter(o -> o.pathTemplate().startsWith(DataAnalysisApi.PREFIX)).toList()).hasSize(13); + assertThat(invoker.listOperations().stream().filter(o -> o.pathTemplate().startsWith(DataAnalysisApi.PREFIX)).toList()).hasSize(14); var client = java.net.http.HttpClient.newHttpClient(); var response = client.send(java.net.http.HttpRequest.newBuilder(java.net.URI.create("http://127.0.0.1:" + server.getBoundPort() + "/qqq/v1/openapi.json")).build(), java.net.http.HttpResponse.BodyHandlers.ofString()); assertThat(response.statusCode()).isEqualTo(200); @@ -269,7 +273,7 @@ class DataAnalysisApiTest extends EngineFixture } /******************************************************************************* - ** Generated fragment is committed verbatim and has all thirteen operations. + ** Generated fragment is committed verbatim and has all fourteen operations. *******************************************************************************/ @Test void contract() throws Exception @@ -351,7 +355,7 @@ class DataAnalysisApiTest extends EngineFixture } relationshipCountDrills(call(HttpMethod.POST, "/tables/{table}/crosstab", Map.of("rowField", "status", "columnField", "customer"), 200)); // All new schemas are discoverable from the same served OpenAPI catalog. - assertThat(com.kingsrook.qbits.dataanalysis.api.DataAnalysisApi.extension().getSemanticVersion()).isEqualTo("1.2.0"); + assertThat(com.kingsrook.qbits.dataanalysis.api.DataAnalysisApi.extension().getSemanticVersion()).isEqualTo("1.3.0"); } /******************************************************************************* @@ -415,4 +419,37 @@ class DataAnalysisApiTest extends EngineFixture } } } + + @Test + void watchPreviewOverHttpIsReadOnlyAndOwnerLocked() throws Exception + { + caller = "watch-owner"; + String owner = "User:" + Math.floorMod(caller.hashCode(), 10_000); + java.time.ZoneId zone = java.time.ZoneId.of("America/Chicago"); + java.time.LocalDate today = java.time.LocalDate.now(zone); + for(int i = 0; i < 5; i++) + { + TestSupport.insert("orders", new com.kingsrook.qqq.backend.core.model.data.QRecord().withValue("id", 100 + i).withValue("customer", 1) + .withValue("created", today.minusDays(i < 2 ? 3 - i : 1).atTime(12, 0).atZone(zone).toInstant())); + } + var row = TestSupport.insert(com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables.WATCH, + WatchDetectorsTest.definition("SPIKE", "COUNT", "{\"k\":2}").withValue("owner_user_id", owner)); + start(); + var request = new UiApiRequest().withMethod(HttpMethod.POST).withPath(DataAnalysisApi.PREFIX + "/watches/{id}/preview") + .withPathParam("id", row.getValueString("id")).withSessionUUID(caller).withUserTimezone("America/Chicago"); + var response = invoker.invoke(request); + assertThat(response.getStatus()).as(response.getBody()).isEqualTo(200); + JsonNode json = response.getJson(); + assertThat(json.path("series").size()).isEqualTo(3); + assertThat(json.path("verdict").path("firing").asBoolean()).isTrue(); + assertThat(json.path("verdict").path("value").asDouble()).isEqualTo(3); + assertThat(json.path("chart").path("kind").asText()).isEqualTo("TIME_SERIES"); + assertThat(json.path("chart").path("annotations").toString()).contains("Baseline band"); + checkDrills(json); + assertThat(com.kingsrook.qbits.dataanalysis.watch.WatchRecords.get(com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables.WATCH, row.getValue("id")).getValue("last_evaluated_at")).isNull(); + assertThat(invoker.invoke(request.withSessionUUID("watch-outsider")).getStatus()).isEqualTo(404); + assertThat(invoker.invoke(request.withSessionUUID("denied")).getStatus()).isEqualTo(403); + assertThat(invoker.invoke(request.withSessionUUID(caller).withPathParam("id", "99999")).getStatus()).isEqualTo(404); + assertThat(invoker.invoke(request.withPathParam("id", "invalid")).getStatus()).isEqualTo(400); + } } diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisHomeTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisHomeTest.java index f8439b7..fc34518 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisHomeTest.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisHomeTest.java @@ -90,9 +90,9 @@ class DataAnalysisHomeTest TestSupport.insert(DataAnalysisTables.SAVED_STAR, new QRecord().withValue("saved_id", mine.getValue("id")).withValue("user_id", "alice")); TestSupport.insert(DataAnalysisTables.SAVED_STAR, new QRecord().withValue("saved_id", bob.getValue("id")).withValue("user_id", "alice")); TestSupport.insert(DataAnalysisTables.SAVED_STAR, new QRecord().withValue("saved_id", mine.getValue("id")).withValue("user_id", "bob")); - TestSupport.insert(DataAnalysisTables.WATCH, new QRecord().withValue("detector", "SPIKE").withValue("enabled", true).withValue("state", "FIRING")); - TestSupport.insert(DataAnalysisTables.WATCH, new QRecord().withValue("detector", "DROP").withValue("enabled", true)); - TestSupport.insert(DataAnalysisTables.WATCH, new QRecord().withValue("detector", "DROP").withValue("enabled", false)); + TestSupport.insert(DataAnalysisTables.WATCH, WatchDetectorsTest.definition("SPIKE", "COUNT", null).withValue("enabled", true).withValue("state", "FIRING")); + TestSupport.insert(DataAnalysisTables.WATCH, WatchDetectorsTest.definition("DROP", "COUNT", null).withValue("enabled", true)); + TestSupport.insert(DataAnalysisTables.WATCH, WatchDetectorsTest.definition("DROP", "COUNT", null).withValue("enabled", false)); run("alice", "COMPLETE", now); run("alice", "COMPLETE", now.minus(7, ChronoUnit.DAYS)); run("alice", "FAILED", now.minus(1, ChronoUnit.DAYS)); diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisMetadataTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisMetadataTest.java index f69dfb7..26d1834 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisMetadataTest.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisMetadataTest.java @@ -67,7 +67,7 @@ class DataAnalysisMetadataTest QInstance instance = TestSupport.instance(); new QInstanceValidator().validate(instance); assertThat(instance.getTables()).containsOnlyKeys(DataAnalysisTables.ALL); - assertThat(instance.getPossibleValueSources().keySet().stream().filter(name -> name.startsWith("data_analysis_"))).hasSize(13); + assertThat(instance.getPossibleValueSources().keySet().stream().filter(name -> name.startsWith("data_analysis_"))).hasSize(16); assertThat(instance.getJoins()).hasSize(3); assertThat(instance.getWidgets()).hasSize(4).containsKey(DataAnalysisHome.HOME_WIDGET); assertThat(instance.getApp("dataAnalysis").getSections().getFirst().getProcesses()).isNullOrEmpty(); @@ -204,7 +204,7 @@ class DataAnalysisMetadataTest } assertThat(ddl).contains("id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY", "create_date TIMESTAMP WITHOUT TIME ZONE", "modify_date TIMESTAMP WITHOUT TIME ZONE", "rows_scanned BIGINT", "result_json TEXT", "shared BOOLEAN", "day INTEGER", - "owner_user_id VARCHAR(250) NOT NULL", "CREATE UNIQUE INDEX IF NOT EXISTS uq_data_analysis_table_config_1 ON data_analysis_table_config (table_name)", + "owner_user_id VARCHAR(250) NOT NULL", "date_field VARCHAR(250) NOT NULL", "last_value NUMERIC", "last_message TEXT", "CREATE UNIQUE INDEX IF NOT EXISTS uq_data_analysis_table_config_1 ON data_analysis_table_config (table_name)", "CREATE UNIQUE INDEX IF NOT EXISTS uq_data_analysis_saved_star_1 ON data_analysis_saved_star (saved_id, user_id)"); assertThat(ddl).endsWith(";\n"); } diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisPossibleValuesTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisPossibleValuesTest.java new file mode 100644 index 0000000..a932c79 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisPossibleValuesTest.java @@ -0,0 +1,167 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.kingsrook.qbits.dataanalysis; + + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.dataanalysis.engine.EffectiveConfigResolver; +import com.kingsrook.qbits.dataanalysis.model.DataAnalysisFieldPossibleValueProvider; +import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables; +import com.kingsrook.qqq.backend.core.actions.values.SearchPossibleValueSourceAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueSearchFilterUseCase; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Dependent possible values through the same search inputs used by form requests. + *******************************************************************************/ +class DataAnalysisPossibleValuesTest +{ + /******************************************************************************* + ** Release the request context. + *******************************************************************************/ + @AfterEach + void cleanup() + { + QContext.clear(); + } + + + + /******************************************************************************* + ** The watch table selection filters saved analyses without widening owner locks. + *******************************************************************************/ + @Test + void watchSavedAnalysisDependsOnTable() throws Exception + { + QInstance instance = TestSupport.instance(); + QRecord orders = TestSupport.insert(DataAnalysisTables.SAVED, TestSupport.saved("alice", false)); + QRecord customers = TestSupport.insert(DataAnalysisTables.SAVED, TestSupport.saved("alice", false).withValue("table_name", "customers")); + TestSupport.insert(DataAnalysisTables.SAVED, TestSupport.saved("bob", true)); + QContext.init(instance, TestSupport.user("alice").withPermissions("data_analysis.saved.read", "data_analysis.watch.edit")); + QFieldMetaData field = instance.getTable(DataAnalysisTables.WATCH).getField("saved_id"); + assertThat(search(field, Map.of("table_name", "orders"))).extracting(value -> (Object) value.getId()).containsExactly(orders.getValueInteger("id")); + assertThat(search(field, Map.of("table_name", "customers"))).extracting(value -> (Object) value.getId()).containsExactly(customers.getValueInteger("id")); + assertThat(search(field, Map.of())).isEmpty(); + assertThat(search(field, Map.of("table_name", "missing"))).isEmpty(); + } + + + + /******************************************************************************* + ** All four table references use the core source, including host-owned sources. + *******************************************************************************/ + @Test + void tableNamesUseCoreTablesSource() throws Exception + { + QInstance instance = TestSupport.instance(); + instance.addTable(new QTableMetaData().withName("orders").withLabel("Orders")); + for(String tableName : List.of(DataAnalysisTables.TABLE_CONFIG, DataAnalysisTables.SAVED, DataAnalysisTables.WATCH, DataAnalysisTables.RUN)) + { + QFieldMetaData field = instance.getTable(tableName).getField("table_name"); + assertThat(field.getPossibleValueSourceName()).isEqualTo("tables"); + assertThat(search(field, Map.of())).extracting(value -> value.getId().toString()).containsExactlyInAnyOrderElementsOf(instance.getTables().keySet()); + } + var hostSource = instance.getPossibleValueSource("tables").withLabel("Host tables"); + QInstance host = new QInstance(); + host.addBackend(instance.getBackend("memory")); + host.addPossibleValueSource(hostSource); + new DataAnalysisQBitProducer().withQBitConfig(new DataAnalysisQBitConfig().withBackendName("memory")).produce(host); + assertThat(host.getPossibleValueSource("tables")).isSameAs(hostSource); + assertThat(host.getPossibleValueSource("tables").getLabel()).isEqualTo("Host tables"); + } + + + + /******************************************************************************* + ** Both CSV settings offer only selected-table fields and retain CSV parsing. + *******************************************************************************/ + @Test + void fieldPickersDependOnTableAndKeepCsvStorage() throws Exception + { + QInstance instance = TestSupport.instance(); + instance.addTable(new QTableMetaData().withName("orders").withLabel("Orders") + .withField(new QFieldMetaData("amount", QFieldType.DECIMAL).withLabel("Order total")) + .withField(new QFieldMetaData("status", QFieldType.STRING))); + instance.addTable(new QTableMetaData().withName("customers").withLabel("Customers") + .withField(new QFieldMetaData("email", QFieldType.STRING).withLabel("Email"))); + for(String fieldName : List.of("analyzable_fields", "excluded_fields")) + { + QFieldMetaData field = instance.getTable(DataAnalysisTables.TABLE_CONFIG).getField(fieldName); + assertThat(field.getType()).isEqualTo(QFieldType.TEXT); + assertThat(instance.getPossibleValueSource(field.getPossibleValueSourceName()).getType()).isEqualTo(QPossibleValueSourceType.CUSTOM); + assertThat(search(field, Map.of("table_name", "orders"))).extracting(value -> (Object) value.getId()).containsExactly("amount", "status"); + assertThat(search(field, Map.of("table_name", "orders"))).extracting(QPossibleValue::getLabel).containsExactly("Order total (amount)", "status (status)"); + assertThat(search(field, Map.of("table_name", "customers"))).extracting(value -> (Object) value.getId()).containsExactly("email"); + assertThat(search(field, Map.of())).isEmpty(); + assertThat(search(field, Map.of("table_name", ""))).isEmpty(); + assertThat(search(field, Map.of("table_name", "missing"))).isEmpty(); + assertThat(search(field, null)).isEmpty(); + } + SearchPossibleValueSourceInput input = new SearchPossibleValueSourceInput().withPossibleValueSourceName(DataAnalysisFieldPossibleValueProvider.NAME) + .withOtherValues(Map.of("table_name", "orders")).withSearchTerm("TOTAL"); + assertThat(new SearchPossibleValueSourceAction().execute(input).getResults()).extracting(value -> (Object) value.getId()).containsExactly("amount"); + input.withSearchTerm("AMOUNT"); + assertThat(new SearchPossibleValueSourceAction().execute(input).getResults()).extracting(value -> (Object) value.getId()).containsExactly("amount"); + input.withIdList(List.of("status", "email")); + assertThat(new SearchPossibleValueSourceAction().execute(input).getResults()).extracting(value -> (Object) value.getId()).containsExactly("status"); + DataAnalysisFieldPossibleValueProvider provider = new DataAnalysisFieldPossibleValueProvider(); + assertThat(provider.getPossibleValue(null)).isNull(); + assertThat(provider.getPossibleValue("amount,status").getId()).isEqualTo("amount,status"); + TestSupport.insert(DataAnalysisTables.TABLE_CONFIG, new QRecord().withValue("table_name", "orders") + .withValue("analyzable_fields", "amount, status").withValue("excluded_fields", "status, amount")); + var config = new EffectiveConfigResolver(new DataAnalysisQBitConfig().withBackendName("memory")).resolve("orders"); + assertThat(config.getAnalyzableFields()).containsExactlyInAnyOrder("amount", "status"); + assertThat(config.getExcludedFields()).containsExactlyInAnyOrder("amount", "status"); + } + + + + /******************************************************************************* + ** Mirror TablePossibleValuesSpecV1: clone and interpret the field filter as FORM. + *******************************************************************************/ + private List> search(QFieldMetaData field, Map otherValues) throws Exception + { + SearchPossibleValueSourceInput input = new SearchPossibleValueSourceInput() + .withPossibleValueSourceName(field.getPossibleValueSourceName()).withOtherValues(otherValues); + if(field.getPossibleValueSourceFilter() != null) + { + var filter = field.getPossibleValueSourceFilter().clone(); + filter.interpretValues(Map.of("input", otherValues), PossibleValueSearchFilterUseCase.FORM); + input.withDefaultQueryFilter(filter); + } + return (new SearchPossibleValueSourceAction().execute(input).getResults()); + } +} diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/PivotContractTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/PivotContractTest.java index 0443a6e..fc85569 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/PivotContractTest.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/PivotContractTest.java @@ -347,4 +347,22 @@ abstract class PivotContractTest extends EngineFixture equalNumber(result.getGrandTotals().getFirst(), "0"); countDrills(result); } + + @Test + void booleanHeadersAndHumanMeasureLabels() throws Exception + { + var booleans = service.pivot(pivot(List.of("paid"), List.of("status"))); + assertThat(booleans.getRowHeaders().stream().flatMap(List::stream).map(PivotHeader::getLabel)).containsExactlyInAnyOrder("Yes", "No", "(Blank)"); + assertThat(booleans.getMeasures().getFirst().getLabel()).isEqualTo("Count"); + var operators = List.of(PivotMeasure.Operator.AVG, PivotMeasure.Operator.SUM, PivotMeasure.Operator.COUNT_DISTINCT, + PivotMeasure.Operator.MIN, PivotMeasure.Operator.MAX, PivotMeasure.Operator.PERCENTILE); + var measures = operators.stream().map(operator -> measure(operator, "amount")).toList(); + measures.getLast().withPercentile(new BigDecimal("0.5")); + var result = service.pivot(pivot(List.of("paid"), List.of()).withValues(measures)); + assertThat(result.getMeasures().stream().map(m -> m.getLabel())).containsExactly("Average Amount", "Sum of Amount", "Distinct Amount", "Minimum Amount", "Maximum Amount", "Median Amount"); + measures.getLast().withPercentile(new BigDecimal("0.9")); + assertThat(service.pivot(pivot(List.of("paid"), List.of()).withValues(measures)).getMeasures().getLast().getLabel()).isEqualTo("90th percentile of Amount"); + var cross = service.crosstab(new com.kingsrook.qbits.dataanalysis.result.CrosstabRequest().withTableName("orders").withRowField("paid").withColumnField("status")); + assertThat(cross.getRows().stream().map(PivotHeader::getLabel)).containsExactlyInAnyOrder("Yes", "No", "(Blank)"); + } } diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/WatchDetectorsTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/WatchDetectorsTest.java new file mode 100644 index 0000000..d16bf62 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/WatchDetectorsTest.java @@ -0,0 +1,174 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import com.kingsrook.qbits.dataanalysis.result.Drill; +import com.kingsrook.qbits.dataanalysis.watch.WatchBucket; +import com.kingsrook.qbits.dataanalysis.watch.WatchDefinition; +import com.kingsrook.qbits.dataanalysis.watch.WatchDetectors; +import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/******************************************************************************* + ** Hand-computed detector cases isolate statistical behavior from the database. + *******************************************************************************/ +class WatchDetectorsTest +{ + static QRecord definition(String detector, String metric, String parameters) + { + return (new QRecord().withValue("table_name", "orders").withValue("owner_user_id", "alice").withValue("detector", detector) + .withValue("metric_json", "{\"operator\":\"" + metric + "\",\"field\":\"amount\"}").withValue("parameters_json", parameters) + .withValue("date_field", "created").withValue("bucket", "DAY").withValue("severity", "WARNING").withValue("enabled", true)); + } + + private WatchDefinition parsed(String detector, String metric, String parameters) throws Exception + { + return (WatchDefinition.parse(definition(detector, metric, parameters))); + } + + private List series(Double... values) + { + List result = new ArrayList<>(); + for(Double value : values) + { + result.add(new WatchBucket("day-" + result.size(), value, Set.of(), new Drill().withTableName("orders"))); + } + return (result); + } + + @Test + void robustZScoreHasDirectionAndStrictThreshold() throws Exception + { + var spike = parsed("SPIKE", "COUNT", "{\"k\":3}"); + assertThat(WatchDetectors.evaluate(spike, series(9.0, 10.0, 11.0, 20.0), "Orders per day").firing()).isTrue(); + assertThat(WatchDetectors.evaluate(spike, series(9.0, 10.0, 11.0, 1.0), "Orders").firing()).isFalse(); + var drop = parsed("DROP", "COUNT", "{\"k\":3}"); + var verdict = WatchDetectors.evaluate(drop, series(40.0, 45.0, 55.0, 12.0), "Orders per day"); + assertThat(verdict.firing()).isTrue(); + assertThat(verdict.baseline()).isEqualTo(45.0); + assertThat(verdict.message()).contains("Orders per day dropped to 12", "usual 40"); + assertThat(verdict.drill().getTableName()).isEqualTo("orders"); + assertThat(WatchDetectors.evaluate(drop, series(40.0, 45.0, 55.0, 100.0), "Orders").firing()).isFalse(); + var exact = parsed("SPIKE", "COUNT", "{\"k\":3,\"threshold\":0.6745}"); + assertThat(WatchDetectors.evaluate(exact, series(9.0, 10.0, 11.0, 11.0), "Orders").firing()).isFalse(); + } + + @Test + void zeroMadUsesRelativeChangeAndHandlesZeroAndNegativeBaselines() throws Exception + { + var spike = parsed("SPIKE", "COUNT", "{\"k\":2}"); + for(double x : new double[] {10, 15}) + { + assertThat(WatchDetectors.evaluate(spike, series(10.0, 10.0, x), "Orders").firing()).isFalse(); + } + assertThat(WatchDetectors.evaluate(spike, series(10.0, 10.0, 16.0), "Orders").firing()).isTrue(); + assertThat(WatchDetectors.evaluate(spike, series(0.0, 0.0, 0.0), "Orders").firing()).isFalse(); + assertThat(WatchDetectors.evaluate(spike, series(0.0, 0.0, 1.0), "Orders").firing()).isTrue(); + assertThat(WatchDetectors.evaluate(spike, series(-10.0, -10.0, -4.0), "Balance").firing()).isTrue(); + var drop = parsed("DROP", "COUNT", "{\"k\":2}"); + assertThat(WatchDetectors.evaluate(drop, series(10.0, 10.0, 4.0), "Orders").firing()).isTrue(); + assertThat(WatchDetectors.evaluate(drop, series(0.0, 0.0, -1.0), "Balance").firing()).isTrue(); + } + + @Test + void rangeIncludesBothBoundsAndNeedsOnlyLatestValue() throws Exception + { + var range = parsed("OUT_OF_RANGE", "SUM", "{\"min\":10,\"max\":20}"); + assertThat(WatchDetectors.evaluate(range, series(10.0), "Amount").firing()).isFalse(); + assertThat(WatchDetectors.evaluate(range, series(20.0), "Amount").firing()).isFalse(); + assertThat(WatchDetectors.evaluate(range, series(9.0), "Amount").firing()).isTrue(); + assertThat(WatchDetectors.evaluate(range, series(21.0), "Amount").firing()).isTrue(); + } + + @Test + void categoriesCompareWithEntireTrailingWindow() throws Exception + { + var definition = parsed("NEW_CATEGORY", "CATEGORY_SET", "{\"k\":2}"); + var baseline = List.of(new WatchBucket("1", 1.0, Set.of("a"), null), new WatchBucket("2", 1.0, Set.of("b"), null)); + List values = new ArrayList<>(baseline); + values.add(new WatchBucket("3", 2.0, Set.of("a", "b"), null)); + assertThat(WatchDetectors.evaluate(definition, values, "Status").firing()).isFalse(); + values.set(2, new WatchBucket("3", 2.0, Set.of("a", "new"), null)); + assertThat(WatchDetectors.evaluate(definition, values, "Status").message()).contains("1 new category"); + values.set(2, new WatchBucket("3", 0.0, Set.of(), null)); + assertThat(WatchDetectors.evaluate(definition, values, "Status").firing()).isFalse(); + } + + @Test + void nullRateDetectsDeltaOrConsecutiveRises() throws Exception + { + var definition = parsed("NULL_RATE_CREEP", "NULL_SHARE", "{\"k\":3}"); + assertThat(WatchDetectors.evaluate(definition, series(0.1, 0.1, 0.1, 0.16), "Null share").firing()).isTrue(); + assertThat(WatchDetectors.evaluate(definition, series(0.1, 0.11, 0.12, 0.13), "Null share").firing()).isTrue(); + assertThat(WatchDetectors.evaluate(definition, series(0.1, 0.12, 0.11, 0.13), "Null share").firing()).isFalse(); + assertThat(WatchDetectors.evaluate(definition, series(0.1, 0.11, 0.11, 0.12), "Null share").firing()).isFalse(); + } + + @ParameterizedTest + @ValueSource(strings = {"SPIKE", "DROP", "NEW_CATEGORY", "NULL_RATE_CREEP"}) + void insufficientHistoryNeverBecomesRecovery(String detector) throws Exception + { + String metric = detector.equals("NEW_CATEGORY") ? "CATEGORY_SET" : detector.equals("NULL_RATE_CREEP") ? "NULL_SHARE" : "COUNT"; + var result = WatchDetectors.evaluate(parsed(detector, metric, null), series(1.0), "Orders"); + assertThat(result.evaluated()).isFalse(); + assertThat(result.message()).contains("Not enough history"); + assertThat(result.value()).isNull(); + assertThat(result.drill()).isNull(); + } + + @Test + void emptyAndMissingNumericBucketsAreNotZero() throws Exception + { + var range = parsed("OUT_OF_RANGE", "AVG", "{\"min\":0,\"max\":1}"); + assertThat(WatchDetectors.evaluate(range, List.of(), "Orders").evaluated()).isFalse(); + assertThat(WatchDetectors.evaluate(range, series((Double) null), "Orders").evaluated()).isFalse(); + assertThat(WatchDetectors.evaluate(range, series(Double.NaN), "Orders").evaluated()).isFalse(); + } + + @ParameterizedTest + @ValueSource(strings = {"[]", "{", "null", "{\"k\":0}", "{\"k\":1001}", "{\"k\":1.5}", "{\"k\":\"14\"}", "{\"threshold\":0}", "{\"threshold\":null}", "{\"relativeChange\":-1}", "{\"delta\":0.1}"}) + void rejectsInvalidSpikeParameters(String json) + { + assertThatThrownBy(() -> parsed("SPIKE", "COUNT", json)).isInstanceOf(QBadRequestException.class); + } + + @Test + void validatesMetricDetectorAndRangeCompatibility() throws Exception + { + assertThatThrownBy(() -> parsed("SPIKE", "CATEGORY_SET", null)).hasMessageContaining("NEW_CATEGORY"); + assertThatThrownBy(() -> parsed("NEW_CATEGORY", "COUNT", null)).hasMessageContaining("CATEGORY_SET"); + assertThatThrownBy(() -> parsed("NULL_RATE_CREEP", "SUM", null)).hasMessageContaining("NULL_SHARE"); + assertThatThrownBy(() -> parsed("OUT_OF_RANGE", "COUNT", "{\"min\":2,\"max\":1}")).hasMessageContaining("min <= max"); + assertThatThrownBy(() -> parsed("OUT_OF_RANGE", "COUNT", null)).hasMessageContaining("requires min and max"); + assertThatThrownBy(() -> parsed("NULL_RATE_CREEP", "NULL_SHARE", "{\"delta\":2}")).hasMessageContaining("delta"); + assertThatThrownBy(() -> parsed("NULL_RATE_CREEP", "NULL_SHARE", "{\"n\":0}")).hasMessageContaining("n must"); + assertThat(parsed("SPIKE", "COUNT", null).k()).isEqualTo(14); + assertThat(parsed("SPIKE", "COUNT", null).threshold()).isEqualTo(3.5); + } +} diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/WatchesTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/WatchesTest.java new file mode 100644 index 0000000..2fd8e38 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/WatchesTest.java @@ -0,0 +1,460 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis; + +import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qbits.dataanalysis.app.DataAnalysisHome; +import com.kingsrook.qbits.dataanalysis.engine.DrillFilters; +import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables; +import com.kingsrook.qbits.dataanalysis.processes.WatchActions; +import com.kingsrook.qbits.dataanalysis.watch.WatchEvaluator; +import com.kingsrook.qbits.dataanalysis.watch.WatchRecords; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.alerts.AlertResolution; +import com.kingsrook.qqq.backend.core.alerts.AlertSignal; +import com.kingsrook.qqq.backend.core.alerts.AlertSinkInterface; +import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/******************************************************************************* + ** Real core memory rows, owner sessions, alerts SPI and record-action summaries. + *******************************************************************************/ +class WatchesTest +{ + static final LocalDate TODAY = LocalDate.now(ZoneOffset.UTC); + static final Instant NOW = TODAY.atTime(12, 0).toInstant(ZoneOffset.UTC); + private QInstance instance; + private DataAnalysisQBitConfig config; + private WatchEvaluator evaluator; + private final List latestIds = new ArrayList<>(); + + public static class Sink implements AlertSinkInterface + { + static final List raised = new ArrayList<>(); + static final List resolved = new ArrayList<>(); + + @Override + public void raise(AlertSignal signal) + { + raised.add(signal); + } + + @Override + public void resolve(AlertResolution resolution) + { + resolved.add(resolution); + } + + @Override + public void heartbeat(String key, Instant at) + { + } + } + + static QSession owner(String user) + { + return (TestSupport.user(user).withPermissions("orders.read", "orders.analyze", "data_analysis.watch.read", "data_analysis.watch.edit", "data_analysis.saved.read") + .withValue(QSession.VALUE_KEY_USER_TIMEZONE, "UTC")); + } + + @BeforeEach + void setup() throws Exception + { + instance = TestSupport.instance(); + instance.addTable(new QTableMetaData().withName("orders").withLabel("Orders").withBackendName("memory").withPrimaryKeyField("id") + .withPermissionRules(DataAnalysisTables.permissionRules("orders")) + .withField(new QFieldMetaData("id", QFieldType.INTEGER)).withField(new QFieldMetaData("amount", QFieldType.DECIMAL)) + .withField(new QFieldMetaData("created", QFieldType.DATE_TIME)).withField(new QFieldMetaData("day", QFieldType.DATE)) + .withField(new QFieldMetaData("category", QFieldType.STRING)).withField(new QFieldMetaData("owner", QFieldType.STRING)) + .withRecordSecurityLock(new RecordSecurityLock().withFieldName("owner").withSecurityKeyType(DataAnalysisTables.USER_KEY))); + new QInstanceEnricher(instance).enrich(); + QContext.init(instance, TestSupport.admin().withPermissions("orders.read", "orders.insert", "orders.edit", "orders.analyze") + .withValue(QSession.VALUE_KEY_USER_TIMEZONE, "UTC")); + int[] counts = {10, 11, 9, 60}; + for(int day = 0; day < 4; day++) + { + for(int i = 0; i < counts[day]; i++) + { + QRecord row = order(TODAY.minusDays(4 - day), "alice", i % 2 == 0 ? null : BigDecimal.TEN, day == 3 ? "new" : "old"); + if(day == 3) + { + latestIds.add(row.getValueInteger("id")); + } + } + } + // Another owner and the incomplete current bucket must never affect Alice. + order(TODAY.minusDays(1), "bob", new BigDecimal("1000000"), "private"); + order(TODAY, "alice", new BigDecimal("1000000"), "incomplete"); + config = new DataAnalysisQBitConfig().withBackendName("memory").withOwnerSessionSupplier(WatchesTest::owner); + new DataAnalysisRuntime(config).addSelfToInstance(instance); + evaluator = DataAnalysisRuntime.watches(); + Sink.raised.clear(); + Sink.resolved.clear(); + AlertsInstanceMetaData.ofOrWithNew(instance).withSink(new QCodeReference(Sink.class)); + } + + private QRecord order(LocalDate day, String owner, BigDecimal amount, String category) throws Exception + { + return (TestSupport.insert("orders", new QRecord().withValue("created", day.atTime(12, 0).toInstant(ZoneOffset.UTC)) + .withValue("day", day).withValue("owner", owner).withValue("amount", amount).withValue("category", category))); + } + + private QRecord watch(String detector, String metric, String parameters) throws Exception + { + return (TestSupport.insert(DataAnalysisTables.WATCH, WatchDetectorsTest.definition(detector, metric, parameters))); + } + + private QRecord stored(QRecord watch) throws Exception + { + return (WatchRecords.get(DataAnalysisTables.WATCH, watch.getValue("id"))); + } + + @AfterEach + void cleanup() + { + QContext.clear(); + } + + @Test + void raisesAndResolvesOnceUsingOwnerRows() throws Exception + { + QRecord watch = watch("SPIKE", "COUNT", "{\"k\":3}"); + QSession caller = QContext.getQSession(); + var verdict = evaluator.evaluate(watch.getValue("id"), NOW); + assertThat(verdict.firing()).isTrue(); + assertThat(verdict.value()).isEqualTo(60.0); + assertThat(QContext.getQSession()).isSameAs(caller); + assertThat(stored(watch).getValueString("state")).isEqualTo("FIRING"); + assertThat(stored(watch).getValueInstant("last_evaluated_at")).isEqualTo(NOW); + assertThat(Sink.raised).hasSize(1); + AlertSignal signal = Sink.raised.getFirst(); + assertThat(signal.getAlertTypeName()).isEqualTo(WatchEvaluator.ALERT_TYPE); + assertThat(signal.getDedupKey()).isEqualTo("data_analysis_watch:" + watch.getValue("id")); + assertThat(signal.getRecordRefs().getFirst().getTableName()).isEqualTo(DataAnalysisTables.WATCH); + assertThat(signal.getLinks().getFirst().getTableName()).isEqualTo("orders"); + assertThat(signal.getLinks().getFirst().getUrl()).isNull(); + assertThat(signal.getLinks().getFirst().getFilter()).isNotNull(); + evaluator.evaluate(watch.getValue("id"), NOW); + assertThat(Sink.raised).hasSize(1); + for(int id : latestIds.subList(10, latestIds.size())) + { + UpdateInput update = new UpdateInput("orders").withRecords(List.of(new QRecord().withValue("id", id) + .withValue("created", TODAY.plusDays(1).atTime(12, 0).toInstant(ZoneOffset.UTC)))); + update.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + assertThat(new UpdateAction().execute(update).getRecords().getFirst().getErrors()).isNullOrEmpty(); + } + assertThat(evaluator.evaluate(watch.getValue("id"), NOW).firing()).isFalse(); + assertThat(stored(watch).getValueString("state")).isEqualTo("OK"); + assertThat(Sink.resolved).hasSize(1); + assertThat(Sink.resolved.getFirst().getDedupKey()).isEqualTo(signal.getDedupKey()); + evaluator.evaluate(watch.getValue("id"), NOW); + assertThat(Sink.resolved).hasSize(1); + } + + @Test + void missingOrMismatchedOwnerSessionsNeverReadAsCaller() throws Exception + { + QRecord watch = watch("SPIKE", "COUNT", "{\"k\":3}"); + config.withOwnerSessionSupplier(id -> null); + assertThat(evaluator.evaluate(watch.getValue("id"), NOW).message()).isEqualTo("Owner session unavailable"); + assertThat(stored(watch).getValue("last_value")).isNull(); + config.withOwnerSessionSupplier(id -> owner("bob")); + assertThat(evaluator.evaluate(watch.getValue("id"), NOW).message()).isEqualTo("Owner session unavailable"); + config.withOwnerSessionSupplier(null); + assertThat(evaluator.evaluate(watch.getValue("id"), NOW).message()).isEqualTo("Owner session unavailable"); + assertThat(Sink.raised).isEmpty(); + } + + @Test + void revokedReadOrAnalyzeSkipsWithoutResolving() throws Exception + { + QRecord watch = watch("SPIKE", "COUNT", "{\"k\":3}"); + evaluator.evaluate(watch.getValue("id"), NOW); + for(String revoked : List.of("orders.read", "orders.analyze")) + { + config.withOwnerSessionSupplier(id -> + { + QSession session = owner(id); + session.getPermissions().remove(revoked); + return (session); + }); + var verdict = evaluator.evaluate(watch.getValue("id"), NOW); + assertThat(verdict.evaluated()).isFalse(); + assertThat(verdict.message()).contains("Owner lacks READ or Analyze"); + assertThat(verdict.value()).isNull(); + assertThat(verdict.drill()).isNull(); + assertThat(stored(watch).getValueString("state")).isEqualTo("FIRING"); + assertThat(stored(watch).getValue("last_value")).isNull(); + } + assertThat(Sink.resolved).isEmpty(); + assertThat(Sink.raised).hasSize(1); + } + + @Test + void previewCoversAllMetricsAndPrivacy() throws Exception + { + QContext.setQSession(owner("alice")); + for(String metric : List.of("COUNT", "SUM", "AVG", "NULL_SHARE", "CATEGORY_SET")) + { + String detector = metric.equals("CATEGORY_SET") ? "NEW_CATEGORY" : metric.equals("NULL_SHARE") ? "NULL_RATE_CREEP" : "SPIKE"; + QRecord definition = WatchDetectorsTest.definition(detector, metric, "{\"k\":3}"); + if(metric.equals("CATEGORY_SET")) + { + definition.withValue("metric_json", "{\"operator\":\"CATEGORY_SET\",\"field\":\"category\"}"); + } + var preview = new DataAnalysisService(config).watchPreview(definition, NOW); + assertThat(preview.series()).hasSize(4); + assertThat(preview.chart().getKind().name()).isEqualTo("TIME_SERIES"); + assertThat(preview.chart().getAnnotations()).isNotEmpty(); + assertThat(preview.verdict().evaluated()).isTrue(); + if(metric.equals("CATEGORY_SET")) + { + assertThat(preview.verdict().firing()).isTrue(); + assertThat(preview.series().getLast().categories()).containsExactly("new"); + } + if(metric.equals("AVG")) + { + assertThat(preview.verdict().value()).isEqualTo(10.0); + } + if(metric.equals("NULL_SHARE")) + { + assertThat(preview.verdict().value()).isEqualTo(0.5); + } + } + var guarded = new DataAnalysisService(new DataAnalysisQBitConfig().withBackendName("memory").withDefaultMinGroupSize(12)) + .watchPreview(WatchDetectorsTest.definition("SPIKE", "COUNT", "{\"k\":3}"), NOW); + assertThat(guarded.series()).isEmpty(); + assertThat(guarded.verdict().message()).contains("Privacy guard"); + assertThat(guarded.verdict().drill()).isNull(); + } + + @Test + void fullFieldPolicyAndRowLimitAreEnforced() throws Exception + { + QContext.setQSession(owner("alice")); + var restricted = new DataAnalysisQBitConfig().withBackendName("memory").withExcludeTables(java.util.Set.of("orders")); + assertThatThrownBy(() -> new DataAnalysisService(restricted).watchPreview(WatchDetectorsTest.definition("SPIKE", "COUNT", "{\"k\":3}"), NOW)) + .hasMessageContaining("unavailable"); + config.withFieldAnalysisAccessResolver(() -> (table, field) -> field.getName().equals("amount") + ? com.kingsrook.qbits.dataanalysis.engine.FieldAnalysisAccess.NONE : com.kingsrook.qbits.dataanalysis.engine.FieldAnalysisAccess.FULL); + QRecord definition = WatchDetectorsTest.definition("SPIKE", "SUM", "{\"k\":3}"); + assertThatThrownBy(() -> new DataAnalysisService(config).watchPreview(definition, NOW)).hasMessageContaining("Unknown analysis field"); + definition.withValue("metric_json", "{\"operator\":\"COUNT\"}").withValue("filter_json", "{\"criteria\":[{\"fieldName\":\"amount\",\"operator\":\"IS_BLANK\"}]}"); + assertThatThrownBy(() -> new DataAnalysisService(config).watchPreview(definition, NOW)).hasMessageContaining("Unknown analysis field"); + } + + @Test + void actionsHaveReviewAndResultLinesAndEnableDisable() throws Exception + { + QRecord spike = watch("SPIKE", "COUNT", "{\"k\":3}"); + QRecord ok = watch("OUT_OF_RANGE", "COUNT", "{\"min\":0,\"max\":100}"); + QRecord secondOk = watch("DROP", "COUNT", "{\"k\":3}"); + RunBackendStepInput input = new RunBackendStepInput().withProcessName(WatchActions.EVALUATE).withRecords(List.of(spike, ok, secondOk)); + RunBackendStepOutput output = new RunBackendStepOutput(); + WatchActions.Review review = new WatchActions.Review(); + review.runOnePage(input, output); + assertThat(review.getProcessSummary(output, false)).hasSize(3).allSatisfy(line -> assertThat(line.toString()).contains("will be evaluated")); + assertThat(Sink.raised).isEmpty(); + WatchActions.Result result = new WatchActions.Result(); + result.runOnePage(input, new RunBackendStepOutput()); + assertThat(result.getProcessSummary(output, true).getFirst().toString()).contains("3 watches evaluated: 1 firing, 2 OK"); + for(String action : List.of(WatchActions.DISABLE, WatchActions.ENABLE)) + { + WatchActions.Result load = new WatchActions.Result(); + load.runOnePage(new RunBackendStepInput().withProcessName(action).withRecords(List.of(spike)), new RunBackendStepOutput()); + assertThat(stored(spike).getValueBoolean("enabled")).isEqualTo(action.equals(WatchActions.ENABLE)); + assertThat(load.getProcessSummary(output, true).getFirst().toString()).contains(action.equals(WatchActions.ENABLE) ? "1 watch enabled" : "1 watch disabled"); + } + } + + @Test + void scheduleEvaluatesEnabledAndHomeLinksFiringFirst() throws Exception + { + QRecord spike = watch("SPIKE", "COUNT", "{\"k\":3}"); + QRecord disabled = watch("DROP", "COUNT", "{\"k\":3}"); + evaluator.setEnabled(disabled.getValue("id"), false); + assertThat(evaluator.evaluateEnabled(NOW)).isEqualTo(1); + assertThat(stored(disabled).getValue("last_evaluated_at")).isNull(); + String html = DataAnalysisHome.render(NOW); + assertThat(html.indexOf("Firing · Spike")).isLessThan(html.indexOf("Ok · Drop")); + assertThat(html).contains("/" + spike.getValue("id") + "\""); + assertThat(instance.getProcess(WatchActions.SCHEDULED).getIsHidden()).isTrue(); + assertThat(instance.getProcess(WatchActions.EVALUATE).getTableName()).isEqualTo(DataAnalysisTables.WATCH); + } + + @Test + void standardCrudValidatesPartialUpdatesAndOwnerLocks() throws Exception + { + QRecord watch = watch("SPIKE", "COUNT", "{\"k\":3}"); + UpdateInput input = new UpdateInput(DataAnalysisTables.WATCH).withRecords(List.of(new QRecord().withValue("id", watch.getValue("id")).withValue("parameters_json", "{\"k\":0}"))); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + assertThat(new UpdateAction().execute(input).getRecords().getFirst().getErrors()).isNotEmpty(); + assertThat(stored(watch).getValueString("parameters_json")).contains("3"); + InsertInput invalid = new InsertInput(DataAnalysisTables.WATCH).withRecord(WatchDetectorsTest.definition("OUT_OF_RANGE", "COUNT", "{}")); + invalid.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + assertThat(new InsertAction().execute(invalid).getRecords().getFirst().getErrors()).isNotEmpty(); + QContext.setQSession(owner("bob")); + assertThatThrownBy(() -> evaluator.preview(watch.getValue("id"), NOW)).hasMessageContaining("unavailable"); + assertThatThrownBy(() -> evaluator.evaluate(watch.getValue("id"), NOW)).hasMessageContaining("unavailable"); + QContext.setQSession(owner("alice")); + QContext.getQSession().getPermissions().remove("data_analysis.watch.edit"); + assertThatThrownBy(() -> evaluator.evaluate(watch.getValue("id"), NOW)).hasMessageContaining("editing is unavailable"); + assertThat(Sink.raised).isEmpty(); + } + + @Test + void savedAnalysisResolvesAsOwnerAndNoAlertsInstallationStillStoresState() throws Exception + { + QRecord saved = TestSupport.insert(DataAnalysisTables.SAVED, TestSupport.saved("alice", false) + .withValue("definition_json", "{\"values\":[{\"operator\":\"COUNT\"}]}")); + QRecord definition = WatchDetectorsTest.definition("SPIKE", "COUNT", "{\"k\":3}") + .withValue("saved_id", saved.getValue("id")).withValue("table_name", null).withValue("metric_json", null); + QRecord watch = TestSupport.insert(DataAnalysisTables.WATCH, definition); + AlertsInstanceMetaData.of(instance).withSink(null); + assertThat(evaluator.evaluate(watch.getValue("id"), NOW).firing()).isTrue(); + assertThat(stored(watch).getValueString("state")).isEqualTo("FIRING"); + assertThat(Sink.raised).isEmpty(); + } + + public static class AutomatedSessions implements com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleCustomizerInterface + { + @Override + public void customizeAutomatedSessionForUser(QInstance instance, QSession session, java.io.Serializable userId) + { + QSession grants = owner(userId.toString()); + session.setPermissions(grants.getPermissions()); + session.setSecurityKeyValues(grants.getSecurityKeyValues()); + session.withValue(QSession.VALUE_KEY_USER_TIMEZONE, "UTC"); + } + } + + @Test + void hostAuthenticationModuleSuppliesTheOwnerSession() throws Exception + { + config.withOwnerSessionSupplier(null); + instance.withInstanceDefaultAuthentication(new com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData() + .withName("owner-auth").withType(com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType.MOCK).withCustomizer(new QCodeReference(AutomatedSessions.class))); + QRecord watch = watch("SPIKE", "COUNT", "{\"k\":3}"); + assertThat(evaluator.evaluate(watch.getValue("id"), NOW).value()).isEqualTo(60.0); + assertThat(Sink.raised).hasSize(1); + } + + @Test + void allCalendarUnitsExcludeCurrentBucketAndHaveExactDrills() throws Exception + { + for(String unit : List.of("HOUR", "DAY", "WEEK", "MONTH")) + { + java.time.ZonedDateTime end = NOW.atZone(ZoneOffset.UTC); + end = switch(unit) + { + case "HOUR" -> end.truncatedTo(java.time.temporal.ChronoUnit.HOURS); + case "WEEK" -> end.toLocalDate().with(java.time.temporal.TemporalAdjusters.previousOrSame(java.time.DayOfWeek.MONDAY)).atStartOfDay(ZoneOffset.UTC); + case "MONTH" -> end.toLocalDate().withDayOfMonth(1).atStartOfDay(ZoneOffset.UTC); + default -> end.toLocalDate().atStartOfDay(ZoneOffset.UTC); + }; + java.time.temporal.ChronoUnit duration = switch(unit) + { + case "HOUR" -> java.time.temporal.ChronoUnit.HOURS; + case "WEEK" -> java.time.temporal.ChronoUnit.WEEKS; + case "MONTH" -> java.time.temporal.ChronoUnit.MONTHS; + default -> java.time.temporal.ChronoUnit.DAYS; + }; + for(int ago : new int[] {2, 1, 1, 0}) + { + TestSupport.insert("orders", new QRecord().withValue("owner", "alice").withValue("category", unit) + .withValue("created", end.minus(ago, duration).toInstant()).withValue("day", end.minus(ago, duration).toLocalDate())); + } + QRecord definition = WatchDetectorsTest.definition("SPIKE", "COUNT", "{\"k\":1}").withValue("bucket", unit) + .withValue("date_field", unit.equals("HOUR") ? "created" : "day") + .withValue("filter_json", com.kingsrook.qqq.backend.core.utils.JsonUtils.toJson(DrillFilters.condition("category", QCriteriaOperator.EQUALS, unit))); + QContext.setQSession(owner("alice")); + var preview = new DataAnalysisService(config).watchPreview(definition, NOW); + assertThat(preview.series()).as(unit).hasSize(2); + assertThat(preview.verdict().value()).as(unit).isEqualTo(2.0); + assertThat(preview.verdict().firing()).as(unit).isTrue(); + var count = new com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput("orders").withFilter(preview.verdict().drill().getFilter()); + count.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + assertThat(new com.kingsrook.qqq.backend.core.actions.tables.CountAction().execute(count).getCount()).as(unit).isEqualTo(2); + QContext.setQSession(TestSupport.admin().withPermissions("orders.insert")); + } + } + + @Test + void streamedProcessRunsSelectedRecordsThroughCore() throws Exception + { + QRecord watch = watch("SPIKE", "COUNT", "{\"k\":3}"); + var input = new com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput().withProcessName(WatchActions.EVALUATE) + .withFrontendStepBehavior(com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput.FrontendStepBehavior.SKIP) + .withCallback(new com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback() + { + @Override + public com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter getQueryFilter() + { + return (DrillFilters.condition("id", QCriteriaOperator.EQUALS, watch.getValue("id"))); + } + }); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + var output = new com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction().execute(input); + assertThat(output.getException()).isEmpty(); + assertThat(output.getValue(com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY).toString()) + .contains("1 watch evaluated: 1 firing, 0 OK"); + assertThat(stored(watch).getValueString("state")).isEqualTo("FIRING"); + } + + @Test + void configuredSchedulerUsesFifteenMinuteInterval() throws Exception + { + var configured = new QInstance(); + configured.addBackend(new com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData().withName("memory") + .withBackendType(com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule.class)); + new DataAnalysisQBitProducer().withQBitConfig(new DataAnalysisQBitConfig().withBackendName("memory").withSchedulerName("hostScheduler")).produce(configured); + var schedule = configured.getProcess(WatchActions.SCHEDULED).getSchedule(); + assertThat(schedule.getSchedulerName()).isEqualTo("hostScheduler"); + assertThat(schedule.getRepeatSeconds()).isEqualTo(900); + new WatchActions.Scheduled().run(new RunBackendStepInput(), new RunBackendStepOutput()); + } +} -- 2.54.0 From d2c9871d98d3bea8e4ce814bc505ecb4a4e38efe Mon Sep 17 00:00:00 2001 From: James Maes Date: Tue, 13 Oct 2026 21:23:23 -0500 Subject: [PATCH 07/10] feat(saved): saved, shared, starred, scheduled and widget analyses, export, SQL handoff Extension 1.4.0: saved analyses of every kind re-run as the viewer (fields they cannot analyze are dropped), owner-or-shared READ through core record locks with owner-only writes (and a database CHECK so a private row can never lose its read owner), personal stars, schedules that run as the owner and mail only recipients who could run the analysis themselves, CSV/XLSX export, the SQL Workbench handoff, an analysis dashboard widget, and an Analyses home built from standard widgets. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 + README.md | 61 +++- docs/DESIGN.md | 2 +- .../dataanalysis/DataAnalysisQBitConfig.java | 55 +++ .../DataAnalysisQBitProducer.java | 7 + .../dataanalysis/DataAnalysisRuntime.java | 20 ++ .../dataanalysis/DataAnalysisSchema.java | 8 + .../dataanalysis/DataAnalysisService.java | 25 ++ .../dataanalysis/api/AnalysisFileSpec.java | 195 +++++++++++ .../dataanalysis/api/DataAnalysisApi.java | 7 +- .../dataanalysis/api/DataAnalysisSchemas.java | 1 + .../qbits/dataanalysis/api/SavedApi.java | 208 ++++++++++++ .../qbits/dataanalysis/api/SavedSchemas.java | 99 ++++++ .../dataanalysis/app/DataAnalysisHome.java | 321 +++++++++--------- .../dataanalysis/app/SavedAnalysisWidget.java | 157 +++++++++ .../dataanalysis/engine/PivotExplorer.java | 3 +- .../engine/RelationshipSupport.java | 3 +- .../engine/StandInAnalysisBackend.java | 3 +- .../DataAnalysisMetaDataProducer.java | 3 +- .../metadata/ScheduleProcesses.java | 93 +++++ .../model/DataAnalysisTables.java | 19 +- .../processes/ScheduleActions.java | 144 ++++++++ .../dataanalysis/saved/AnalysisExport.java | 304 +++++++++++++++++ .../dataanalysis/saved/SavedAnalyses.java | 151 ++++++++ .../dataanalysis/saved/SavedCustomizer.java | 91 +++++ .../dataanalysis/saved/SavedDefinition.java | 313 +++++++++++++++++ .../qbits/dataanalysis/saved/SavedResult.java | 60 ++++ .../qbits/dataanalysis/saved/SqlHandoff.java | 147 ++++++++ .../schedule/ScheduleCustomizer.java | 116 +++++++ .../dataanalysis/schedule/ScheduleRunner.java | 305 +++++++++++++++++ .../dataanalysis/schedule/ScheduleTiming.java | 111 ++++++ .../openapi/qqq-v1/data-analysis.yaml | 297 +++++++++++++++- .../dataanalysis/DataAnalysisApiTest.java | 65 +++- .../dataanalysis/DataAnalysisHomeTest.java | 150 +++----- .../DataAnalysisMetadataTest.java | 7 +- .../DataAnalysisPossibleValuesTest.java | 8 +- .../DataAnalysisSecurityTest.java | 6 +- .../qbits/dataanalysis/SavedAnalysesTest.java | 195 +++++++++++ .../qbits/dataanalysis/SavedSqlH2Test.java | 72 ++++ .../qbits/dataanalysis/SchedulesTest.java | 231 +++++++++++++ .../qbits/dataanalysis/TestSupport.java | 4 +- .../qbits/dataanalysis/WatchesTest.java | 8 +- 42 files changed, 3786 insertions(+), 296 deletions(-) create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/api/AnalysisFileSpec.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/api/SavedApi.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/api/SavedSchemas.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/app/SavedAnalysisWidget.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/metadata/ScheduleProcesses.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/processes/ScheduleActions.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/saved/AnalysisExport.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/saved/SavedAnalyses.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/saved/SavedCustomizer.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/saved/SavedDefinition.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/saved/SavedResult.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/saved/SqlHandoff.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/schedule/ScheduleCustomizer.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/schedule/ScheduleRunner.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/schedule/ScheduleTiming.java create mode 100644 src/test/java/com/kingsrook/qbits/dataanalysis/SavedAnalysesTest.java create mode 100644 src/test/java/com/kingsrook/qbits/dataanalysis/SavedSqlH2Test.java create mode 100644 src/test/java/com/kingsrook/qbits/dataanalysis/SchedulesTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 744d857..31fc8eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ Following [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and [Semantic ### Added +- Phase f saved execution for all seven kinds, core owner-or-shared READ with owner-only WRITE, typed definition validation and viewer-specific field reduction. +- Idempotent personal star endpoints and visible saved list; transient viewer runs, core CSV/XLSX downloads and QueryStat-backed SQL Workbench handoff. +- Five-minute owner-session schedule runner, per-recipient FULL access checks and record-scoped reruns, core messaging with attachments, explicit failure statuses/alerts, timezone/weekly/monthly/cron timing and streamed ETL Send now. +- Standard statistics/chart/table dashboard widgets, linked saved analysis widgets and real home rows with narrow tables and honest empty states. +- Extension 1.4.0, seven new assistant-discoverable endpoints, regenerated OpenAPI and memory/H2/HTTP/export/email/security regression coverage. +- Migration requirement: backfill saved `read_owner_user_id` before enabling shared reads and widen schedule `last_status` to TEXT. Stock core EMAIL attachment limitations fail visibly; supported messaging providers receive full attachments. + - Phase e anomaly watches: owner-locked configuration, validated metrics/detector parameters, complete calendar buckets, five pure detectors, and scheduled evaluation through owner sessions. - Core alerts SPI type `data.analysis.anomaly` (core disallows the spec's underscore), stable watch keys, relative drill links, raise/recovery transitions, and persisted outcomes without an alerts implementation. - Standard streamed ETL Evaluate now / Enable / Disable record actions and firing-first linked watches on the Analyses home. diff --git a/README.md b/README.md index 411a05c..d12de46 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Statistical analysis and data dives on any QQQ table, filter or saved view: prof The host API provides column and table profiles, pivots, comparisons, distribution shifts, correlations, scatter samples, crosstabs, distinct-value pages, duplicate groups, drill-through filters, privacy suppression, and asynchronous exact profile runs with a persistent per-user cache. Metadata, six visible tables, owner locks, PostgreSQL DDL generation and the Analyses home are also available. -Fourteen HTTP operations are available under `/qqq/v1/qbit/data-analysis`; the Analyze UI is a later phase. Pivot and relationship backends are available. Anomaly watches, their scheduled evaluator and alert transitions are available. Saved-analysis execution, scheduled delivery and exports remain later phases. +Twenty-one HTTP operations are available under `/qqq/v1/qbit/data-analysis` (extension **1.4.0**). Saved execution, shared reads, personal stars, scheduled email, CSV/XLSX exports, SQL Workbench handoff and standard dashboard widgets are implemented. The separate Analyze/pivot/relationship frontend remains outside this repository. ## Requirements @@ -66,7 +66,7 @@ The four `table_name` fields use core's `tables` possible-value source; an exist Every engine operation and home count runs under the current `QContext` session with `PermissionEnforcement.ENFORCE`. A denied section explains the missing permission instead of showing zero. Mine and starred counts explicitly filter the current user even for administrators. Shared counts include only readable shared rows. Stars pointing at unreadable or deleted saved analyses do not count. Run counts cover starts within the last seven days, excluding future rows. -Saved analyses have a READ_AND_WRITE lock on `owner_user_id`; stars and runs have locks on `user_id`. Schedules inherit the saved analysis owner's lock through their join. Null owners are denied. The host's authentication/session adapter must supply: +Saved analyses have a WRITE lock on `owner_user_id` and a separate READ lock on the hidden `read_owner_user_id` projection. Core pre-insert/update customizers set that projection to the owner for private rows and null for shared rows; the READ lock allows null. This expresses **owner OR shared READ**, while core retains owner-only writes and administrator all-access keys. Stars and runs have locks on `user_id`. Schedules inherit the saved analysis owner's lock through their join. Null owners are denied. The host's authentication/session adapter must supply: ```java session.withSecurityKeyValue("data_analysis_user", session.getUser().getIdReference()); @@ -77,7 +77,18 @@ session.withPermission("data_analysis.admin") QQQ permissions and security-key values are separate. An admin permission alone does not bypass record locks. User-id fields must use the same stable string identity as `QUser.idReference`. -**Shared READ is deliberately restricted in this scaffold.** The ordinary owner lock does not express owner OR shared while keeping writes owner-only, so setting `shared=true` currently does not expose a saved analysis to other users. Phase f will introduce separate read and write policies: reads permit owner OR shared only after checking `data_analysis.saved.read`; writes retain owner/admin checks. Query, get, count, joins, child lists and exports will receive cross-user regression tests before enabling shared READ. Collaboration sharing and favorites (§7.3, qqq#1136) will then replace the local sharing/star contracts. Shared analyses must always execute as their viewer. +**Shared READ is enabled through core record locks.** A viewer still needs `data_analysis.saved.read`. Query/get/count, standard CRUD, saved runs, lists and exports use core enforcement. Shared viewers cannot edit, delete, change ownership, or schedule another owner's analysis. No custom query filter grants access. Sharing exposes the saved definition and metadata, not the owner's data: every run computes under the viewer's source-table READ/Analyze permissions and field policy. There is no per-recipient/group sharing or access-request workflow yet; collaboration's sharing contract ([qqq#1136](https://github.com/QRun-IO/qqq/issues/1136)) will replace the local flag. Stars move to `collaboration_favorite` when collaboration ships. + +**Existing databases must migrate before enabling the new metadata.** A missing/null read-owner projection would make an old private row readable. Apply this through the host's migration system while writes are stopped, before deploying this version: + +```sql +ALTER TABLE data_analysis_saved ADD COLUMN IF NOT EXISTS read_owner_user_id VARCHAR(250); +UPDATE data_analysis_saved + SET read_owner_user_id = CASE WHEN shared IS TRUE THEN NULL ELSE owner_user_id END; +ALTER TABLE data_analysis_schedule ALTER COLUMN last_status TYPE TEXT; +``` + +The metadata-generated create script includes these fields for new installations; it does not migrate existing databases. Fix older placeholder definitions (such as `{}` for a column profile) to valid typed requests. Edit existing enabled schedules through core CRUD to validate recipients and initialize `next_run_at`. Minimum group size defaults to 0 (off). Hosts should set it for personal-data tables. Groups, bins and distinct listings below N are merged into an Other row with no value or drill. Min/max, percentiles, mean and standard deviation are suppressed when the assessed population is below N. Outlier listings and their count's drill are removed whenever the guard is enabled. The guard runs before chart construction and before `result_json` is stored. @@ -233,15 +244,55 @@ A transition from OK to FIRING raises through core `QAlerts`, with the watch rec `DataAnalysisSchema.postgresScript(config)` generates the new fields and owner requirement. Existing installations must add `date_field VARCHAR(250)`, `owner_user_id VARCHAR(250)`, `last_value NUMERIC`, and `last_message TEXT`, then backfill each watch's actual owner/date field before making the first two columns NOT NULL and enabling evaluation. The CREATE-IF-NOT-EXISTS generator does not migrate existing tables. +## Saved analyses, stars and exports + +Create/edit saved analyses through ordinary core table CRUD. `kind` selects `COLUMN_PROFILE`, `TABLE_PROFILE`, `PIVOT`, `COMPARE`, `CORRELATION`, `CROSSTAB` or `SHIFT`; `definition_json` stores that kind's typed request. For example, a column profile has `{"fieldName":"amount","filter":{...}}`, and a pivot has `{"rows":[{"field":"status"}],"values":[{"operator":"COUNT"}]}`. `table_name` identifies the source table; a supplied `tableName` in the JSON must match. The customizers reject malformed JSON, missing selections, nonexistent tables/fields and invalid partial edits with plain record errors. A typed definition filter takes precedence over legacy `filter_json`. + +`POST /saved/{id}/run` re-runs as the viewer and returns `{kind, warnings, columnProfile, tableProfile, pivot, compare, correlation, crosstab, shift}`, with the matching typed member populated. Selected fields without FULL analysis access are removed and `Some fields are not available to you` is returned. If a required axis, all required selections, or a filter predicate is unavailable, no computation is performed and the typed members are null; filters are never dropped to broaden the population. Saved profile runs are transient, so they do not require run-table write permission and cannot reuse an owner's cache. + +`GET /saved` lists core-visible saved metadata with each viewer's `starred` flag. `PUT /saved/{id}/star` and `DELETE /saved/{id}/star` are idempotent and always target the signed-in user, using standard saved INSERT/DELETE grants. The local star table remains available through ordinary CRUD and has a unique `(saved_id,user_id)` key. + +`GET /saved/{id}/export?format=csv|xlsx` re-runs as the viewer. `POST /export` accepts a discriminated request: + +```json +{"kind":"PIVOT","tableName":"orders","request":{"rows":[{"field":"status"}],"values":[{"operator":"COUNT"}]},"format":"xlsx"} +``` + +Both routes return file bytes with attachment headers. Core `ReportFormat` CSV/XLSX streamers write computed result tables: profile metrics/top values, pivot grids, comparisons, correlation/crosstab tables and shift buckets. Values retain their formatting and privacy suppression. Warnings are included in downloads. All paths above are relative to `/qqq/v1/qbit/data-analysis` and appear in assistant operation discovery. + +## Scheduled delivery + +Configure `schedulerName`, `messagingProviderName`, `fromAddress`, `ownerSessionSupplier` and `recipientAddressSupplier` in `DataAnalysisQBitConfig`. The session supplier resolves both owners and recipients by their stable user id; without it, the authentication module's `createAutomatedSessionForUser` is used. The address supplier maps those ids to verified directory email addresses; schedule recipients are **user ids**, never arbitrary email addresses. + +Schedules use `recipients` as a JSON array (1–100 ids), `format` as `HTML_SUMMARY`, `CSV` or `XLSX`, and either Quartz cron (including the seconds field) or `frequency` + `day` + `time` + `time_zone`. Weekly days are ISO Monday=1 through Sunday=7; monthly days are 1–31, clamped to the last day of shorter months. Times are local ISO times such as `09:00`; omitted time zone is UTC. Spring-forward gaps move forward, and the first occurrence is chosen in an autumn overlap. `next_run_at` is strictly after the evaluation time. + +The hidden `data_analysis_run_schedules` process repeats every 300 seconds when a scheduler is configured. Its scheduler session needs `data_analysis_run_schedules.hasAccess`. Inventory/status operations use a narrow maintenance session; source computation always switches to the saved owner. Every recipient must resolve to the correct identity and pass source READ, Analyze and FULL access for all selected/filter fields. Different recipients receive a fresh computation under their own record locks. Skipped ids appear in `last_status`. + +Email includes escaped HTML headline metrics and a result table (first 20 rows for pivots), with the requested CSV/XLSX attachment via core `SendMessageAction`. `HTML_SUMMARY` has no attachment. Missing messaging configuration and delivery errors persist an explicit failure and raise the declared **`data.analysis.schedule.failed`** alert. Core rejects underscores in alert names, so the spec's `data_analysis.schedule_failed` spelling cannot be registered. With no alert sink, core logs the signal and the status remains visible. + +The installed core's stock `EMAIL` provider silently ignores `SendMessageInput.attachmentList`. CSV/XLSX schedules therefore reject that provider with an explicit status/alert; configure an attachment-capable core messaging provider. A test provider verifies the complete message and attachment contract. This qbit does not alter core's SMTP transport. + +**Send now** uses the standard streamed ETL Review/Result record action. Reviews do not deliver; execution reauthorizes and reports actual outcomes. Disabled schedules can be sent manually. A runtime serializes scheduled and manual attempts, but there is no cross-node transaction or exactly-once email guarantee; run the scheduler on one node. New configurations compute their next occurrence; failed attempts advance to the next occurrence instead of retrying every five minutes. + +## SQL Workbench and dashboard widgets + +`POST /tables/{table}/sql` accepts `{kind,request}` and returns `{available,sql,note}`. It requires an installed SQL Workbench app/table and the configured app access permission (default `sql_workbench.hasAccess`), plus source permissions. Memory/non-RDBMS backends report unavailable. The engine executes the request as the viewer and reads the actual main aggregate's core `QueryStat`; grouped dimensions and the widest metric batch identify the main statement. Bound `?` placeholders remain, and the note states that QQQ applies record security at runtime and the SQL is for reading. + +Core exposes `QueryStat` on the backend but not the backend reference on `AggregateAction`. A narrow read-only reflective bridge accesses that reference after normal core execution. Query-stat capability is enabled only on the request's cloned table metadata; no global logger or statistics-manager settings change. If core changes that private field or does not provide SQL, the endpoint reports unavailable rather than fabricating a statement. + +The Analyses home uses standard `multiStatistics` and `table` widgets: mine/shared/starred counts, enabled/firing watches, my runs in the last seven days, starred analysis links, firing watches and recent run links. All counts and rows use the viewer's session. Tables have at most four columns and use Next's responsive table/card renderer, with explicit empty/permission states. + +`SavedAnalysisWidget.widget()` supplies the **`DATA_ANALYSIS_SAVED`** metadata. Clone/rename it for another dashboard and set `.withDefaultValue("savedId", id)`. Its standard chart renderer re-runs as the viewer, shows formatted headline metrics, projects the semantic series/pivot/crosstab into a standard chart, and links back to the saved record. It never displays an owner's cached result. The standard chart is a dashboard summary; the full analysis UI remains responsible for specialized box/heatmap presentations. + ## Adopting in another app 1. Add a backend, install the producer and run the usual `QInstanceValidator`. 2. For PostgreSQL, obtain `DataAnalysisSchema.postgresScript(config)` and apply it through your application's migration system. It derives fields and unique indexes from table metadata. It does not execute DDL or upgrade existing schemas. 3. Map user-id security keys and permissions in the authentication adapter. Configure core audit persistence if field audit history is needed. 4. Review per-table include/exclude sets, limits and minimum group sizes before enabling analysis. Integrate Saved Analyses and Watches with host ESB/Quick Search when those optional host features are installed; this scaffold does not add those dependencies. -5. Use the Analyses home as the owner of analysis data. Other dashboards should link there until saved-analysis widgets are implemented. +5. Use the Analyses home as the owner of analysis data. Other dashboards can place a configured DATA_ANALYSIS_SAVED widget linked to its saved record. -Phase **b** profiles, drill-through, runs/cache and HTTP endpoints, phase **c** pivot backend, phase **d** compare/relationships backend, and phase **e** anomaly watches are implemented. Remaining work: **b** analysis UI; **c** pivot UI; **d** comparison/relationship UI; **f** saved-analysis execution, sharing, stars integration, schedules, chart widgets, full home and exports. The current home summarizes stored rows; the host facade executes profiles. The home does not send deliveries. +Backend phases **b–f** are implemented, including saved/shared/starred analyses, schedules, standard widgets, home, export and SQL handoff. The separate profile, pivot and relationship frontend remains outside this repository. ## Build diff --git a/docs/DESIGN.md b/docs/DESIGN.md index e3b2d27..f9d9275 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -2,7 +2,7 @@ Approved specification: [QQQ docs/DATA-ANALYSIS.md](https://github.com/QRun-IO/qqq/blob/develop/docs/DATA-ANALYSIS.md), introduced in [PR #1137](https://github.com/QRun-IO/qqq/pull/1137). -Data Analysis will provide profiles, pivots, comparisons, relationships and anomaly watches on existing QQQ data. It will push supported computation to the backend, use bounded streamed fallbacks, enforce the viewer’s permissions and privacy limits, and store configuration, saved definitions, schedules and run history in ordinary visible tables. This repository phase supplies those tables, permissions, configuration, PostgreSQL schema generation and an Analyses home with real session-scoped counts; analytical computation and delivery are planned. +Data Analysis will provide profiles, pivots, comparisons, relationships and anomaly watches on existing QQQ data. It will push supported computation to the backend, use bounded streamed fallbacks, enforce the viewer’s permissions and privacy limits, and store configuration, saved definitions, schedules and run history in ordinary visible tables. Backend phases b–f are implemented here: typed analysis engines, saved/shared/starred definitions, schedules, exports, SQL handoff and standard widgets with session-scoped data. Specialized analysis frontend work lives in Next. README documents the required shared-read migration and the installed core transport limitations. ## Phases (spec §11) diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitConfig.java b/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitConfig.java index ea24926..64858c6 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitConfig.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitConfig.java @@ -426,4 +426,59 @@ public class DataAnalysisQBitConfig implements QBitConfig ownerSessionSupplier = supplier; return (this); } + + private String messagingProviderName; + private String fromAddress; + private transient java.util.function.Function recipientAddressSupplier; + + /******************************************************************************* + ** Core messaging provider used for scheduled delivery. + *******************************************************************************/ + public String getMessagingProviderName() + { + return (messagingProviderName); + } + + /******************************************************************************* + ** Select the host's configured SMTP or SES provider. + *******************************************************************************/ + public DataAnalysisQBitConfig withMessagingProviderName(String value) + { + messagingProviderName = value; + return (this); + } + + /******************************************************************************* + ** Verified sender address supplied by the host. + *******************************************************************************/ + public String getFromAddress() + { + return (fromAddress); + } + + /******************************************************************************* + ** Set the scheduled email sender. + *******************************************************************************/ + public DataAnalysisQBitConfig withFromAddress(String value) + { + fromAddress = value; + return (this); + } + + /******************************************************************************* + ** Resolve a user id to its current email; never treat recipients as addresses. + *******************************************************************************/ + public java.util.function.Function getRecipientAddressSupplier() + { + return (recipientAddressSupplier); + } + + /******************************************************************************* + ** Bind the host directory's user-to-email lookup. + *******************************************************************************/ + public DataAnalysisQBitConfig withRecipientAddressSupplier(java.util.function.Function value) + { + recipientAddressSupplier = value; + return (this); + } } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitProducer.java b/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitProducer.java index 4e38760..5e22281 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitProducer.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitProducer.java @@ -144,6 +144,13 @@ public class DataAnalysisQBitProducer implements QBitMetaDataProducer +{ + /******************************************************************************* + ** POST an arbitrary supported typed request plus CSV/XLSX format. + *******************************************************************************/ + @Override + public BasicOperation defineBasicOperation() + { + return (new BasicOperation().withPath(DataAnalysisApi.PREFIX + "/export").withHttpMethod(HttpMethod.POST) + .withShortSummary("Export an analysis result").withLongDescription("Execute a typed analysis as the signed-in viewer and download its result table as CSV or XLSX. Requires source READ/Analyze and FULL fields; uses core report formats.")); + } + + /******************************************************************************* + ** Saved export has only id/format parameters; ad hoc export uses a JSON body. + *******************************************************************************/ + @Override + public List defineRequestParameters() + { + return (List.of()); + } + + /******************************************************************************* + ** Typed discriminated request plus file format. + *******************************************************************************/ + @Override + public RequestBody defineRequestBody() + { + return (new RequestBody().withRequired(true).withContent(Map.of("application/json", new Content() + .withSchema(new Schema().withRef("#/components/schemas/DataAnalysisSavedRequestV1"))))); + } + + /******************************************************************************* + ** Both file representations are declared, without JSON response wrapping. + *******************************************************************************/ + @Override + public Map defineResponses() + { + Schema file = new Schema().withRef("#/components/schemas/DataAnalysisFileV1"); + return (Map.of(200, new Response().withDescription("Analysis result file").withContent(new java.util.TreeMap<>(Map.of("text/csv", new Content().withSchema(file), + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", new Content().withSchema(file)))))); + } + + /******************************************************************************* + ** Read transport data only; execution waits for middleware session setup. + *******************************************************************************/ + @Override + public Input buildInput(Context context) throws Exception + { + Input input = new Input(); + if(this instanceof Saved) + { + input.id = SavedApi.id(context.pathParam("id")); + input.format = context.queryParam("format"); + } + else + { + if(context.contentType() == null || !context.contentType().toLowerCase(java.util.Locale.ROOT).startsWith("application/json")) + { + throw new QBadRequestException("The request body must be JSON."); + } + input.body = SavedDefinition.object(context.body()); + input.format = input.body.get("format") instanceof String value ? value : null; + } + AnalysisExport.format(input.format); + return (input); + } + + /******************************************************************************* + ** Actual file bytes, a stable filename and the matching MIME type. + *******************************************************************************/ + @Override + public void handleOutput(Context context, Output output) + { + context.contentType(output.file.mimeType()).header("Content-Disposition", "attachment; filename=\"" + output.file.filename() + "\"") + .header("Cache-Control", "no-store").result(output.file.bytes()); + } + + /******************************************************************************* + ** Saved GET variant uses the same file executor. + *******************************************************************************/ + public static class Saved extends AnalysisFileSpec + { + /******************************************************************************* + ** Core-visible saved definition is re-run as viewer before export. + *******************************************************************************/ + @Override + public BasicOperation defineBasicOperation() + { + return (new BasicOperation().withPath(DataAnalysisApi.PREFIX + "/saved/{id}/export").withHttpMethod(HttpMethod.GET) + .withShortSummary("Export a saved analysis").withLongDescription("Re-run a visible saved analysis as the viewer, applying current source permissions and dropping unavailable fields, then download its result table as CSV or XLSX.")); + } + + /******************************************************************************* + ** Required id and CSV/XLSX query format. + *******************************************************************************/ + @Override + public List defineRequestParameters() + { + return (List.of(new Parameter().withName("id").withIn(In.PATH).withRequired(true).withDescription("Visible saved analysis id.").withSchema(new Schema().withType(Type.INTEGER)), + new Parameter().withName("format").withIn(In.QUERY).withRequired(true).withDescription("Result file format.").withSchema(new Schema().withType(Type.STRING).withEnumValues(List.of("csv", "xlsx"))))); + } + + /******************************************************************************* + ** GET has no request body. + *******************************************************************************/ + @Override + public RequestBody defineRequestBody() + { + return (null); + } + } + + /******************************************************************************* + ** Input data only, inherited middleware fields carry the authenticated session. + *******************************************************************************/ + public static class Input extends AbstractMiddlewareInput + { + private Integer id; + private String format; + private Map body; + } + + /******************************************************************************* + ** Binary output stays out of JSON serialization. + *******************************************************************************/ + public static class Output implements AbstractMiddlewareOutputInterface + { + private AnalysisExport.File file; + } + + /******************************************************************************* + ** Invoked after the normal middleware session pipeline. + *******************************************************************************/ + public static class Executor extends AbstractMiddlewareExecutor + { + /******************************************************************************* + ** No direct source access outside the existing authorized analysis engine. + *******************************************************************************/ + @Override + public void execute(Input input, Output output) throws QException + { + var result = input.id == null ? SavedApi.definition(input.body, null).run(DataAnalysisRuntime.service(), true) + : DataAnalysisRuntime.saved().run(input.id); + output.file = AnalysisExport.file(result, input.format); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisApi.java b/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisApi.java index ebd7532..692d5c8 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisApi.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisApi.java @@ -76,8 +76,10 @@ public class DataAnalysisApi *******************************************************************************/ public static MiddlewareSpecExtension extension() { - return (new MiddlewareSpecExtension().withName("data-analysis").withSemanticVersion("1.3.0") + return (new MiddlewareSpecExtension().withName("data-analysis").withSemanticVersion("1.4.0") .withDescription("Profiles, pivots, comparisons, distribution shifts, correlations, scatter samples, crosstabs and anomaly watch previews as the signed-in user.") + .withSpec(SavedApi.Run.class).withSpec(SavedApi.ListSaved.class).withSpec(SavedApi.Star.class).withSpec(SavedApi.Unstar.class) + .withSpec(SavedApi.Sql.class).withSpec(AnalysisFileSpec.class).withSpec(AnalysisFileSpec.Saved.class) .withSpec(CapabilitiesSpec.class).withSpec(ColumnProfileSpec.class).withSpec(DistinctValuesSpec.class) .withSpec(CompareSpec.class).withSpec(ShiftSpec.class).withSpec(CorrelationSpec.class).withSpec(ScatterSpec.class).withSpec(CrosstabSpec.class) .withSpec(WatchPreviewSpec.class).withSpec(PivotSpec.class).withSpec(TableProfileSpec.class).withSpec(DuplicatesSpec.class).withSpec(StartRunSpec.class).withSpec(GetRunSpec.class)); @@ -463,8 +465,7 @@ public class DataAnalysisApi result.canAnalyze = false; result.fields = List.of(); result.backendCapabilities = List.of(); - result.sqlWorkbenchAvailable = QContext.getQInstance().getApp("sqlWorkbench") != null - || QContext.getQInstance().getTables().keySet().stream().anyMatch(t -> t.startsWith("sql_workbench_")); + result.sqlWorkbenchAvailable = com.kingsrook.qbits.dataanalysis.saved.SqlHandoff.installedAndAllowed(); try { var capabilities = DataAnalysisRuntime.service().capabilities(name); diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java b/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java index 2a0d845..d095fa7 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java @@ -116,6 +116,7 @@ public class DataAnalysisSchemas result.put("DataAnalysisPivotV1", new DataAnalysisPivotV1().toSchema()); result.putAll(RelationshipSchemas.components()); result.putAll(WatchSchemas.components()); + result.putAll(SavedSchemas.components()); return (result); } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/api/SavedApi.java b/src/main/java/com/kingsrook/qbits/dataanalysis/api/SavedApi.java new file mode 100644 index 0000000..abbc6e2 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/api/SavedApi.java @@ -0,0 +1,208 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.api; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.dataanalysis.DataAnalysisRuntime; +import com.kingsrook.qbits.dataanalysis.saved.SavedDefinition; +import com.kingsrook.qbits.dataanalysis.saved.SqlHandoff; +import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.openapi.model.HttpMethod; + +/******************************************************************************* + ** Saved execution, stars, visible lists and SQL discovery through the UI API. + *******************************************************************************/ +public final class SavedApi +{ + /******************************************************************************* + ** Namespace only. + *******************************************************************************/ + private SavedApi() + { + } + + /******************************************************************************* + ** Stable integer-id errors, without probing inaccessible records. + *******************************************************************************/ + public static Integer id(String value) throws QBadRequestException + { + try + { + return (Integer.valueOf(value)); + } + catch(Exception e) + { + throw new QBadRequestException("Saved analysis id must be an integer."); + } + } + + /******************************************************************************* + ** Accept a discriminated typed request; path table overrides only absent tableName. + *******************************************************************************/ + @SuppressWarnings("unchecked") + public static SavedDefinition definition(Map body, String pathTable) throws QException + { + if(!(body.get("kind") instanceof String kind)) + { + throw new QBadRequestException("An analysis kind is required."); + } + Map request = new LinkedHashMap<>(body); + request.remove("kind"); + request.remove("format"); + if(body.containsKey("request")) + { + if(!(body.get("request") instanceof Map map)) + { + throw new QBadRequestException("request must be a typed analysis object."); + } + request = new LinkedHashMap<>((Map) map); + } + Object table = pathTable == null ? body.getOrDefault("tableName", request.get("tableName")) : pathTable; + if(!(table instanceof String name)) + { + throw new QBadRequestException("tableName is required."); + } + return (SavedDefinition.parse(kind, name, request)); + } + + /******************************************************************************* + ** Run the visible saved definition under the viewer's current source permissions. + *******************************************************************************/ + public static class Run extends DataAnalysisApi.ProfileSpec + { + /******************************************************************************* + ** Publish assistant-oriented semantics and field-reduction behavior. + *******************************************************************************/ + public Run() + { + super("/saved/{id}/run", HttpMethod.POST, "Run a saved analysis", + "Re-run a visible saved analysis as the signed-in viewer. Requires saved READ and source READ/Analyze. Drops unavailable fields with a warning and returns the result typed by kind; inaccessible required axes or predicates produce no result.", null, "SavedResult"); + } + + /******************************************************************************* + ** Core locks determine visibility, not an endpoint-specific owner check. + *******************************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + return (DataAnalysisRuntime.saved().run(id(input.getPathParam("id")))); + } + } + + /******************************************************************************* + ** Visible saved metadata with this viewer's personal stars. + *******************************************************************************/ + public static class ListSaved extends DataAnalysisApi.ProfileSpec + { + /******************************************************************************* + ** A discoverable list rather than exposing the owner's cached result JSON. + *******************************************************************************/ + public ListSaved() + { + super("/saved", HttpMethod.GET, "List visible saved analyses", + "List core-visible saved analyses with name, kind, source table label, owner, shared state, this viewer's starred flag and last update.", null, "SavedList"); + } + + /******************************************************************************* + ** The list and star lookup both use enforced core queries. + *******************************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + return (Map.of("analyses", DataAnalysisRuntime.saved().list())); + } + } + + /******************************************************************************* + ** Idempotent star PUT for the signed-in user. + *******************************************************************************/ + public static class Star extends DataAnalysisApi.ProfileSpec + { + /******************************************************************************* + ** No user-id body is accepted; stars always belong to the caller. + *******************************************************************************/ + public Star() + { + super("/saved/{id}/star", HttpMethod.PUT, "Star a saved analysis", + "Star a visible saved analysis for the signed-in user. Idempotent; uses standard saved insert permission and the personal star record lock.", null, "Star"); + } + + /******************************************************************************* + ** Persist through core table CRUD. + *******************************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + return (Map.of("starred", DataAnalysisRuntime.saved().star(id(input.getPathParam("id")), true))); + } + } + + /******************************************************************************* + ** Idempotent star DELETE for the signed-in user. + *******************************************************************************/ + public static class Unstar extends DataAnalysisApi.ProfileSpec + { + /******************************************************************************* + ** Removal never targets another user's star. + *******************************************************************************/ + public Unstar() + { + super("/saved/{id}/star", HttpMethod.DELETE, "Unstar a saved analysis", + "Remove the signed-in user's star on a visible saved analysis. Idempotent; uses standard saved delete permission and owner locks.", null, "Star"); + } + + /******************************************************************************* + ** Persist through core table CRUD. + *******************************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + return (Map.of("starred", DataAnalysisRuntime.saved().star(id(input.getPathParam("id")), false))); + } + } + + /******************************************************************************* + ** Actual backend aggregate SQL, captured only during this request. + *******************************************************************************/ + public static class Sql extends DataAnalysisApi.ProfileSpec + { + /******************************************************************************* + ** The assistant gets a reading aid, never an alternative execution endpoint. + *******************************************************************************/ + public Sql() + { + super("/tables/{table}/sql", HttpMethod.POST, "Read an analysis aggregate SQL statement", + "Execute a typed analysis as the viewer and capture the main RDBMS aggregate SQL from core QueryStat. Requires installed SQL Workbench and its access permission. Non-RDBMS backends report unavailable; SQL is for reading and record security remains enforced at runtime.", "SavedRequest", "Sql"); + } + + /******************************************************************************* + ** Requests pass through the same parser and analysis authorization. + *******************************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + return (SqlHandoff.execute(definition(input.getBody(), input.getPathParam("table")), DataAnalysisRuntime.service())); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/api/SavedSchemas.java b/src/main/java/com/kingsrook/qbits/dataanalysis/api/SavedSchemas.java new file mode 100644 index 0000000..b3d6770 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/api/SavedSchemas.java @@ -0,0 +1,99 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.api; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.openapi.model.Schema; +import com.kingsrook.qqq.openapi.model.Type; + +/******************************************************************************* + ** Versioned saved envelopes reuse all seven existing typed request/result schemas. + *******************************************************************************/ +public final class SavedSchemas +{ + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private SavedSchemas() + { + } + + /******************************************************************************* + ** JSON Schema union supported by the standard OpenAPI serializer. + *******************************************************************************/ + public static class RequestUnion extends Schema + { + /******************************************************************************* + ** All supported typed analysis requests, selected by the sibling kind. + *******************************************************************************/ + public List getAnyOf() + { + return (List.of("ColumnProfile", "TableProfile", "Pivot", "Compare", "Correlation", "Crosstab", "Shift").stream() + .map(name -> ref(name + "Request")).toList()); + } + } + + /******************************************************************************* + ** Every new endpoint's request and response is discoverable by clients. + *******************************************************************************/ + public static Map components() + { + Map result = new LinkedHashMap<>(); + Map fields = new LinkedHashMap<>(); + fields.put("kind", new Schema().withType(Type.STRING).withEnumValues(List.of("COLUMN_PROFILE", "TABLE_PROFILE", "PIVOT", "COMPARE", "CORRELATION", "CROSSTAB", "SHIFT"))); + fields.put("tableName", new Schema().withType(Type.STRING).withDescription("Source QQQ table name; required for ad hoc export, supplied by the path for SQL.")); + fields.put("request", new RequestUnion().withDescription("Typed request matching kind. Field names, filters and dimensions use the existing analysis request contracts.")); + fields.put("format", new Schema().withType(Type.STRING).withEnumValues(List.of("csv", "xlsx")).withDescription("Required for export; ignored for SQL.")); + result.put("DataAnalysisSavedRequestV1", new DataAnalysisSchemas.RequiredSchema(List.of("kind", "request")).withType(Type.OBJECT).withProperties(fields)); + Map response = new LinkedHashMap<>(); + response.put("kind", fields.get("kind")); + response.put("warnings", new Schema().withType(Type.ARRAY).withItems(new Schema().withType(Type.STRING))); + new java.util.TreeMap<>(Map.of("columnProfile", "ColumnProfile", "tableProfile", "TableProfile", "pivot", "Pivot", "compare", "Compare", + "correlation", "Correlation", "crosstab", "Crosstab", "shift", "Shift")).forEach((key, value) -> response.put(key, ref(value).withNullable(true))); + result.put("DataAnalysisSavedResultV1", new Schema().withType(Type.OBJECT).withProperties(response)); + Map entry = new LinkedHashMap<>(); + entry.put("id", new Schema().withType(Type.INTEGER)); + for(String name : List.of("name", "kind", "tableName", "tableLabel", "owner", "updated")) + { + entry.put(name, new Schema().withType(Type.STRING)); + } + entry.put("updated", new Schema().withType(Type.STRING).withFormat("date-time").withNullable(true)); + entry.put("shared", new Schema().withType(Type.BOOLEAN)); + entry.put("starred", new Schema().withType(Type.BOOLEAN)); + result.put("DataAnalysisSavedEntryV1", new Schema().withType(Type.OBJECT).withProperties(entry)); + result.put("DataAnalysisSavedListV1", new Schema().withType(Type.OBJECT).withProperties(new java.util.TreeMap<>(Map.of("analyses", new Schema().withType(Type.ARRAY).withItems(ref("SavedEntry")))))); + result.put("DataAnalysisStarV1", new Schema().withType(Type.OBJECT).withProperties(new java.util.TreeMap<>(Map.of("starred", new Schema().withType(Type.BOOLEAN))))); + result.put("DataAnalysisSqlV1", new Schema().withType(Type.OBJECT).withProperties(new java.util.TreeMap<>(Map.of("available", new Schema().withType(Type.BOOLEAN), + "sql", new Schema().withType(Type.STRING).withNullable(true), "note", new Schema().withType(Type.STRING))))); + result.put("DataAnalysisFileV1", new Schema().withType(Type.STRING).withFormat("binary")); + return (result); + } + + /******************************************************************************* + ** Reuse the extension's stable schema prefix. + *******************************************************************************/ + private static Schema ref(String name) + { + return (new Schema().withRef("#/components/schemas/DataAnalysis" + name + "V1")); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/app/DataAnalysisHome.java b/src/main/java/com/kingsrook/qbits/dataanalysis/app/DataAnalysisHome.java index e76f51f..c9012b8 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/app/DataAnalysisHome.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/app/DataAnalysisHome.java @@ -20,35 +20,35 @@ package com.kingsrook.qbits.dataanalysis.app; - import java.time.Instant; -import java.util.Arrays; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; -import java.util.Locale; -import java.util.stream.Collectors; +import java.util.Map; +import java.util.Objects; import com.kingsrook.qbits.dataanalysis.DataAnalysisQBitConfig; -import com.kingsrook.qbits.dataanalysis.engine.AnalysisFormatting; +import com.kingsrook.qbits.dataanalysis.DataAnalysisRuntime; +import com.kingsrook.qbits.dataanalysis.engine.DrillFilters; import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables; +import com.kingsrook.qbits.dataanalysis.saved.SavedAnalyses; import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType; import com.kingsrook.qqq.backend.core.actions.tables.CountAction; -import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; -import com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.MultiStatisticsData; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QWidgetData; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData; import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; -import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; @@ -58,35 +58,49 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; - /******************************************************************************* - ** Analyses app home: real counts under the current user's permissions and locks. + ** Standard QQQ statistics and narrow table widgets, all queried as the viewer. *******************************************************************************/ public class DataAnalysisHome extends AbstractWidgetRenderer { public static final String HOME_WIDGET = "dataAnalysisHome"; - - + public static final String STARRED = "dataAnalysisStarred"; + public static final String FIRING = "dataAnalysisFiring"; + public static final String RECENT = "dataAnalysisRecent"; /******************************************************************************* - ** The home itself has no data privilege; each section checks its table READ. + ** The existing home name now supplies standard multi-statistics data. *******************************************************************************/ public static QWidgetMetaData widget() { - return (new QWidgetMetaData().withName(HOME_WIDGET).withLabel("Overview").withType(WidgetType.HTML.getType()) - .withCodeReference(new QCodeReference(DataAnalysisHome.class)).withGridColumns(12).withIsCard(false) + return (widget(HOME_WIDGET, "Analysis overview", WidgetType.MULTI_STATISTICS)); + } + + /******************************************************************************* + ** Full-width tables have three or four columns and honest empty states. + *******************************************************************************/ + public static List widgets() + { + return (List.of(widget(), widget(STARRED, "Starred analyses", WidgetType.TABLE), widget(FIRING, "Firing watches", WidgetType.TABLE), widget(RECENT, "My recent runs", WidgetType.TABLE))); + } + + /******************************************************************************* + ** The widget itself grants no privilege; each data section checks table READ. + *******************************************************************************/ + private static QWidgetMetaData widget(String name, String label, WidgetType type) + { + return (new QWidgetMetaData().withName(name).withLabel(label).withType(type.getType()).withGridColumns(12).withIsCard(true) + .withShowReloadButton(true).withCodeReference(new QCodeReference(DataAnalysisHome.class)) .withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED))); } - - /******************************************************************************* - ** Only main tables appear in navigation; schedules and stars are record children. + ** Main tables remain navigation; schedules and stars remain child lists. *******************************************************************************/ public static QAppMetaData app(DataAnalysisQBitConfig config, List tables) { QAppMetaData app = new QAppMetaData().withName(config.getAppName()).withLabel(config.getAppLabel()) - .withIcon(new QIcon("insights")).withWidgets(List.of(HOME_WIDGET)) + .withIcon(new QIcon("insights")).withWidgets(widgets().stream().map(QWidgetMetaData::getName).toList()) .withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED)) .withSection(new QAppSection().withName("analyses").withLabel("Analyses").withIcon(new QIcon("insights")).withTables(DataAnalysisTables.NAV_TABLES)); app.setParentAppName(config.getParentAppName()); @@ -100,177 +114,172 @@ public class DataAnalysisHome extends AbstractWidgetRenderer /******************************************************************************* - ** RawHTML is carried as a JSON string by the HTML widget, not a text/html response. - ** Entities keep static separators independent of any host charset handling. + ** Dispatch data by registered metadata name, never by an arbitrary table input. *******************************************************************************/ @Override public RenderWidgetOutput render(RenderWidgetInput input) throws QException { - return (new RenderWidgetOutput(new RawHTML(null, render(Instant.now())))); + return (new RenderWidgetOutput(render(input.getWidgetMetaData().getName(), Instant.now()))); } - + /******************************************************************************* + ** Counts and table rows share core-enforced permissions and record locks. + *******************************************************************************/ + public static QWidgetData render(String name, Instant now) throws QException + { + if(HOME_WIDGET.equals(name)) + { + return (statistics(now)); + } + if(STARRED.equals(name)) + { + TableData data = table("Starred analyses", DataAnalysisTables.SAVED, "No starred analyses yet. Star a saved analysis to see it here.", "Name", "Table", "Kind"); + if(canRead(DataAnalysisTables.SAVED)) + { + for(var entry : DataAnalysisRuntime.saved().list().stream().filter(SavedAnalyses.Entry::starred).toList()) + { + data.getRows().add(values(link(DataAnalysisTables.SAVED, entry.id(), entry.name()), entry.tableLabel(), entry.kind())); + } + } + return (data); + } + String source = FIRING.equals(name) ? DataAnalysisTables.WATCH : DataAnalysisTables.RUN; + TableData data = FIRING.equals(name) + ? table("Firing watches", source, "No firing watches.", "Watch", "Table", "Last evaluated", "Message") + : table("My recent runs", source, "No analysis runs yet.", "Analysis", "Table", "Status", "Started"); + if(!canRead(source)) + { + return (data); + } + QQueryFilter filter = FIRING.equals(name) ? equal("state", "FIRING") + : equal("user_id", SavedAnalyses.user()).withCriteria("started_at", QCriteriaOperator.LESS_THAN_OR_EQUALS, now); + filter.withOrderBy(new QFilterOrderBy(FIRING.equals(name) ? "last_evaluated_at" : "started_at", false)).withLimit(10); + for(var row : SavedAnalyses.query(source, filter)) + { + String tableName = row.getValueString("table_name"); + var metadata = QContext.getQInstance().getTable(tableName); + String label = metadata == null ? Objects.toString(tableName, "Saved analysis") : metadata.getLabel(); + data.getRows().add(FIRING.equals(name) + ? values(link(source, row.getValue("id"), row.getValueString("detector")), label, row.getValue("last_evaluated_at"), row.getValue("last_message")) + : values(link(source, row.getValue("id"), row.getValueString("operation")), label, row.getValue("status"), row.getValue("started_at"))); + } + return (data); + } /******************************************************************************* - ** Counts are evaluated at one instant; no system session or synthetic numbers. + ** One standard group per subject, with real counts or an unavailable explanation. *******************************************************************************/ - public static String render(Instant now) throws QException + public static MultiStatisticsData statistics(Instant now) throws QException { - StringBuilder html = new StringBuilder("
    "); - String userId = QContext.getQSession() == null || QContext.getQSession().getUser() == null ? null : QContext.getQSession().getUser().getIdReference(); - html.append("

    Saved analyses

    "); - if(userId == null) + List groups = new ArrayList<>(); + var saved = group("Saved analyses", DataAnalysisTables.SAVED); + if(canRead(DataAnalysisTables.SAVED)) { - html.append("

    Sign in to see your saved analyses and stars.

    "); + List entries = DataAnalysisRuntime.saved().list(); + String user = SavedAnalyses.user(); + saved.withStatisticList(List.of(stat("Mine", (int) entries.stream().filter(e -> e.owner().equals(user)).count(), DataAnalysisTables.SAVED), + stat("Shared", (int) entries.stream().filter(e -> Boolean.TRUE.equals(e.shared())).count(), DataAnalysisTables.SAVED), + stat("Starred", (int) entries.stream().filter(SavedAnalyses.Entry::starred).count(), DataAnalysisTables.SAVED))); } - else if(canRead(DataAnalysisTables.SAVED)) - { - int mine = count(DataAnalysisTables.SAVED, equal("owner_user_id", userId)); - int shared = count(DataAnalysisTables.SAVED, equal("shared", true)); - html.append("

    Mine: ").append(AnalysisFormatting.number(mine, 0)).append(" · Shared (visible to you): ").append(AnalysisFormatting.number(shared, 0)).append("

    "); - if(canRead(DataAnalysisTables.SAVED_STAR)) - { - CountInput stars = new CountInput(DataAnalysisTables.SAVED).withIncludeDistinctCount(true) - .withQueryJoin(new QueryJoin(QContext.getQInstance().getJoin(DataAnalysisTables.joinName(DataAnalysisTables.SAVED_STAR))).withType(QueryJoin.Type.INNER)) - .withFilter(equal(DataAnalysisTables.SAVED_STAR + ".user_id", userId)); - stars.setPermissionEnforcement(PermissionEnforcement.ENFORCE); - int starred = new CountAction().execute(stars).getDistinctCount(); - html.append("

    Starred by me: ").append(AnalysisFormatting.number(starred, 0)).append("

    "); - } - if(mine == 0 && shared == 0) - { - html.append("

    No saved analyses yet. Open any table and choose Analyze on a column header to explore its data.

    "); - } - link(html, DataAnalysisTables.SAVED); - } - else - { - html.append("

    Saved analyses are unavailable: you need data_analysis.saved.read.

    "); - } - html.append("

    Watches

    "); + groups.add(saved); + var watches = group("Watches", DataAnalysisTables.WATCH); if(canRead(DataAnalysisTables.WATCH)) { - int total = count(DataAnalysisTables.WATCH, new QQueryFilter()); - int enabled = count(DataAnalysisTables.WATCH, equal("enabled", true)); - int firing = count(DataAnalysisTables.WATCH, equal("state", "FIRING")); - html.append("

    Total: ").append(AnalysisFormatting.number(total, 0)).append(" · Enabled: ").append(AnalysisFormatting.number(enabled, 0)).append(" · Firing: ").append(AnalysisFormatting.number(firing, 0)).append("

    "); - if(total == 0) - { - html.append("

    No watches yet.

    "); - } - QueryInput watches = new QueryInput(DataAnalysisTables.WATCH).withShouldFetchHeavyFields(true) - .withFilter(new QQueryFilter().withOrderBy(new QFilterOrderBy("state", true)).withOrderBy(new QFilterOrderBy("id", false)).withLimit(5)); - watches.setPermissionEnforcement(PermissionEnforcement.ENFORCE); - List recent = new QueryAction().execute(watches).getRecords(); - if(!recent.isEmpty()) - { - html.append(""); - } - link(html, DataAnalysisTables.WATCH); + watches.withStatisticList(List.of(stat("Enabled", count(DataAnalysisTables.WATCH, equal("enabled", true)), DataAnalysisTables.WATCH), + stat("Firing", count(DataAnalysisTables.WATCH, equal("state", "FIRING")), DataAnalysisTables.WATCH))); } - else + groups.add(watches); + var runs = group("My runs", DataAnalysisTables.RUN); + if(canRead(DataAnalysisTables.RUN)) { - html.append("

    Watches are unavailable: you need data_analysis.watch.read.

    "); + runs.withStatisticList(List.of(stat("Last 7 days", count(DataAnalysisTables.RUN, equal("user_id", SavedAnalyses.user()) + .withCriteria("started_at", QCriteriaOperator.GREATER_THAN_OR_EQUALS, now.minus(7, ChronoUnit.DAYS)) + .withCriteria("started_at", QCriteriaOperator.LESS_THAN_OR_EQUALS, now)), DataAnalysisTables.RUN))); } - html.append("

    Recent analysis runs

    "); - if(userId == null) - { - html.append("

    Sign in to see your recent analysis runs.

    "); - } - else if(canRead(DataAnalysisTables.RUN)) - { - QueryInput query = new QueryInput(DataAnalysisTables.RUN).withFilter(equal("user_id", userId) - .withCriteria(new QFilterCriteria("started_at", QCriteriaOperator.LESS_THAN_OR_EQUALS, now)) - .withOrderBy(new QFilterOrderBy("started_at", false)).withOrderBy(new QFilterOrderBy("id", false)).withLimit(5)); - query.setPermissionEnforcement(PermissionEnforcement.ENFORCE); - List runs = new QueryAction().execute(query).getRecords(); - if(runs.isEmpty()) - { - html.append("

    No analysis runs yet.

    "); - } - else - { - html.append("
      "); - for(QRecord run : runs) - { - QTableMetaData table = QContext.getQInstance().getTable(run.getValueString("table_name")); - String tableLabel = table == null ? run.getValueString("table_name") : table.getLabel(); - String path = QContext.getQInstance().getTablePath(DataAnalysisTables.RUN) + "/" + run.getValueInteger("id"); - html.append("
    • ").append(escape(tableLabel)).append(" · ") - .append(escape(words(run.getValueString("operation")))).append("
      Status
      ") - .append(escape(words(run.getValueString("status")))).append("
      Finished
      ") - .append(run.getValue("finished_at") == null ? "Not finished" : escape(AnalysisFormatting.instant(run.getValueInstant("finished_at")))) - .append("
      • "); - } - html.append("
    "); - } - link(html, DataAnalysisTables.RUN); - } - else - { - html.append("

    Run history is unavailable: you need data_analysis.run.read.

    "); - } - return (html.append("
    ").toString()); + groups.add(runs); + return (new MultiStatisticsData("Analyses", groups)); } - - /******************************************************************************* - ** Explicit permission checks avoid presenting denied sections as empty data. + ** Empty/denied data are different messages, not synthetic zeroes. *******************************************************************************/ - private static boolean canRead(String table) + private static MultiStatisticsData.StatisticsGroupData group(String label, String table) { - return (PermissionsHelper.hasTablePermission(new CountInput(table), table, TablePermissionSubType.READ)); + return (new MultiStatisticsData.StatisticsGroupData().withHeader(label).withStatisticList(List.of()) + .withSubheader(canRead(table) ? "Visible to you" : "Unavailable: READ permission required.")); } - - /******************************************************************************* - ** Equality filter for one count. + ** Link every count to its actual table path. *******************************************************************************/ - private static QQueryFilter equal(String field, java.io.Serializable value) + private static MultiStatisticsData.StatisticsGroupData.Statistic stat(String label, int count, String table) throws QException { - return (new QQueryFilter().withCriteria(new QFilterCriteria(field, QCriteriaOperator.EQUALS, value))); + return (new MultiStatisticsData.StatisticsGroupData.Statistic(label, count, QContext.getQInstance().getTablePath(table))); } - - /******************************************************************************* - ** Stored operation and status keys are displayed as ordinary words. + ** Standard table cells support escaped record links; other columns are plain text. *******************************************************************************/ - private static String words(String value) + private static TableData table(String label, String table, String empty, String... headers) throws QException { - return (value == null ? "Unknown" : Arrays.stream(value.toLowerCase(Locale.ROOT).split("_")) - .filter(part -> !part.isEmpty()).map(part -> Character.toUpperCase(part.charAt(0)) + part.substring(1)).collect(Collectors.joining(" "))); + List columns = new ArrayList<>(); + for(int i = 0; i < headers.length; i++) + { + columns.add(new TableData.Column(i == 0 ? "html" : "text", headers[i], "c" + i, null, "left")); + } + return (new TableData(label, columns, new ArrayList<>()).withNoRowsFoundHTML(canRead(table) ? empty : "Unavailable: READ permission required.") + .withLinkText("View all").withLinkURL(QContext.getQInstance().getTablePath(table)).withRowsPerPage(10)); } - + /******************************************************************************* + ** Preserve real values while avoiding null map entries and executable HTML. + *******************************************************************************/ + private static Map values(Object... values) + { + Map row = new LinkedHashMap<>(); + for(int i = 0; i < values.length; i++) + { + row.put("c" + i, Objects.toString(values[i], "")); + } + return (row); + } /******************************************************************************* - ** Metadata and persisted values must never become executable widget markup. + ** Standard table HTML cell contains only an escaped link and escaped text. + *******************************************************************************/ + private static String link(String table, Object id, String label) throws QException + { + return ("" + escape(label) + ""); + } + + /******************************************************************************* + ** Metadata labels and stored names are never trusted markup. *******************************************************************************/ private static String escape(String value) { - return (value == null ? "Unknown" : value.replace("&", "&").replace("\"", """).replace("<", "<").replace(">", ">").replace("'", "'")); + return (Objects.toString(value, "").replace("&", "&").replace("<", "<").replace(">", ">").replace("\"", """).replace("'", "'")); } - + /******************************************************************************* + ** Explicit checks distinguish denied sections from empty sections. + *******************************************************************************/ + private static boolean canRead(String table) + { + CountInput input = new CountInput(table); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + return (PermissionsHelper.hasTablePermission(input, table, TablePermissionSubType.READ)); + } /******************************************************************************* - ** CountAction enforces table permission and record locks using QContext's session. + ** Simple filter; record security is applied exclusively by core. + *******************************************************************************/ + private static QQueryFilter equal(String field, java.io.Serializable value) + { + return (DrillFilters.condition(field, QCriteriaOperator.EQUALS, value)); + } + + /******************************************************************************* + ** Counts use the viewer's session and explicitly enforce permissions. *******************************************************************************/ private static int count(String table, QQueryFilter filter) throws QException { @@ -278,16 +287,4 @@ public class DataAnalysisHome extends AbstractWidgetRenderer input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); return (new CountAction().execute(input).getCount()); } - - - - /******************************************************************************* - ** Use the instance's actual app path, including custom names and parent apps. - *******************************************************************************/ - private static void link(StringBuilder html, String table) throws QException - { - String path = QContext.getQInstance().getTablePath(table); - html.append("View all"); - } } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/app/SavedAnalysisWidget.java b/src/main/java/com/kingsrook/qbits/dataanalysis/app/SavedAnalysisWidget.java new file mode 100644 index 0000000..aad5441 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/app/SavedAnalysisWidget.java @@ -0,0 +1,157 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.app; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import com.kingsrook.qbits.dataanalysis.DataAnalysisRuntime; +import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables; +import com.kingsrook.qbits.dataanalysis.result.Chart; +import com.kingsrook.qbits.dataanalysis.result.Metric; +import com.kingsrook.qbits.dataanalysis.saved.AnalysisExport; +import com.kingsrook.qbits.dataanalysis.watch.WatchRecords; +import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChartData; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChartSubheaderData; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; + +/******************************************************************************* + ** Saved analysis rendered through Next's existing standard chart widget. + *******************************************************************************/ +public class SavedAnalysisWidget extends AbstractWidgetRenderer +{ + public static final String NAME = "DATA_ANALYSIS_SAVED"; + + /******************************************************************************* + ** Hosts clone this metadata and set savedId to place an analysis on a dashboard. + *******************************************************************************/ + public static QWidgetMetaData widget() + { + return (new QWidgetMetaData().withName(NAME).withLabel("Saved analysis").withType(WidgetType.BAR_CHART.getType()) + .withGridColumns(12).withIsCard(true).withCodeReference(new QCodeReference(SavedAnalysisWidget.class)) + .withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED))); + } + + /******************************************************************************* + ** Values come from a fresh viewer run; no stored owner result is ever reused. + *******************************************************************************/ + @Override + public RenderWidgetOutput render(RenderWidgetInput input) throws QException + { + var defaults = ((QWidgetMetaData) input.getWidgetMetaData()).getDefaultValues(); + Object value = defaults == null ? null : defaults.get("savedId"); + if(value == null) + { + return (new RenderWidgetOutput(new ChartData("Saved analysis", "Choose a saved analysis for this widget.", "Analysis", List.of(), List.of()))); + } + Integer id; + try + { + id = Integer.valueOf(value.toString()); + } + catch(NumberFormatException e) + { + throw new QBadRequestException("savedId must be an integer."); + } + var saved = WatchRecords.get(DataAnalysisTables.SAVED, id); + var result = DataAnalysisRuntime.saved().run(id); + List metrics = AnalysisExport.headlines(result); + Chart chart = null; + if(result.columnProfile() != null) + { + chart = result.columnProfile().getChart(); + } + else if(result.tableProfile() != null) + { + chart = result.tableProfile().getHeatmap(); + } + else if(result.correlation() != null) + { + chart = result.correlation().getChart(); + } + else if(result.shift() != null) + { + chart = result.shift().getChart(); + } + else if(result.compare() != null && !result.compare().getFields().isEmpty()) + { + chart = result.compare().getFields().getFirst().getChart(); + } + String description = metrics.stream().filter(Objects::nonNull).map(m -> m.getLabel() + ": " + AnalysisExport.value(m)) + .collect(java.util.stream.Collectors.joining("; ")); + if(!result.warnings().isEmpty()) + { + description += " " + String.join("; ", result.warnings()); + } + ChartData data = new ChartData().withTitle(saved.getValueString("name")).withDescription(description).withHeight(280); + String path = QContext.getQInstance().getTablePath(DataAnalysisTables.SAVED) + "/" + id; + Number mainNumber = metrics.stream().filter(m -> !Boolean.TRUE.equals(m.getSuppressed()) && m.getValue() instanceof Number) + .map(m -> (Number) m.getValue()).findFirst().orElse(null); + data.withChartSubheaderData(new ChartSubheaderData().withMainNumber(mainNumber).withMainNumberUrl(path)); + List labels = new ArrayList<>(); + List datasets = new ArrayList<>(); + if(chart != null && !chart.getSeries().isEmpty()) + { + chart.getSeries().getFirst().getPoints().forEach(p -> labels.add(Objects.toString(p.getLabel(), Objects.toString(p.getX(), "")))); + for(var series : chart.getSeries()) + { + datasets.add(new ChartData.Data.Dataset().withLabel(series.getName()) + .withData(series.getPoints().stream().map(p -> (Number) p.getY()).toList())); + } + } + else if(result.pivot() != null) + { + List numbers = new ArrayList<>(); + for(var cell : result.pivot().getCells()) + { + labels.add(java.util.stream.Stream.concat(result.pivot().getRowHeaders().get(cell.getRowIndex()).stream(), + result.pivot().getColumnHeaders().get(cell.getColumnIndex()).stream()).map(p -> p.getLabel()).collect(java.util.stream.Collectors.joining(" / "))); + Metric metric = cell.getValues().getFirst(); + numbers.add(!Boolean.TRUE.equals(metric.getSuppressed()) && metric.getValue() instanceof Number n ? n : null); + } + datasets.add(new ChartData.Data.Dataset().withLabel(result.pivot().getMeasures().getFirst().getLabel()).withData(numbers)); + } + else if(result.crosstab() != null) + { + List numbers = new ArrayList<>(); + for(var cell : result.crosstab().getCells()) + { + labels.add(result.crosstab().getRows().get(cell.getRowIndex()).getLabel() + " / " + result.crosstab().getColumns().get(cell.getColumnIndex()).getLabel()); + Metric metric = cell.getCount(); + numbers.add(!Boolean.TRUE.equals(metric.getSuppressed()) && metric.getValue() instanceof Number n ? n : null); + } + datasets.add(new ChartData.Data.Dataset().withLabel("Count").withData(numbers)); + } + data.withChartData(new ChartData.Data().withLabels(labels).withUrls(labels.stream().map(label -> path).toList()).withDatasets(datasets)); + data.withFooterHTML("View saved analysis"); + return (new RenderWidgetOutput(data)); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/PivotExplorer.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/PivotExplorer.java index b436fc2..46245a0 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/PivotExplorer.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/PivotExplorer.java @@ -41,7 +41,6 @@ import com.kingsrook.qbits.dataanalysis.result.PivotRequest; import com.kingsrook.qbits.dataanalysis.result.PivotResult; import com.kingsrook.qbits.dataanalysis.result.ValueType; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; -import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -617,7 +616,7 @@ public class PivotExplorer groups.forEach(input::withGroupBy); new LinkedHashSet<>(aggregates).forEach(input::withAggregate); input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); - var output = new AggregateAction().execute(input); + var output = com.kingsrook.qbits.dataanalysis.saved.SqlHandoff.aggregate(input); partial |= output.getTruncated(); return (limit == null ? output.getResults() : output.getResults().stream().sorted(java.util.Comparator.comparingLong(this::count).reversed() .thenComparing(r -> String.valueOf(r.getGroupByValues().values()))).limit(limit).toList()); diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/RelationshipSupport.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/RelationshipSupport.java index aad7842..f7f2162 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/RelationshipSupport.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/RelationshipSupport.java @@ -33,7 +33,6 @@ import com.kingsrook.qbits.dataanalysis.result.Significance; import com.kingsrook.qbits.dataanalysis.result.ValueType; import com.kingsrook.qbits.dataanalysis.stats.Statistics; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; -import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction; import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate; @@ -79,7 +78,7 @@ public class RelationshipSupport extends StandInAnalysisBackend { input.withFilter(DrillFilters.restoreTypes(table, input.getFilter())).withAllowStreamedFallback(true).withStreamedRowLimit(config.getMaxRows()); input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); - var output = new AggregateAction().execute(input); + var output = com.kingsrook.qbits.dataanalysis.saved.SqlHandoff.aggregate(input); if(Boolean.TRUE.equals(output.getTruncated())) { throw new QBadRequestException("Analysis exceeds the streamed row limit; narrow the filter."); diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/StandInAnalysisBackend.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/StandInAnalysisBackend.java index 48b342f..14fefea 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/StandInAnalysisBackend.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/StandInAnalysisBackend.java @@ -38,7 +38,6 @@ import java.util.Objects; import java.util.Set; import java.util.function.Function; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; -import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction; import com.kingsrook.qqq.backend.core.actions.tables.CountAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.context.QContext; @@ -171,7 +170,7 @@ public class StandInAnalysisBackend implements AnalysisBackend restored.setOrderBys(input.getFilter().getOrderBys()); input.setFilter(restored); input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); - return (new AggregateAction().execute(input).getResults()); + return (com.kingsrook.qbits.dataanalysis.saved.SqlHandoff.aggregate(input).getResults()); } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/metadata/DataAnalysisMetaDataProducer.java b/src/main/java/com/kingsrook/qbits/dataanalysis/metadata/DataAnalysisMetaDataProducer.java index ad08846..6121fb7 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/metadata/DataAnalysisMetaDataProducer.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/metadata/DataAnalysisMetaDataProducer.java @@ -66,7 +66,8 @@ public class DataAnalysisMetaDataProducer extends QBitComponentMetaDataProducer< .filter(qbit -> DataAnalysisQBitProducer.ARTIFACT_ID.equals(qbit.getArtifactId())).findFirst().orElseThrow().getName()); output.add(table); } - output.add(DataAnalysisHome.widget()); + DataAnalysisHome.widgets().forEach(output::add); + output.add(com.kingsrook.qbits.dataanalysis.app.SavedAnalysisWidget.widget()); output.add(DataAnalysisHome.app(getQBitConfig(), tables)); return (output); } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/metadata/ScheduleProcesses.java b/src/main/java/com/kingsrook/qbits/dataanalysis/metadata/ScheduleProcesses.java new file mode 100644 index 0000000..fc1f6ae --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/metadata/ScheduleProcesses.java @@ -0,0 +1,93 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.metadata; + +import java.util.List; +import com.kingsrook.qbits.dataanalysis.DataAnalysisQBitConfig; +import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables; +import com.kingsrook.qbits.dataanalysis.processes.ScheduleActions; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; + +/******************************************************************************* + ** Watch record actions and one hidden scheduled process, never app children. + *******************************************************************************/ +public class ScheduleProcesses extends QBitComponentMetaDataProducer +{ + /******************************************************************************* + ** Build after the watch table has been registered. + *******************************************************************************/ + @Override + public int getSortOrder() + { + return (200); + } + + /******************************************************************************* + ** Component scanning also encounters unconfigured producers. + *******************************************************************************/ + @Override + public boolean isEnabled() + { + return (getQBitConfig() != null); + } + + /******************************************************************************* + ** Ordinary core actions provide selected records, Review, and Result screens. + *******************************************************************************/ + @Override + public MetaDataProducerMultiOutput produce(QInstance instance) + { + MetaDataProducerMultiOutput output = new MetaDataProducerMultiOutput(); + for(String name : List.of(ScheduleActions.SEND)) + { + String label = "Send now"; + QProcessMetaData process = StreamedETLWithFrontendProcess.processMetaDataBuilder() + .withName(name).withLabel(label).withIcon(new QIcon("notifications_active")) + .withTableName(DataAnalysisTables.SCHEDULE).withSourceTable(DataAnalysisTables.SCHEDULE).withDestinationTable(DataAnalysisTables.SCHEDULE) + .withExtractStepClass(ExtractViaQueryStep.class).withTransformStepClass(ScheduleActions.Review.class).withLoadStepClass(ScheduleActions.Result.class) + .withSupportsFullValidation(true).withDoFullValidation(true).withMinInputRecords(1) + .withPreviewMessage("Review the selected schedules.").getProcessMetaData(); + // Table READ/EDIT and owner locks are enforced by extraction and both steps. + process.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED)); + output.add(process); + } + QProcessMetaData scheduled = new QProcessMetaData().withName(ScheduleActions.SCHEDULED).withLabel("Run analysis schedules").withIsHidden(true) + .withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION).withPermissionBaseName(ScheduleActions.SCHEDULED)) + .withStep(new QBackendStepMetaData().withName("run").withCode(new QCodeReference(ScheduleActions.Scheduled.class))); + if(getQBitConfig().getSchedulerName() != null) + { + scheduled.withSchedule(new QScheduleMetaData().withSchedulerName(getQBitConfig().getSchedulerName()).withRepeatSeconds(300).withInitialDelaySeconds(300)); + } + output.add(scheduled); + return (output); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/model/DataAnalysisTables.java b/src/main/java/com/kingsrook/qbits/dataanalysis/model/DataAnalysisTables.java index 1e77895..34b9327 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/model/DataAnalysisTables.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/model/DataAnalysisTables.java @@ -239,8 +239,15 @@ public final class DataAnalysisTables .withField(field("chart_kind", QFieldType.STRING).withPossibleValueSourceName("data_analysis_chart_kind")) .withField(field("owner_user_id", QFieldType.STRING).withIsRequired(true)) .withField(field("shared", QFieldType.BOOLEAN).withDefaultValue(false)) + .withField(field("read_owner_user_id", QFieldType.STRING).withIsEditable(false).withIsHidden(true)) .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD)) - .withRecordSecurityLock(ownerLock("owner_user_id"))); + .withRecordSecurityLock(ownerLock("owner_user_id").withLockScope(RecordSecurityLock.LockScope.WRITE)) + .withRecordSecurityLock(ownerLock("read_owner_user_id").withLockScope(RecordSecurityLock.LockScope.READ) + .withNullValueBehavior(RecordSecurityLock.NullValueBehavior.ALLOW)) + .withCustomizer(com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers.PRE_INSERT_RECORD, + new QCodeReference(com.kingsrook.qbits.dataanalysis.saved.SavedCustomizer.class)) + .withCustomizer(com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers.PRE_UPDATE_RECORD, + new QCodeReference(com.kingsrook.qbits.dataanalysis.saved.SavedCustomizer.class))); tables.add(base(config, SAVED_STAR, "Stars", "star", "data_analysis.saved") .withRecordLabelFormatAndFields("%s", "user_id") .withField(field("saved_id", QFieldType.INTEGER).withPossibleValueSourceName("data_analysis_saved").withIsRequired(true)) @@ -259,9 +266,13 @@ public final class DataAnalysisTables .withField(field("recipients", QFieldType.TEXT)) .withField(field("enabled", QFieldType.BOOLEAN).withDefaultValue(false)) .withField(field("last_run_at", QFieldType.DATE_TIME)) - .withField(field("last_status", QFieldType.STRING).withPossibleValueSourceName("data_analysis_run_status")) + .withField(field("last_status", QFieldType.TEXT).withIsEditable(false)) .withField(field("next_run_at", QFieldType.DATE_TIME)) - .withRecordSecurityLock(ownerLock(SAVED + ".owner_user_id").withJoinNameChain(List.of(joinName(SCHEDULE))))); + .withRecordSecurityLock(ownerLock(SAVED + ".owner_user_id").withJoinNameChain(List.of(joinName(SCHEDULE)))) + .withCustomizer(com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers.PRE_INSERT_RECORD, + new QCodeReference(com.kingsrook.qbits.dataanalysis.schedule.ScheduleCustomizer.class)) + .withCustomizer(com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers.PRE_UPDATE_RECORD, + new QCodeReference(com.kingsrook.qbits.dataanalysis.schedule.ScheduleCustomizer.class))); tables.add(base(config, WATCH, "Watches", "notifications_active", "data_analysis.watch") .withRecordLabelFormatAndFields("%s", "detector") .withField(field("saved_id", QFieldType.INTEGER).withPossibleValueSourceName("data_analysis_saved") @@ -305,7 +316,7 @@ public final class DataAnalysisTables for(QTableMetaData table : tables) { List fields = new ArrayList<>(table.getFields().keySet()); - fields.removeAll(List.of("id", "createDate", "modifyDate")); + fields.removeAll(List.of("id", "createDate", "modifyDate", "read_owner_user_id")); int headlineSize = Math.min(4, fields.size()); table.withSection(SectionFactory.defaultT1(fields.subList(0, headlineSize).toArray(String[]::new))); if(fields.size() > headlineSize) diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/processes/ScheduleActions.java b/src/main/java/com/kingsrook/qbits/dataanalysis/processes/ScheduleActions.java new file mode 100644 index 0000000..d5e56a3 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/processes/ScheduleActions.java @@ -0,0 +1,144 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.processes; + +import java.time.Instant; +import java.util.ArrayList; +import com.kingsrook.qbits.dataanalysis.DataAnalysisRuntime; +import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; +import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface; + +/******************************************************************************* + ** Standard streamed ETL Review and Result actions for scheduled analyses. + *******************************************************************************/ +public final class ScheduleActions +{ + public static final String SEND = "data_analysis_schedule_send"; + public static final String SCHEDULED = "data_analysis_run_schedules"; + + /******************************************************************************* + ** Steps only. + *******************************************************************************/ + private ScheduleActions() + { + } + + /******************************************************************************* + ** Review has no delivery side effects. + *******************************************************************************/ + public static class Review extends AbstractTransformStep implements ProcessSummaryProviderInterface + { + private final ArrayList lines = new ArrayList<>(); + + /******************************************************************************* + ** Core extraction enforces record locks; verify the edit grant as well. + *******************************************************************************/ + @Override + public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + if(!PermissionsHelper.hasTablePermission(new UpdateInput(DataAnalysisTables.SCHEDULE), DataAnalysisTables.SCHEDULE, TablePermissionSubType.EDIT)) + { + throw new QPermissionDeniedException("Schedule editing is unavailable."); + } + for(QRecord record : input.getRecords()) + { + lines.add(new ProcessSummaryLine(Status.INFO, null, "Schedule " + record.getValueString("id") + " will be sent to authorized recipients.")); + output.addRecord(record); + } + } + + /******************************************************************************* + ** Standard Review summary. + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(RunBackendStepOutput output, boolean result) + { + return (result ? new ArrayList<>() : lines); + } + } + + /******************************************************************************* + ** Every selected record contributes its actual delivery status to Result. + *******************************************************************************/ + public static class Result extends AbstractLoadStep implements ProcessSummaryProviderInterface + { + private final ArrayList lines = new ArrayList<>(); + + /******************************************************************************* + ** Reauthorize when executing after Review. + *******************************************************************************/ + @Override + public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + for(QRecord record : input.getRecords()) + { + try + { + String status = DataAnalysisRuntime.schedules().sendNow(record.getValue("id"), Instant.now()); + lines.add(new ProcessSummaryLine(status.startsWith("Failed") ? Status.ERROR : Status.OK, null, status)); + output.addRecord(record); + } + catch(QException e) + { + lines.add(new ProcessSummaryLine(Status.ERROR, null, e.getMessage())); + } + } + } + + /******************************************************************************* + ** Actual results across streamed pages. + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(RunBackendStepOutput output, boolean result) + { + return (lines); + } + } + + /******************************************************************************* + ** Hidden scheduler every five minutes. + *******************************************************************************/ + public static class Scheduled implements BackendStep + { + /******************************************************************************* + ** Count actual due schedule attempts. + *******************************************************************************/ + @Override + public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + output.addValue("count", DataAnalysisRuntime.schedules().runDue(Instant.now())); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/saved/AnalysisExport.java b/src/main/java/com/kingsrook/qbits/dataanalysis/saved/AnalysisExport.java new file mode 100644 index 0000000..62dddec --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/saved/AnalysisExport.java @@ -0,0 +1,304 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.saved; + +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import com.kingsrook.qbits.dataanalysis.result.Metric; +import com.kingsrook.qbits.dataanalysis.result.PivotHeader; +import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType; + +/******************************************************************************* + ** Export already-authorized typed results using core CSV and XLSX streamers. + *******************************************************************************/ +public final class AnalysisExport +{ + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private AnalysisExport() + { + } + + /******************************************************************************* + ** A result table, also reused by scheduled HTML summaries. + *******************************************************************************/ + public record ResultTable(List columns, List> rows) + { + } + + /******************************************************************************* + ** Download bytes and metadata; filenames are fixed rather than user supplied. + *******************************************************************************/ + public record File(byte[] bytes, String mimeType, String filename) + { + } + + /******************************************************************************* + ** Accept only the two documented export formats. + *******************************************************************************/ + public static ReportFormat format(String value) throws QBadRequestException + { + if("csv".equalsIgnoreCase(value)) + { + return (ReportFormat.CSV); + } + if("xlsx".equalsIgnoreCase(value)) + { + return (ReportFormat.XLSX); + } + throw new QBadRequestException("format must be csv or xlsx."); + } + + /******************************************************************************* + ** Core's report streamers accept computed records without a source-table query. + *******************************************************************************/ + public static File file(SavedResult result, String format) throws QException + { + ReportFormat type = format(format); + ResultTable table = table(result); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + var destination = new ReportDestination().withReportFormat(type).withReportOutputStream(bytes); + var view = new QReportView().withName("analysis").withLabel("Analysis").withType(ReportType.TABLE); + List fields = new ArrayList<>(); + for(int i = 0; i < table.columns().size(); i++) + { + fields.add(new QFieldMetaData("c" + i, QFieldType.STRING).withLabel(table.columns().get(i))); + view.withColumn(new QReportField("c" + i).withLabel(table.columns().get(i)).withType(QFieldType.STRING)); + } + var streamer = type.newReportStreamer(); + streamer.preRun(destination, List.of(view)); + streamer.start(new ExportInput().withReportDestination(destination), fields, "Analysis", view); + List records = new ArrayList<>(); + for(List row : table.rows()) + { + QRecord record = new QRecord(); + for(int i = 0; i < row.size(); i++) + { + record.withValue("c" + i, row.get(i)); + } + records.add(record); + } + for(String warning : result.warnings()) + { + records.add(new QRecord().withValue("c0", "Warning: " + warning)); + } + streamer.addRecords(records); + streamer.finish(); + return (new File(bytes.toByteArray(), type.getMimeType(), "analysis." + type.getExtension())); + } + + /******************************************************************************* + ** Preserve formatted values and suppressed cells; never export underlying raw data. + *******************************************************************************/ + public static ResultTable table(SavedResult result) + { + List columns = new ArrayList<>(List.of("Section", "Metric", "Value")); + List> rows = new ArrayList<>(); + if(result.columnProfile() != null) + { + var p = result.columnProfile(); + metrics(rows, "Counts", p.getCounts()); + if(p.getNumeric() != null) + { + metrics(rows, "Numeric", p.getNumeric().getMetrics()); + } + if(p.getText() != null) + { + metrics(rows, "Text", p.getText().getMetrics()); + } + if(p.getDates() != null) + { + metrics(rows, "Dates", p.getDates().getMetrics()); + } + if(p.getBooleans() != null) + { + metrics(rows, "Boolean", p.getBooleans()); + } + p.getTopValues().forEach(v -> rows.add(row("Top values", v.getLabel(), v.getCount()))); + } + else if(result.tableProfile() != null) + { + var p = result.tableProfile(); + metrics(rows, "Table", List.of(p.getRowCount())); + for(var field : p.getFields()) + { + rows.add(row(field.getLabel(), "Completeness", field.getCompleteness())); + rows.add(row(field.getLabel(), "Distinct", field.getDistinct())); + rows.add(row(field.getLabel(), "Uniqueness", field.getUniqueness())); + } + } + else if(result.pivot() != null) + { + var p = result.pivot(); + columns = new ArrayList<>(List.of("Row")); + for(var headers : p.getColumnHeaders()) + { + for(var measure : p.getMeasures()) + { + columns.add(label(headers) + " / " + measure.getLabel()); + } + } + for(int r = 0; r < p.getRowHeaders().size(); r++) + { + List row = new ArrayList<>(); + row.add(label(p.getRowHeaders().get(r))); + for(int c = 0; c < p.getColumnHeaders().size(); c++) + { + final int ri = r; + final int ci = c; + var cell = p.getCells().stream().filter(x -> x.getRowIndex() == ri && x.getColumnIndex() == ci).findFirst(); + for(int m = 0; m < p.getMeasures().size(); m++) + { + row.add(cell.isPresent() ? value(cell.get().getValues().get(m)) : ""); + } + } + rows.add(row); + } + } + else if(result.compare() != null) + { + columns = List.of("Field", "Value", "Metric", "A", "B", "Delta", "Percent delta"); + for(var field : result.compare().getFields()) + { + for(var comparison : field.getValues()) + { + for(var metric : comparison.getSideA().getMetrics()) + { + var b = comparison.getSideB().getMetrics().stream().filter(m -> Objects.equals(m.getKey(), metric.getKey())).findFirst().orElse(null); + rows.add(row(field.getField(), comparison.getLabel(), metric.getLabel(), value(metric), value(b), value(comparison.getDelta()), value(comparison.getPercentDelta()))); + } + } + } + } + else if(result.correlation() != null) + { + columns = List.of("X", "Y", "Paired rows", "Correlation"); + result.correlation().getCells().forEach(c -> rows.add(row(c.getXField(), c.getYField(), c.getN(), c.getR()))); + } + else if(result.crosstab() != null) + { + columns = List.of("Row", "Column", "Count", "Row share", "Column share"); + var p = result.crosstab(); + p.getCells().forEach(c -> rows.add(row(p.getRows().get(c.getRowIndex()).getLabel(), p.getColumns().get(c.getColumnIndex()).getLabel(), + value(c.getCount()), value(c.getRowShare()), value(c.getColumnShare())))); + } + else if(result.shift() != null) + { + for(var bucket : result.shift().getBuckets()) + { + metrics(rows, bucket.getLabel(), bucket.getMetrics()); + } + if(result.shift().getPsi() != null) + { + metrics(rows, "Shift", List.of(result.shift().getPsi())); + } + } + return (new ResultTable(columns, rows)); + } + + /******************************************************************************* + ** Headline metrics shared by standard chart widgets and email summaries. + *******************************************************************************/ + public static List headlines(SavedResult result) + { + List metrics = new ArrayList<>(); + if(result.columnProfile() != null) + { + metrics.addAll(result.columnProfile().getCounts()); + if(result.columnProfile().getNumeric() != null) + { + metrics.addAll(result.columnProfile().getNumeric().getMetrics().stream() + .filter(m -> List.of("mean", "median", "min", "max").contains(m.getKey())).toList()); + } + } + else if(result.tableProfile() != null) + { + metrics.add(result.tableProfile().getRowCount()); + } + else if(result.pivot() != null) + { + metrics.addAll(result.pivot().getGrandTotals()); + } + else if(result.compare() != null) + { + result.compare().getFields().stream().limit(4).forEach(f -> f.getValues().stream().limit(1).forEach(v -> metrics.add(v.getDelta()))); + } + else if(result.shift() != null) + { + metrics.add(result.shift().getPsi()); + } + else + { + int count = result.correlation() != null ? result.correlation().getFields().size() : result.crosstab() != null ? result.crosstab().getCells().size() : 0; + if(result.correlation() != null || result.crosstab() != null) + { + metrics.add(new Metric().withKey("visible").withLabel(result.correlation() != null ? "Numeric fields" : "Visible combinations") + .withValue(count).withValueType(com.kingsrook.qbits.dataanalysis.result.ValueType.COUNT).withFormatted(String.valueOf(count))); + } + } + return (metrics.stream().filter(Objects::nonNull).toList()); + } + + /******************************************************************************* + ** No nulls in report cells; unknown and suppressed are distinct from zero. + *******************************************************************************/ + private static List row(Object... values) + { + return (java.util.Arrays.stream(values).map(v -> v == null ? "" : String.valueOf(v)).toList()); + } + + /******************************************************************************* + ** Render a tuple using the engine's already-authorized display labels. + *******************************************************************************/ + private static String label(List headers) + { + return (headers.stream().map(PivotHeader::getLabel).collect(java.util.stream.Collectors.joining(" / "))); + } + + /******************************************************************************* + ** Metrics always use their semantic formatted value. + *******************************************************************************/ + public static String value(Metric metric) + { + return (metric == null ? "" : Boolean.TRUE.equals(metric.getSuppressed()) ? "Suppressed" : Objects.toString(metric.getFormatted(), "")); + } + + /******************************************************************************* + ** Profile metrics become one row per headline statistic. + *******************************************************************************/ + private static void metrics(List> rows, String section, List metrics) + { + metrics.forEach(m -> rows.add(row(section, m.getLabel(), value(m)))); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/saved/SavedAnalyses.java b/src/main/java/com/kingsrook/qbits/dataanalysis/saved/SavedAnalyses.java new file mode 100644 index 0000000..0a34225 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/saved/SavedAnalyses.java @@ -0,0 +1,151 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.saved; + +import java.io.Serializable; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import com.kingsrook.qbits.dataanalysis.DataAnalysisService; +import com.kingsrook.qbits.dataanalysis.engine.DrillFilters; +import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables; +import com.kingsrook.qbits.dataanalysis.watch.WatchRecords; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; + +/******************************************************************************* + ** Saved definitions always execute in the calling context, including shared reads. + *******************************************************************************/ +public class SavedAnalyses +{ + private final DataAnalysisService service; + + /******************************************************************************* + ** Use the instance's existing engine and cache policy. + *******************************************************************************/ + public SavedAnalyses(DataAnalysisService service) + { + this.service = service; + } + + /******************************************************************************* + ** Standard core READ and record locks precede source-table authorization. + *******************************************************************************/ + public SavedResult run(Serializable id) throws QException + { + return (SavedDefinition.parse(WatchRecords.get(DataAnalysisTables.SAVED, id)).run(service, false)); + } + + /******************************************************************************* + ** Readable list rows contain presentation metadata, never cached owner results. + *******************************************************************************/ + public record Entry(Serializable id, String name, String kind, String tableName, String tableLabel, String owner, + Boolean shared, Boolean starred, Instant updated) + { + } + + /******************************************************************************* + ** Only stars belonging to this viewer decorate their core-visible saved rows. + *******************************************************************************/ + public List list() throws QException + { + Set stars = new HashSet<>(); + for(QRecord row : query(DataAnalysisTables.SAVED_STAR, DrillFilters.condition("user_id", QCriteriaOperator.EQUALS, user()))) + { + stars.add(row.getValue("saved_id")); + } + List result = new ArrayList<>(); + for(QRecord row : query(DataAnalysisTables.SAVED, new QQueryFilter().withOrderBy(new QFilterOrderBy("modifyDate", false)))) + { + String tableName = row.getValueString("table_name"); + var table = QContext.getQInstance().getTable(tableName); + result.add(new Entry(row.getValue("id"), row.getValueString("name"), row.getValueString("kind"), tableName, + table == null ? tableName : table.getLabel(), row.getValueString("owner_user_id"), row.getValueBoolean("shared"), + stars.contains(row.getValue("id")), row.getValueInstant("modifyDate"))); + } + return (result); + } + + /******************************************************************************* + ** Idempotent per-runtime PUT/DELETE, with standard insert/delete permissions. + *******************************************************************************/ + public synchronized Boolean star(Serializable id, boolean starred) throws QException + { + WatchRecords.get(DataAnalysisTables.SAVED, id); + QQueryFilter filter = DrillFilters.and(DrillFilters.condition("saved_id", QCriteriaOperator.EQUALS, id), + DrillFilters.condition("user_id", QCriteriaOperator.EQUALS, user())); + List existing = query(DataAnalysisTables.SAVED_STAR, filter); + if(starred && existing.isEmpty()) + { + InsertInput input = new InsertInput(DataAnalysisTables.SAVED_STAR).withRecords(List.of(new QRecord().withValue("saved_id", id).withValue("user_id", user()))); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + QRecord record = new InsertAction().execute(input).getRecords().getFirst(); + if(record.getErrors() != null && !record.getErrors().isEmpty()) + { + throw new QException("Could not save the star."); + } + } + else if(!starred && !existing.isEmpty()) + { + DeleteInput input = new DeleteInput(DataAnalysisTables.SAVED_STAR).withPrimaryKeys(existing.stream().map(r -> r.getValue("id")).toList()); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + new DeleteAction().execute(input); + } + return (starred); + } + + /******************************************************************************* + ** Shared enforced query helper for visible records, not a security-filter layer. + *******************************************************************************/ + public static List query(String table, QQueryFilter filter) throws QException + { + QueryInput input = new QueryInput(table).withShouldFetchHeavyFields(true).withFilter(filter); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + return (new QueryAction().execute(input).getRecords()); + } + + /******************************************************************************* + ** An authenticated identity is required even in an unprotected host app. + *******************************************************************************/ + public static String user() throws QPermissionDeniedException + { + var session = QContext.getQSession(); + if(session == null || session.getUser() == null || session.getUser().getIdReference() == null) + { + throw new QPermissionDeniedException("Sign in to use saved analyses."); + } + return (session.getUser().getIdReference()); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/saved/SavedCustomizer.java b/src/main/java/com/kingsrook/qbits/dataanalysis/saved/SavedCustomizer.java new file mode 100644 index 0000000..c5de253 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/saved/SavedCustomizer.java @@ -0,0 +1,91 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.saved; + +import java.util.List; +import java.util.Optional; +import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables; +import com.kingsrook.qbits.dataanalysis.permissions.DataAnalysisPermissions; +import com.kingsrook.qbits.dataanalysis.watch.WatchRecords; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; + +/******************************************************************************* + ** Validate all core CRUD and maintain the read-lock projection atomically. + *******************************************************************************/ +public class SavedCustomizer implements TableCustomizerInterface +{ + /******************************************************************************* + ** Inserts cannot forge an owner or the derived shared-read key. + *******************************************************************************/ + @Override + public List preInsert(InsertInput input, List records, boolean preview) throws QException + { + for(QRecord row : records) + { + validate(row, row); + } + return (records); + } + + /******************************************************************************* + ** Merge partial edits so changing shared or owner also changes the read lock. + *******************************************************************************/ + @Override + public List preUpdate(UpdateInput input, List records, boolean preview, Optional> oldRecords) throws QException + { + for(QRecord patch : records) + { + QRecord merged = new QRecord(WatchRecords.get(DataAnalysisTables.SAVED, patch.getValue("id"))); + merged.getValues().putAll(patch.getValues()); + validate(patch, merged); + } + return (records); + } + + /******************************************************************************* + ** Report a plain validation message on the standard record editor. + *******************************************************************************/ + private void validate(QRecord output, QRecord merged) + { + try + { + String owner = merged.getValueString("owner_user_id"); + var session = QContext.getQSession(); + if(owner == null || owner.isBlank() || session == null || (!(session.getSecurityKeyValues() != null && session.getSecurityKeyValues().getOrDefault(DataAnalysisPermissions.ADMIN, List.of()).contains(true)) + && (session.getUser() == null || !owner.equals(session.getUser().getIdReference())))) + { + throw new QException("Only an administrator may assign another analysis owner."); + } + SavedDefinition.parse(merged); + output.withValue("read_owner_user_id", Boolean.TRUE.equals(merged.getValueBoolean("shared")) ? null : owner); + } + catch(QException e) + { + output.addError(new BadInputStatusMessage(e.getMessage())); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/saved/SavedDefinition.java b/src/main/java/com/kingsrook/qbits/dataanalysis/saved/SavedDefinition.java new file mode 100644 index 0000000..056036b --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/saved/SavedDefinition.java @@ -0,0 +1,313 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.saved; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qbits.dataanalysis.DataAnalysisService; +import com.kingsrook.qbits.dataanalysis.result.AnalysisRequest; +import com.kingsrook.qbits.dataanalysis.result.ColumnProfileRequest; +import com.kingsrook.qbits.dataanalysis.result.CompareRequest; +import com.kingsrook.qbits.dataanalysis.result.CorrelationRequest; +import com.kingsrook.qbits.dataanalysis.result.CrosstabRequest; +import com.kingsrook.qbits.dataanalysis.result.PivotRequest; +import com.kingsrook.qbits.dataanalysis.result.ShiftRequest; +import com.kingsrook.qbits.dataanalysis.result.TableProfileRequest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; + +/******************************************************************************* + ** One parser for persisted definitions, exports, schedules and SQL requests. + *******************************************************************************/ +public record SavedDefinition(String kind, AnalysisRequest request) +{ + private static final Map> TYPES = Map.of( + "COLUMN_PROFILE", ColumnProfileRequest.class, "TABLE_PROFILE", TableProfileRequest.class, + "PIVOT", PivotRequest.class, "COMPARE", CompareRequest.class, "CORRELATION", CorrelationRequest.class, + "CROSSTAB", CrosstabRequest.class, "SHIFT", ShiftRequest.class); + private static final Set FIELD_KEYS = Set.of("fieldName", "field", "dateField", "rowField", "columnField"); + public static final String FIELD_WARNING = "Some fields are not available to you"; + + /******************************************************************************* + ** The definition owns the typed filter; legacy filter_json remains supported. + *******************************************************************************/ + public static SavedDefinition parse(QRecord row) throws QException + { + Map body = object(row.getValueString("definition_json")); + if(body.get("filter") == null && row.getValueString("filter_json") != null) + { + body.put("filter", object(row.getValueString("filter_json"))); + } + return (parse(row.getValueString("kind"), row.getValueString("table_name"), body)); + } + + /******************************************************************************* + ** Reject malformed JSON with a plain, stable message. + *******************************************************************************/ + @SuppressWarnings("unchecked") + public static Map object(String json) throws QBadRequestException + { + try + { + Object value = JsonUtils.toObject(json, Object.class); + if(value instanceof Map map) + { + return (new LinkedHashMap<>((Map) map)); + } + } + catch(Exception expected) + { + // Normalize parser diagnostics; never return raw saved JSON in errors. + } + throw new QBadRequestException("definition_json must be a JSON object."); + } + + /******************************************************************************* + ** Validate metadata existence before saving, without querying source records. + *******************************************************************************/ + public static SavedDefinition parse(String kind, String table, Map body) throws QException + { + if(!TYPES.containsKey(kind == null ? "" : kind)) + { + throw new QBadRequestException("Choose a supported analysis kind."); + } + if(table == null || QContext.getQInstance().getTable(table) == null) + { + throw new QBadRequestException("Choose an existing analysis table."); + } + if(body.get("tableName") != null && !table.equals(body.get("tableName"))) + { + throw new QBadRequestException("The definition table must match table_name."); + } + Map copy = new LinkedHashMap<>(body); + copy.put("tableName", table); + try + { + validateJson(copy); + AnalysisRequest request = JsonUtils.toObject(JsonUtils.toJson(copy), TYPES.get(kind)); + requireShape(request); + for(String field : fields(copy)) + { + String[] parts = field.split("\\.", 2); + var metadata = QContext.getQInstance().getTable(parts.length == 2 ? parts[0] : table); + if(metadata == null || metadata.getField(parts.length == 2 ? parts[1] : field) == null) + { + throw new IllegalArgumentException("A definition field does not exist."); + } + } + return (new SavedDefinition(kind, request)); + } + catch(Exception e) + { + throw new QBadRequestException("Invalid analysis definition: " + (e instanceof IllegalArgumentException ? e.getMessage() : "check the request fields and types.")); + } + } + + /******************************************************************************* + ** Reject coercible scalar types and malformed predicates before Jackson conversion. + *******************************************************************************/ + private static void validateJson(Object value) + { + if(value instanceof Map map) + { + for(var entry : map.entrySet()) + { + String name = String.valueOf(entry.getKey()); + Object item = entry.getValue(); + if(item == null) + { + continue; + } + if((FIELD_KEYS.contains(name) || Set.of("tableName", "bucket", "operator").contains(name)) && !(item instanceof String)) + { + throw new IllegalArgumentException("Field names and operators must be strings."); + } + if(name.equals("fields") && (!(item instanceof List list) || list.stream().anyMatch(v -> !(v instanceof String)))) + { + throw new IllegalArgumentException("fields must be an array of field names."); + } + if(Set.of("filter", "filterA", "filterB").contains(name) && !(item instanceof Map)) + { + throw new IllegalArgumentException("Filters must be query filter objects."); + } + if(name.equals("criteria") && (!(item instanceof List list) || list.stream().anyMatch(v -> !(v instanceof Map c) + || !(c.get("fieldName") instanceof String) || !(c.get("operator") instanceof String)))) + { + throw new IllegalArgumentException("Filter criteria require a field and operator."); + } + if(Set.of("exact", "showTotals").contains(name) && !(item instanceof Boolean)) + { + throw new IllegalArgumentException("Flags must be booleans."); + } + validateJson(item); + } + } + else if(value instanceof List list) + { + list.forEach(SavedDefinition::validateJson); + } + } + + /******************************************************************************* + ** Required selections must not turn an invalid draft into a different query. + *******************************************************************************/ + private static void requireShape(AnalysisRequest request) + { + boolean valid = switch(request) + { + case ColumnProfileRequest r -> r.getFieldName() != null; + case TableProfileRequest r -> r.getFields() == null || !r.getFields().isEmpty(); + case CompareRequest r -> r.getFields() != null && !r.getFields().isEmpty(); + case CorrelationRequest r -> r.getFields() != null && r.getFields().size() >= 2; + case CrosstabRequest r -> r.getRowField() != null && r.getColumnField() != null; + case ShiftRequest r -> r.getField() != null && r.getDateField() != null; + case PivotRequest r -> r.getValues() != null && !r.getValues().isEmpty() + && r.getValues().stream().allMatch(v -> v.getOperator() != null && (v.getOperator().name().equals("COUNT") || v.getField() != null)); + default -> false; + }; + if(!valid) + { + throw new IllegalArgumentException("Choose the fields and measures required by this analysis kind."); + } + } + + /******************************************************************************* + ** Include selected and filter fields, including both compare populations. + *******************************************************************************/ + public static Set fields(Object value) + { + Set names = new LinkedHashSet<>(); + if(value instanceof Map map) + { + for(var entry : map.entrySet()) + { + if(FIELD_KEYS.contains(entry.getKey()) && entry.getValue() instanceof String name) + { + names.add(name); + } + else if("fields".equals(entry.getKey()) && entry.getValue() instanceof List list) + { + list.forEach(name -> names.add((String) name)); + } + else + { + names.addAll(fields(entry.getValue())); + } + } + } + else if(value instanceof List list) + { + list.forEach(item -> names.addAll(fields(item))); + } + return (names); + } + + /******************************************************************************* + ** A fresh tree makes field reduction request-local and cannot change storage. + *******************************************************************************/ + public Map body() throws QException + { + return (object(JsonUtils.toJson(request))); + } + + /******************************************************************************* + ** Filter rights are not weakened: an unavailable predicate prevents execution. + ** Required denied axes return an empty typed envelope rather than change meaning. + *******************************************************************************/ + @SuppressWarnings("unchecked") + public SavedResult run(DataAnalysisService service, boolean strict) throws QException + { + service.capabilities(request.getTableName()); + Map body = body(); + Set denied = new LinkedHashSet<>(); + for(String field : fields(body)) + { + if(!service.isAvailableField(request.getTableName(), field)) + { + denied.add(field); + } + } + if(strict && !denied.isEmpty()) + { + throw new QBadRequestException(FIELD_WARNING); + } + List warnings = denied.isEmpty() ? List.of() : List.of(FIELD_WARNING); + for(String filter : List.of("filter", "filterA", "filterB")) + { + if(fields(body.get(filter)).stream().anyMatch(denied::contains)) + { + return (SavedResult.of(kind, null, warnings)); + } + } + if(body.get("fields") instanceof List list) + { + body.put("fields", list.stream().filter(f -> !denied.contains(f)).toList()); + } + for(String axis : List.of("rows", "columns", "values")) + { + if(body.get(axis) instanceof List list) + { + body.put(axis, list.stream().filter(item -> fields(item).stream().noneMatch(denied::contains)).toList()); + } + } + if(FIELD_KEYS.stream().anyMatch(key -> denied.contains(body.get(key)))) + { + return (SavedResult.of(kind, null, warnings)); + } + AnalysisRequest reduced; + try + { + reduced = JsonUtils.toObject(JsonUtils.toJson(body), TYPES.get(kind)); + requireShape(reduced); + } + catch(Exception e) + { + return (SavedResult.of(kind, null, warnings)); + } + Object result = switch(reduced) + { + case ColumnProfileRequest r -> + { + var backend = service.backend(r.getTableName()); + yield service.compute(r, backend, backend.scope(r.getTableName(), r.getFilter(), Boolean.TRUE.equals(r.getExact()))); + } + case TableProfileRequest r -> + { + var backend = service.backend(r.getTableName()); + yield service.compute(r, backend, backend.scope(r.getTableName(), r.getFilter(), Boolean.TRUE.equals(r.getExact()))); + } + case PivotRequest r -> service.pivot(r); + case CompareRequest r -> service.compare(r); + case CorrelationRequest r -> service.correlation(r); + case CrosstabRequest r -> service.crosstab(r); + case ShiftRequest r -> service.shift(r); + default -> throw new QBadRequestException("Unsupported analysis kind."); + }; + return (SavedResult.of(kind, result, warnings)); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/saved/SavedResult.java b/src/main/java/com/kingsrook/qbits/dataanalysis/saved/SavedResult.java new file mode 100644 index 0000000..954cde5 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/saved/SavedResult.java @@ -0,0 +1,60 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.saved; + +import java.util.List; +import com.kingsrook.qbits.dataanalysis.result.ColumnProfile; +import com.kingsrook.qbits.dataanalysis.result.CompareResult; +import com.kingsrook.qbits.dataanalysis.result.CorrelationResult; +import com.kingsrook.qbits.dataanalysis.result.CrosstabResult; +import com.kingsrook.qbits.dataanalysis.result.PivotResult; +import com.kingsrook.qbits.dataanalysis.result.ShiftResult; +import com.kingsrook.qbits.dataanalysis.result.TableProfile; + +/******************************************************************************* + ** Discriminated result: exactly the member named by kind is populated when runnable. + *******************************************************************************/ +public record SavedResult(String kind, List warnings, ColumnProfile columnProfile, TableProfile tableProfile, + PivotResult pivot, CompareResult compare, CorrelationResult correlation, CrosstabResult crosstab, ShiftResult shift) +{ + /******************************************************************************* + ** Preserve typed engine results without an untyped JSON payload. + *******************************************************************************/ + public static SavedResult of(String kind, Object result, List warnings) + { + java.util.ArrayList notices = new java.util.ArrayList<>(warnings); + if(result instanceof ColumnProfile p) + { + notices.addAll(p.getWarnings()); + } + else if(result instanceof TableProfile p) + { + notices.addAll(p.getWarnings()); + } + else if(result instanceof PivotResult p) + { + notices.addAll(p.getWarnings()); + } + return (new SavedResult(kind, notices.stream().distinct().toList(), result instanceof ColumnProfile p ? p : null, result instanceof TableProfile p ? p : null, + result instanceof PivotResult p ? p : null, result instanceof CompareResult p ? p : null, + result instanceof CorrelationResult p ? p : null, result instanceof CrosstabResult p ? p : null, result instanceof ShiftResult p ? p : null)); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/saved/SqlHandoff.java b/src/main/java/com/kingsrook/qbits/dataanalysis/saved/SqlHandoff.java new file mode 100644 index 0000000..681969b --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/saved/SqlHandoff.java @@ -0,0 +1,147 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.saved; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qbits.dataanalysis.DataAnalysisService; +import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface; +import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput; + +/******************************************************************************* + ** Request-local capture of core's actual aggregate SQL, never a separate SQL planner. + *******************************************************************************/ +public final class SqlHandoff +{ + private static final ThreadLocal> CAPTURE = new ThreadLocal<>(); + private static final String NOTE = "Record security filters are applied by QQQ at run time. This SQL is for reading; ? placeholders remain bound by QQQ."; + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private SqlHandoff() + { + } + + /******************************************************************************* + ** Statement ranking prefers the main grouped matrix, then the widest metric batch. + *******************************************************************************/ + private record Statement(String table, String sql, int dimensions, int measures) + { + } + + /******************************************************************************* + ** Transported capability, statement and security note. + *******************************************************************************/ + public record Result(Boolean available, String sql, String note) + { + } + + /******************************************************************************* + ** Installation plus its configured access permission are both required. + *******************************************************************************/ + public static boolean installedAndAllowed() + { + var instance = QContext.getQInstance(); + var app = instance.getApp("sqlWorkbench"); + boolean installed = app != null || instance.getTables().keySet().stream().anyMatch(t -> t.startsWith("sql_workbench_")); + String base = app != null && app.getPermissionRules() != null && app.getPermissionRules().getPermissionBaseName() != null + ? app.getPermissionRules().getPermissionBaseName() : "sql_workbench"; + return (installed && QContext.getQSession() != null && QContext.getQSession().hasPermission(base + ".hasAccess")); + } + + /******************************************************************************* + ** The backend metadata inheritance check avoids a production RDBMS dependency. + *******************************************************************************/ + public static Result execute(SavedDefinition definition, DataAnalysisService service) throws QException + { + if(!installedAndAllowed()) + { + return (new Result(false, null, "SQL Workbench is not installed or available to you.")); + } + service.capabilities(definition.request().getTableName()); + var table = QContext.getQInstance().getTable(definition.request().getTableName()); + Class type = QContext.getQInstance().getBackend(table.getBackendName()).getClass(); + boolean rdbms = false; + while(type != null) + { + rdbms |= type.getSimpleName().equals("RDBMSBackendMetaData"); + type = type.getSuperclass(); + } + if(!rdbms) + { + return (new Result(false, null, "SQL is unavailable for a non-RDBMS backend.")); + } + List statements = new ArrayList<>(); + CAPTURE.set(statements); + try + { + definition.run(service, true); + } + finally + { + CAPTURE.remove(); + } + java.util.Comparator ranking = List.of("PIVOT", "CROSSTAB", "SHIFT").contains(definition.kind()) + ? java.util.Comparator.comparingInt(Statement::dimensions).thenComparingInt(Statement::measures) + : java.util.Comparator.comparingInt(Statement::measures).thenComparingInt(s -> -s.dimensions()); + Statement main = statements.stream().filter(s -> s.table().equals(table.getName())).max(ranking).orElse(null); + return (main == null ? new Result(false, null, "Core did not expose SQL for this aggregate. " + NOTE) : new Result(true, main.sql(), NOTE)); + } + + /******************************************************************************* + ** Execute the ordinary authorized core action, then read its backend QueryStat. + ** Core exposes QueryStat on the backend but not its action reference; this narrow + ** bridge never changes action state, global logging, or QueryStatManager settings. + *******************************************************************************/ + public static AggregateOutput aggregate(AggregateInput input) throws QException + { + if(CAPTURE.get() != null) + { + input.setTableMetaData(input.getTable().clone().withCapability(com.kingsrook.qqq.backend.core.model.metadata.tables.Capability.QUERY_STATS)); + } + AggregateAction action = new AggregateAction(); + AggregateOutput output = action.execute(input); + if(CAPTURE.get() != null) + { + try + { + Field field = AggregateAction.class.getDeclaredField("aggregateInterface"); + field.setAccessible(true); + var stat = ((AggregateInterface) field.get(action)).getQueryStat(); + if(stat != null && stat.getQueryText() != null) + { + CAPTURE.get().add(new Statement(input.getTableName(), stat.getQueryText(), input.getGroupBys().size(), input.getAggregates().size())); + } + } + catch(ReflectiveOperationException | RuntimeException expected) + { + // A future core change makes handoff unavailable rather than invent SQL. + } + } + return (output); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/schedule/ScheduleCustomizer.java b/src/main/java/com/kingsrook/qbits/dataanalysis/schedule/ScheduleCustomizer.java new file mode 100644 index 0000000..884ea43 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/schedule/ScheduleCustomizer.java @@ -0,0 +1,116 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.schedule; + +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables; +import com.kingsrook.qbits.dataanalysis.saved.SavedDefinition; +import com.kingsrook.qbits.dataanalysis.watch.WatchRecords; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; + +/******************************************************************************* + ** Core CRUD validates recipients and computes the next occurrence on configuration edits. + *******************************************************************************/ +public class ScheduleCustomizer implements TableCustomizerInterface +{ + private static final Set CONFIG = Set.of("saved_id", "cron", "frequency", "day", "time", "time_zone", "enabled", "recipients", "format"); + + /******************************************************************************* + ** Disabled empty drafts may be completed later. + *******************************************************************************/ + @Override + public List preInsert(InsertInput input, List records, boolean preview) throws QException + { + for(QRecord row : records) + { + validate(row, row); + } + return (records); + } + + /******************************************************************************* + ** Status-only scheduler writes do not move the next occurrence twice. + *******************************************************************************/ + @Override + public List preUpdate(UpdateInput input, List records, boolean preview, Optional> oldRecords) throws QException + { + for(QRecord patch : records) + { + if(patch.getValues().keySet().stream().anyMatch(CONFIG::contains)) + { + QRecord row = new QRecord(WatchRecords.get(DataAnalysisTables.SCHEDULE, patch.getValue("id"))); + row.getValues().putAll(patch.getValues()); + validate(patch, row); + } + } + return (records); + } + + /******************************************************************************* + ** User ids are stored as a JSON array, not an unchecked email recipient list. + *******************************************************************************/ + public static List recipients(QRecord row) throws QException + { + try + { + Object value = JsonUtils.toObject(row.getValueString("recipients"), Object.class); + if(value instanceof List list && !list.isEmpty() && list.size() <= 100 + && list.stream().allMatch(v -> v instanceof String s && !s.isBlank())) + { + return (list.stream().map(String.class::cast).distinct().toList()); + } + } + catch(Exception expected) + { + // Use the same plain error for JSON and shape errors. + } + throw new QException("Recipients must be a JSON array of 1 to 100 user ids."); + } + + /******************************************************************************* + ** Scheduling references a real, visible saved analysis. + *******************************************************************************/ + private void validate(QRecord output, QRecord row) + { + try + { + SavedDefinition.parse(WatchRecords.get(DataAnalysisTables.SAVED, row.getValue("saved_id"))); + if(Boolean.TRUE.equals(row.getValueBoolean("enabled")) || row.getValueString("frequency") != null || row.getValueString("cron") != null) + { + recipients(row); + output.withValue("next_run_at", ScheduleTiming.next(row, Instant.now())); + } + } + catch(QException e) + { + output.addError(new BadInputStatusMessage(e.getMessage())); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/schedule/ScheduleRunner.java b/src/main/java/com/kingsrook/qbits/dataanalysis/schedule/ScheduleRunner.java new file mode 100644 index 0000000..5aee7b8 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/schedule/ScheduleRunner.java @@ -0,0 +1,305 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.schedule; + +import java.io.Serializable; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import com.kingsrook.qbits.dataanalysis.DataAnalysisQBitConfig; +import com.kingsrook.qbits.dataanalysis.DataAnalysisService; +import com.kingsrook.qbits.dataanalysis.engine.DrillFilters; +import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables; +import com.kingsrook.qbits.dataanalysis.permissions.DataAnalysisPermissions; +import com.kingsrook.qbits.dataanalysis.saved.AnalysisExport; +import com.kingsrook.qbits.dataanalysis.saved.SavedAnalyses; +import com.kingsrook.qbits.dataanalysis.saved.SavedDefinition; +import com.kingsrook.qbits.dataanalysis.saved.SavedResult; +import com.kingsrook.qbits.dataanalysis.watch.WatchRecords; +import com.kingsrook.qqq.backend.core.actions.messaging.SendMessageAction; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; +import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.alerts.AlertRecordRef; +import com.kingsrook.qqq.backend.core.alerts.AlertSeverity; +import com.kingsrook.qqq.backend.core.alerts.AlertSignal; +import com.kingsrook.qqq.backend.core.alerts.QAlerts; +import com.kingsrook.qqq.backend.core.context.CapturedContext; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; +import com.kingsrook.qqq.backend.core.model.actions.messaging.Attachment; +import com.kingsrook.qqq.backend.core.model.actions.messaging.Content; +import com.kingsrook.qqq.backend.core.model.actions.messaging.Party; +import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageInput; +import com.kingsrook.qqq.backend.core.model.actions.messaging.email.EmailContentRole; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.model.session.QUser; +import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleDispatcher; + +/******************************************************************************* + ** Scheduled work uses owner and recipient sessions; maintenance only inventories rows. + *******************************************************************************/ +public class ScheduleRunner +{ + public static final String ALERT_TYPE = "data.analysis.schedule.failed"; + private final DataAnalysisQBitConfig config; + private final DataAnalysisService service; + + /******************************************************************************* + ** One instance serializes manual and automatic delivery on this application node. + *******************************************************************************/ + public ScheduleRunner(DataAnalysisQBitConfig config, DataAnalysisService service) + { + this.config = config; + this.service = service; + } + + /******************************************************************************* + ** Enumerate enabled due rows with a maintenance session restricted to metadata. + *******************************************************************************/ + public synchronized Integer runDue(Instant now) throws QException + { + QSession maintenance = new QSession().withUser(new QUser().withIdReference("data-analysis-scheduler")) + .withPermissions("data_analysis.saved.read", "data_analysis.saved.edit") + .withSecurityKeyValue(DataAnalysisPermissions.ADMIN, true); + if(QContext.getQSession().getBackendVariants() != null) + { + maintenance.setBackendVariants(new java.util.HashMap<>(QContext.getQSession().getBackendVariants())); + } + int[] count = {0}; + QContext.withTemporaryContext(new CapturedContext(QContext.getQInstance(), maintenance), () -> + { + var filter = DrillFilters.and(DrillFilters.condition("enabled", QCriteriaOperator.EQUALS, true), + DrillFilters.condition("next_run_at", QCriteriaOperator.LESS_THAN_OR_EQUALS, now)); + for(QRecord row : SavedAnalyses.query(DataAnalysisTables.SCHEDULE, filter)) + { + sendNow(row.getValue("id"), now); + count[0]++; + } + }); + return (count[0]); + } + + /******************************************************************************* + ** Manual delivery is authorized before source work; all attempts record their outcome. + *******************************************************************************/ + public synchronized String sendNow(Serializable id, Instant now) throws QException + { + UpdateInput authorization = new UpdateInput(DataAnalysisTables.SCHEDULE); + authorization.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + if(!PermissionsHelper.hasTablePermission(authorization, DataAnalysisTables.SCHEDULE, TablePermissionSubType.EDIT)) + { + throw new QPermissionDeniedException("Schedule editing is unavailable."); + } + QRecord schedule = WatchRecords.get(DataAnalysisTables.SCHEDULE, id); + String status; + boolean failed = false; + try + { + status = deliver(schedule); + } + catch(Exception e) + { + failed = true; + status = "Failed: " + (e instanceof QException ? e.getMessage() : "Delivery unavailable; check the host configuration."); + } + Instant next = null; + try + { + next = ScheduleTiming.next(schedule, now); + } + catch(QException e) + { + failed = true; + status += "; " + e.getMessage(); + } + QRecord patch = new QRecord().withValue("id", id).withValue("last_run_at", now).withValue("last_status", status).withValue("next_run_at", next); + UpdateInput input = new UpdateInput(DataAnalysisTables.SCHEDULE).withRecords(List.of(patch)); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + QRecord stored = new UpdateAction().execute(input).getRecords().getFirst(); + if(stored.getErrors() != null && !stored.getErrors().isEmpty()) + { + throw new QException("Schedule status could not be stored."); + } + if(failed) + { + QAlerts.raise(new AlertSignal(ALERT_TYPE).withDedupKey("data_analysis_schedule:" + id).withSeverity(AlertSeverity.ERROR) + .withTitle("Analysis schedule failed").withSummary(status).withRecordRef(AlertRecordRef.of(DataAnalysisTables.SCHEDULE, id))); + } + return (status); + } + + /******************************************************************************* + ** The host resolves identities; an unavailable identity never becomes a system user. + *******************************************************************************/ + private QSession session(String id) throws QException + { + QSession session; + try + { + session = config.getOwnerSessionSupplier() == null + ? new QAuthenticationModuleDispatcher().getQModule(QContext.getQInstance().getAuthentication()).createAutomatedSessionForUser(QContext.getQInstance(), id) + : config.getOwnerSessionSupplier().apply(id); + } + catch(Exception e) + { + throw new QException("User session unavailable."); + } + if(session == null || session.getUser() == null || !Objects.equals(id, session.getUser().getIdReference()) + || session.getSecurityKeyValues() == null || !session.getSecurityKeyValues().getOrDefault(DataAnalysisTables.USER_KEY, List.of()).contains(id)) + { + throw new QException("User session unavailable."); + } + return (session); + } + + /******************************************************************************* + ** Re-run for each recipient to respect their record locks, not just table grants. + *******************************************************************************/ + private String deliver(QRecord schedule) throws QException + { + if(config.getMessagingProviderName() == null || QContext.getQInstance().getMessagingProvider(config.getMessagingProviderName()) == null) + { + throw new QException("No messaging provider configured."); + } + if(config.getFromAddress() == null || config.getRecipientAddressSupplier() == null) + { + throw new QException("Configure fromAddress and recipientAddressSupplier for scheduled email."); + } + if(List.of("CSV", "XLSX").contains(Objects.toString(schedule.getValueString("format"), ""))) + { + var provider = new com.kingsrook.qqq.backend.core.modules.messaging.QMessagingProviderDispatcher() + .getMessagingProviderInterface(QContext.getQInstance().getMessagingProvider(config.getMessagingProviderName())); + if(provider.getClass().equals(com.kingsrook.qqq.backend.core.model.metadata.messaging.email.EmailMessagingProvider.class)) + { + throw new QException("Core EMAIL provider does not support attachments; configure an attachment-capable messaging provider."); + } + } + QRecord saved = WatchRecords.get(DataAnalysisTables.SAVED, schedule.getValue("saved_id")); + String owner = saved.getValueString("owner_user_id"); + SavedDefinition[] definition = new SavedDefinition[1]; + SavedResult[] ownerResult = new SavedResult[1]; + QContext.withTemporaryContext(new CapturedContext(QContext.getQInstance(), session(owner)), () -> + { + definition[0] = SavedDefinition.parse(WatchRecords.get(DataAnalysisTables.SAVED, saved.getValue("id"))); + if("TABLE_PROFILE".equals(definition[0].kind()) && definition[0].body().get("fields") == null) + { + var body = definition[0].body(); + body.put("fields", service.capabilities(definition[0].request().getTableName()).getFields()); + definition[0] = SavedDefinition.parse(definition[0].kind(), definition[0].request().getTableName(), body); + } + ownerResult[0] = definition[0].run(service, true); + }); + List skipped = new ArrayList<>(); + List failures = new ArrayList<>(); + int sent = 0; + for(String recipient : ScheduleCustomizer.recipients(schedule)) + { + SavedResult[] result = new SavedResult[1]; + String address; + try + { + QSession viewer = session(recipient); + QContext.withTemporaryContext(new CapturedContext(QContext.getQInstance(), viewer), () -> + { + result[0] = recipient.equals(owner) ? ownerResult[0] : definition[0].run(service, true); + }); + address = config.getRecipientAddressSupplier().apply(recipient); + if(address == null || address.isBlank()) + { + throw new QException("No email address."); + } + } + catch(Exception e) + { + skipped.add(recipient); + continue; + } + try + { + SendMessageInput message = new SendMessageInput().withMessagingProviderName(config.getMessagingProviderName()) + .withFrom(new Party().withAddress(config.getFromAddress())).withTo(new Party().withAddress(address)) + .withSubject("Analysis: " + saved.getValueString("name")) + .withContent(new Content().withContentRole(EmailContentRole.HTML).withBody(html(saved.getValueString("name"), result[0]))); + String format = schedule.getValueString("format"); + if("CSV".equals(format) || "XLSX".equals(format)) + { + var file = AnalysisExport.file(result[0], format); + message.withAttachment(new Attachment().withName(file.filename()).withContents(file.bytes())); + } + new SendMessageAction().execute(message); + sent++; + } + catch(Exception e) + { + failures.add(recipient); + } + } + String status = "Sent " + sent + "; skipped: " + (skipped.isEmpty() ? "none" : String.join(", ", skipped)); + if(!failures.isEmpty()) + { + throw new QException(status + "; delivery failed for: " + String.join(", ", failures)); + } + return (status); + } + + /******************************************************************************* + ** Formatted result rows are escaped; pivots include only the first twenty rows. + *******************************************************************************/ + public static String html(String name, SavedResult result) + { + var table = AnalysisExport.table(result); + StringBuilder html = new StringBuilder("

    ").append(escape(name)).append("

    "); + for(String warning : result.warnings()) + { + html.append("

    ").append(escape(warning)).append("

    "); + } + html.append("
    "); + for(var metric : AnalysisExport.headlines(result)) + { + html.append("
    ").append(escape(metric.getLabel())).append("
    ").append(escape(AnalysisExport.value(metric))).append("
    "); + } + html.append("
    "); + table.columns().forEach(c -> html.append("")); + html.append(""); + for(var row : table.rows().stream().limit(20).toList()) + { + html.append(""); + row.forEach(value -> html.append("")); + html.append(""); + } + return (html.append("
    ").append(escape(c)).append("
    ").append(escape(value)).append("
    ").toString()); + } + + /******************************************************************************* + ** Source labels and values are text, never trusted email markup. + *******************************************************************************/ + private static String escape(String value) + { + return (Objects.toString(value, "").replace("&", "&").replace("<", "<").replace(">", ">").replace("\"", """).replace("'", "'")); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/schedule/ScheduleTiming.java b/src/main/java/com/kingsrook/qbits/dataanalysis/schedule/ScheduleTiming.java new file mode 100644 index 0000000..90a84f2 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/schedule/ScheduleTiming.java @@ -0,0 +1,111 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.schedule; + +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.temporal.TemporalAdjusters; +import java.util.Date; +import java.util.TimeZone; +import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import org.quartz.CronExpression; + +/******************************************************************************* + ** Calendar schedules use local wall time; cron uses Quartz's existing parser. + *******************************************************************************/ +public final class ScheduleTiming +{ + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private ScheduleTiming() + { + } + + /******************************************************************************* + ** Strictly after now; missing monthly dates clamp to that month's last day. + ** Java resolves spring gaps forward and chooses the first autumn occurrence. + *******************************************************************************/ + public static Instant next(QRecord row, Instant now) throws QBadRequestException + { + try + { + ZoneId zone = ZoneId.of(row.getValueString("time_zone") == null ? "UTC" : row.getValueString("time_zone")); + String cron = row.getValueString("cron"); + if(cron != null && !cron.isBlank()) + { + CronExpression expression = new CronExpression(cron); + expression.setTimeZone(TimeZone.getTimeZone(zone)); + Date next = expression.getNextValidTimeAfter(Date.from(now)); + if(next == null) + { + throw new IllegalArgumentException("Cron has no future occurrence."); + } + return (next.toInstant()); + } + LocalTime time = LocalTime.parse(row.getValueString("time")); + String frequency = row.getValueString("frequency"); + Integer day = row.getValueInteger("day"); + LocalDate date = now.atZone(zone).toLocalDate(); + for(int attempt = 0; attempt < 3; attempt++) + { + LocalDate candidate = switch(frequency) + { + case "DAILY" -> date; + case "WEEKLY" -> date.with(TemporalAdjusters.nextOrSame(java.time.DayOfWeek.of(day))); + case "MONTHLY" -> date.withDayOfMonth(Math.min(requireDay(day), date.lengthOfMonth())); + default -> throw new IllegalArgumentException("Choose DAILY, WEEKLY or MONTHLY."); + }; + Instant instant = candidate.atTime(time).atZone(zone).toInstant(); + if(instant.isAfter(now)) + { + return (instant); + } + date = switch(frequency) + { + case "WEEKLY" -> candidate.plusWeeks(1); + case "MONTHLY" -> candidate.plusMonths(1).withDayOfMonth(1); + default -> candidate.plusDays(1); + }; + } + throw new IllegalArgumentException("No future occurrence."); + } + catch(Exception e) + { + throw new QBadRequestException("Invalid schedule: choose a valid frequency/day, local time, time zone or Quartz cron."); + } + } + + /******************************************************************************* + ** Month days are one-based and never silently wrap into another month. + *******************************************************************************/ + private static int requireDay(Integer day) + { + if(day == null || day < 1 || day > 31) + { + throw new IllegalArgumentException("Monthly day must be 1 to 31."); + } + return (day); + } +} diff --git a/src/main/resources/openapi/qqq-v1/data-analysis.yaml b/src/main/resources/openapi/qqq-v1/data-analysis.yaml index 0541bf6..332a379 100644 --- a/src/main/resources/openapi/qqq-v1/data-analysis.yaml +++ b/src/main/resources/openapi/qqq-v1/data-analysis.yaml @@ -781,6 +781,9 @@ components: description: "Blanks." nullable: true type: "object" + DataAnalysisFileV1: + format: "binary" + type: "string" DataAnalysisLimitsV1: nullable: true properties: @@ -1154,6 +1157,111 @@ components: nullable: true type: "string" type: "object" + DataAnalysisSavedEntryV1: + properties: + id: + type: "integer" + name: + type: "string" + kind: + type: "string" + tableName: + type: "string" + tableLabel: + type: "string" + owner: + type: "string" + updated: + format: "date-time" + nullable: true + type: "string" + shared: + type: "boolean" + starred: + type: "boolean" + type: "object" + DataAnalysisSavedListV1: + properties: + analyses: + items: + $ref: "#/components/schemas/DataAnalysisSavedEntryV1" + type: "array" + type: "object" + DataAnalysisSavedRequestV1: + properties: + kind: + enum: + - "COLUMN_PROFILE" + - "TABLE_PROFILE" + - "PIVOT" + - "COMPARE" + - "CORRELATION" + - "CROSSTAB" + - "SHIFT" + type: "string" + tableName: + description: "Source QQQ table name; required for ad hoc export, supplied\ + \ by the path for SQL." + type: "string" + request: + anyOf: + - $ref: "#/components/schemas/DataAnalysisColumnProfileRequestV1" + - $ref: "#/components/schemas/DataAnalysisTableProfileRequestV1" + - $ref: "#/components/schemas/DataAnalysisPivotRequestV1" + - $ref: "#/components/schemas/DataAnalysisCompareRequestV1" + - $ref: "#/components/schemas/DataAnalysisCorrelationRequestV1" + - $ref: "#/components/schemas/DataAnalysisCrosstabRequestV1" + - $ref: "#/components/schemas/DataAnalysisShiftRequestV1" + description: "Typed request matching kind. Field names, filters and dimensions\ + \ use the existing analysis request contracts." + format: + description: "Required for export; ignored for SQL." + enum: + - "csv" + - "xlsx" + type: "string" + required: + - "kind" + - "request" + type: "object" + DataAnalysisSavedResultV1: + properties: + kind: + enum: + - "COLUMN_PROFILE" + - "TABLE_PROFILE" + - "PIVOT" + - "COMPARE" + - "CORRELATION" + - "CROSSTAB" + - "SHIFT" + type: "string" + warnings: + items: + type: "string" + type: "array" + columnProfile: + $ref: "#/components/schemas/DataAnalysisColumnProfileV1" + nullable: true + compare: + $ref: "#/components/schemas/DataAnalysisCompareV1" + nullable: true + correlation: + $ref: "#/components/schemas/DataAnalysisCorrelationV1" + nullable: true + crosstab: + $ref: "#/components/schemas/DataAnalysisCrosstabV1" + nullable: true + pivot: + $ref: "#/components/schemas/DataAnalysisPivotV1" + nullable: true + shift: + $ref: "#/components/schemas/DataAnalysisShiftV1" + nullable: true + tableProfile: + $ref: "#/components/schemas/DataAnalysisTableProfileV1" + nullable: true + type: "object" DataAnalysisScatterRequestV1: properties: tableName: @@ -1329,6 +1437,21 @@ components: nullable: true type: "number" type: "object" + DataAnalysisSqlV1: + properties: + available: + type: "boolean" + note: + type: "string" + sql: + nullable: true + type: "string" + type: "object" + DataAnalysisStarV1: + properties: + starred: + type: "boolean" + type: "object" DataAnalysisStartRunRequestV1: additionalProperties: false properties: @@ -1641,9 +1764,181 @@ info: description: "Profiles, pivots, comparisons, distribution shifts, correlations,\ \ scatter samples, crosstabs and anomaly watch previews as the signed-in user." title: "QQQ Middleware API extension: data-analysis" - version: "1.3.0" + version: "1.4.0" openapi: "3.0.3" paths: + /qqq/v1/qbit/data-analysis/saved/{id}/run: + post: + description: "Re-run a visible saved analysis as the signed-in viewer. Requires\ + \ saved READ and source READ/Analyze. Drops unavailable fields with a warning\ + \ and returns the result typed by kind; inaccessible required axes or predicates\ + \ produce no result." + parameters: + - description: "Owner-scoped analysis run or watch id." + in: "path" + name: "id" + required: true + schema: + type: "string" + responses: + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisSavedResultV1" + description: "Successful Response" + security: + - sessionUuidCookie: + - "N/A" + summary: "Run a saved analysis" + /qqq/v1/qbit/data-analysis/saved: + get: + description: "List core-visible saved analyses with name, kind, source table\ + \ label, owner, shared state, this viewer's starred flag and last update." + responses: + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisSavedListV1" + description: "Successful Response" + security: + - sessionUuidCookie: + - "N/A" + summary: "List visible saved analyses" + /qqq/v1/qbit/data-analysis/saved/{id}/star: + delete: + description: "Remove the signed-in user's star on a visible saved analysis.\ + \ Idempotent; uses standard saved delete permission and owner locks." + parameters: + - description: "Owner-scoped analysis run or watch id." + in: "path" + name: "id" + required: true + schema: + type: "string" + responses: + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisStarV1" + description: "Successful Response" + security: + - sessionUuidCookie: + - "N/A" + summary: "Unstar a saved analysis" + put: + description: "Star a visible saved analysis for the signed-in user. Idempotent;\ + \ uses standard saved insert permission and the personal star record lock." + parameters: + - description: "Owner-scoped analysis run or watch id." + in: "path" + name: "id" + required: true + schema: + type: "string" + responses: + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisStarV1" + description: "Successful Response" + security: + - sessionUuidCookie: + - "N/A" + summary: "Star a saved analysis" + /qqq/v1/qbit/data-analysis/tables/{table}/sql: + post: + description: "Execute a typed analysis as the viewer and capture the main RDBMS\ + \ aggregate SQL from core QueryStat. Requires installed SQL Workbench and\ + \ its access permission. Non-RDBMS backends report unavailable; SQL is for\ + \ reading and record security remains enforced at runtime." + parameters: + - description: "QQQ table name to analyze." + in: "path" + name: "table" + required: true + schema: + type: "string" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisSavedRequestV1" + required: true + responses: + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisSqlV1" + description: "Successful Response" + security: + - sessionUuidCookie: + - "N/A" + summary: "Read an analysis aggregate SQL statement" + /qqq/v1/qbit/data-analysis/export: + post: + description: "Execute a typed analysis as the signed-in viewer and download\ + \ its result table as CSV or XLSX. Requires source READ/Analyze and FULL fields;\ + \ uses core report formats." + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/DataAnalysisSavedRequestV1" + required: true + responses: + 200: + content: + application/vnd.openxmlformats-officedocument.spreadsheetml.sheet: + schema: + $ref: "#/components/schemas/DataAnalysisFileV1" + text/csv: + schema: + $ref: "#/components/schemas/DataAnalysisFileV1" + description: "Analysis result file" + security: + - sessionUuidCookie: + - "N/A" + summary: "Export an analysis result" + /qqq/v1/qbit/data-analysis/saved/{id}/export: + get: + description: "Re-run a visible saved analysis as the viewer, applying current\ + \ source permissions and dropping unavailable fields, then download its result\ + \ table as CSV or XLSX." + parameters: + - description: "Visible saved analysis id." + in: "path" + name: "id" + required: true + schema: + type: "integer" + - description: "Result file format." + in: "query" + name: "format" + required: true + schema: + enum: + - "csv" + - "xlsx" + type: "string" + responses: + 200: + content: + application/vnd.openxmlformats-officedocument.spreadsheetml.sheet: + schema: + $ref: "#/components/schemas/DataAnalysisFileV1" + text/csv: + schema: + $ref: "#/components/schemas/DataAnalysisFileV1" + description: "Analysis result file" + security: + - sessionUuidCookie: + - "N/A" + summary: "Export a saved analysis" /qqq/v1/qbit/data-analysis/tables/{table}/capabilities: get: description: "Discover whether the current user can analyze this table, readable\ diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisApiTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisApiTest.java index b5c8256..b528fda 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisApiTest.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisApiTest.java @@ -119,8 +119,17 @@ class DataAnalysisApiTest extends EngineFixture *******************************************************************************/ private JsonNode call(HttpMethod method, String path, Object body, int status) throws Exception { + String savedId = path.startsWith("/saved/") ? path.split("/")[2] : null; + if(savedId != null) + { + path = path.replace("/saved/" + savedId + "/", "/saved/{id}/"); + } var request = new UiApiRequest().withMethod(method).withPath(DataAnalysisApi.PREFIX + path) .withPathParam("table", "orders").withSessionUUID(caller).withUserTimezone("America/Chicago"); + if(savedId != null) + { + request.withPathParam("id", savedId); + } if(body != null) { request.withJsonBody(body); @@ -173,7 +182,7 @@ class DataAnalysisApiTest extends EngineFixture assertThat(denied.getStatus()).isEqualTo(403); assertThat(run.path(operation.equals("COLUMN_PROFILE") ? "columnProfile" : "tableProfile").isObject()).isTrue(); } - assertThat(invoker.listOperations().stream().filter(o -> o.pathTemplate().startsWith(DataAnalysisApi.PREFIX)).toList()).hasSize(14); + assertThat(invoker.listOperations().stream().filter(o -> o.pathTemplate().startsWith(DataAnalysisApi.PREFIX)).toList()).hasSize(21); var client = java.net.http.HttpClient.newHttpClient(); var response = client.send(java.net.http.HttpRequest.newBuilder(java.net.URI.create("http://127.0.0.1:" + server.getBoundPort() + "/qqq/v1/openapi.json")).build(), java.net.http.HttpResponse.BodyHandlers.ofString()); assertThat(response.statusCode()).isEqualTo(200); @@ -355,7 +364,7 @@ class DataAnalysisApiTest extends EngineFixture } relationshipCountDrills(call(HttpMethod.POST, "/tables/{table}/crosstab", Map.of("rowField", "status", "columnField", "customer"), 200)); // All new schemas are discoverable from the same served OpenAPI catalog. - assertThat(com.kingsrook.qbits.dataanalysis.api.DataAnalysisApi.extension().getSemanticVersion()).isEqualTo("1.3.0"); + assertThat(com.kingsrook.qbits.dataanalysis.api.DataAnalysisApi.extension().getSemanticVersion()).isEqualTo("1.4.0"); } /******************************************************************************* @@ -452,4 +461,56 @@ class DataAnalysisApiTest extends EngineFixture assertThat(invoker.invoke(request.withSessionUUID(caller).withPathParam("id", "99999")).getStatus()).isEqualTo(404); assertThat(invoker.invoke(request.withPathParam("id", "invalid")).getStatus()).isEqualTo(400); } + + @Test + void savedOperationsAndFileResponsesOverHttp() throws Exception + { + var shared = TestSupport.insert(com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables.SAVED, + SavedAnalysesTest.definition("alice", true, "TABLE_PROFILE", Map.of("fields", List.of("amount", "secret")))); + var privateRow = TestSupport.insert(com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables.SAVED, + SavedAnalysesTest.definition("alice", false, "COLUMN_PROFILE", Map.of("fieldName", "amount"))); + start(); + caller = "watch-saved-viewer"; + String path = "/saved/" + shared.getValue("id"); + var result = call(HttpMethod.POST, path + "/run", null, 200); + assertThat(result.path("kind").asText()).isEqualTo("TABLE_PROFILE"); + assertThat(result.path("warnings").toString()).contains("Some fields are not available to you"); + assertThat(result.path("tableProfile").path("fields").size()).isEqualTo(1); + call(HttpMethod.POST, "/saved/" + privateRow.getValue("id") + "/run", null, 404); + call(HttpMethod.POST, "/saved/invalid/run", null, 400); + assertThat(call(HttpMethod.PUT, path + "/star", null, 200).path("starred").asBoolean()).isTrue(); + call(HttpMethod.PUT, path + "/star", null, 200); + assertThat(call(HttpMethod.GET, "/saved", null, 200).path("analyses").get(0).path("starred").asBoolean()).isTrue(); + assertThat(call(HttpMethod.DELETE, path + "/star", null, 200).path("starred").asBoolean()).isFalse(); + assertThat(call(HttpMethod.POST, "/tables/{table}/sql", Map.of("kind", "TABLE_PROFILE", "request", Map.of("fields", List.of("amount"))), 200).path("available").asBoolean()).isFalse(); + var client = java.net.http.HttpClient.newHttpClient(); + for(String format : List.of("csv", "xlsx")) + { + String url = "http://127.0.0.1:" + server.getBoundPort() + "/qqq/v1" + DataAnalysisApi.PREFIX + path + "/export?format=" + format; + var response = client.send(java.net.http.HttpRequest.newBuilder(java.net.URI.create(url)).header("Cookie", "sessionUUID=" + caller).GET().build(), java.net.http.HttpResponse.BodyHandlers.ofByteArray()); + assertThat(response.statusCode()).as(new String(response.body(), java.nio.charset.StandardCharsets.UTF_8)).isEqualTo(200); + assertThat(response.headers().firstValue("Content-Disposition").orElse("")).contains("analysis." + format); + if(format.equals("xlsx")) + { + try(var workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(new java.io.ByteArrayInputStream(response.body()))) + { + assertThat(workbook.getSheetAt(0).getLastRowNum()).isPositive(); + } + } + else + { + assertThat(new String(response.body(), java.nio.charset.StandardCharsets.UTF_8)).contains("Completeness"); + } + } + var export = invoker.invoke(new UiApiRequest().withMethod(HttpMethod.POST).withPath(DataAnalysisApi.PREFIX + "/export").withSessionUUID(caller) + .withJsonBody(Map.of("kind", "COLUMN_PROFILE", "tableName", "orders", "request", Map.of("fieldName", "amount"), "format", "csv"))); + assertThat(export.getStatus()).as(export.getBody()).isEqualTo(200); + assertThat(export.getBody()).contains("Counts"); + call(HttpMethod.POST, "/export", Map.of("kind", "TABLE_PROFILE", "tableName", "orders", "request", Map.of(), "format", "pdf"), 400); + caller = "denied"; + call(HttpMethod.POST, path + "/run", null, 403); + var operations = invoker.listOperations().stream().filter(o -> o.pathTemplate().startsWith(DataAnalysisApi.PREFIX)).toList(); + assertThat(operations).hasSize(21); + assertThat(operations.stream().map(Object::toString).toList().toString()).contains("/saved/{id}/run", "/saved/{id}/star", "/saved/{id}/export", "/tables/{table}/sql", "/export"); + } } diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisHomeTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisHomeTest.java index fc34518..46288f7 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisHomeTest.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisHomeTest.java @@ -20,131 +20,87 @@ package com.kingsrook.qbits.dataanalysis; - import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.List; import com.kingsrook.qbits.dataanalysis.app.DataAnalysisHome; import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.MultiStatisticsData; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData; import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.session.QSession; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; - -/******************************************************************************* - ** Real home counts, empty states and permission-scoped results. - *******************************************************************************/ class DataAnalysisHomeTest { - /******************************************************************************* - ** Release the thread-local context. - *******************************************************************************/ @AfterEach void cleanup() { QContext.clear(); } - - - /******************************************************************************* - ** Empty counts explain why; missing permissions are not represented as zero. - *******************************************************************************/ @Test - void emptyAndDenied() throws Exception + void standardWidgetsWithEmptyAndDeniedStates() throws Exception { - QInstance instance = TestSupport.instance(); + var instance = TestSupport.instance(); QContext.init(instance, TestSupport.user("alice").withPermissions("data_analysis.saved.read", "data_analysis.watch.read", "data_analysis.run.read")); - String html = DataAnalysisHome.render(Instant.now()); - assertThat(html).contains("Mine: 0", "Starred by me: 0", "No saved analyses yet", - "No watches yet.", "No analysis runs yet.", "View all", "Open any table and choose Analyze on a column header to explore its data.", "

    "); - var widget = new DataAnalysisHome().render(new RenderWidgetInput()).getWidgetData(); - assertThat(widget).isInstanceOf(com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML.class); - assertThat(((com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML) widget).getType()).isEqualTo("html"); - assertThat(html).contains("·").doesNotContain("Â", "later", "phase", " c < 128)).isTrue(); - QContext.init(instance, TestSupport.user("alice")); - html = DataAnalysisHome.render(Instant.now()); - assertThat(html).contains("you need data_analysis.saved.read", "you need data_analysis.watch.read").doesNotContain("Mine: 0", "Complete: 0"); - QContext.init(instance, new QSession()); - assertThat(DataAnalysisHome.render(Instant.now())).contains("Sign in to see your saved analyses"); - } - - - - /******************************************************************************* - ** Different owners, shared flags, stars, disabled watches and date boundaries. - *******************************************************************************/ - @Test - void countsUseCurrentUserAndRealRows() throws Exception - { - QInstance instance = TestSupport.instance(); - Instant now = Instant.parse("2026-10-13T12:00:00Z"); - QRecord mine = TestSupport.insert(DataAnalysisTables.SAVED, TestSupport.saved("alice", true)); - TestSupport.insert(DataAnalysisTables.SAVED, TestSupport.saved("alice", false)); - QRecord bob = TestSupport.insert(DataAnalysisTables.SAVED, TestSupport.saved("bob", true)); - TestSupport.insert(DataAnalysisTables.SAVED_STAR, new QRecord().withValue("saved_id", mine.getValue("id")).withValue("user_id", "alice")); - TestSupport.insert(DataAnalysisTables.SAVED_STAR, new QRecord().withValue("saved_id", bob.getValue("id")).withValue("user_id", "alice")); - TestSupport.insert(DataAnalysisTables.SAVED_STAR, new QRecord().withValue("saved_id", mine.getValue("id")).withValue("user_id", "bob")); - TestSupport.insert(DataAnalysisTables.WATCH, WatchDetectorsTest.definition("SPIKE", "COUNT", null).withValue("enabled", true).withValue("state", "FIRING")); - TestSupport.insert(DataAnalysisTables.WATCH, WatchDetectorsTest.definition("DROP", "COUNT", null).withValue("enabled", true)); - TestSupport.insert(DataAnalysisTables.WATCH, WatchDetectorsTest.definition("DROP", "COUNT", null).withValue("enabled", false)); - run("alice", "COMPLETE", now); - run("alice", "COMPLETE", now.minus(7, ChronoUnit.DAYS)); - run("alice", "FAILED", now.minus(1, ChronoUnit.DAYS)); - run("alice", "COMPLETE", now.minus(7, ChronoUnit.DAYS).minusSeconds(1)); - run("alice", "FAILED", now.plusSeconds(1)); - run("alice", "RUNNING", now); - run("bob", "FAILED", now); - QContext.init(instance, TestSupport.user("alice").withPermissions("data_analysis.saved.read", "data_analysis.watch.read", "data_analysis.run.read")); - String html = DataAnalysisHome.render(now); - assertThat(html).contains("Mine: 2", "Shared (visible to you): 1", "Starred by me: 1", "Enabled: 2", "Firing: 1", "Recent analysis runs", "Complete", "Failed", "Running", "Not finished") - .doesNotContain("No saved analyses", "No watches yet", "No analysis runs yet"); - QContext.init(instance, TestSupport.user("bob").withPermissions("data_analysis.saved.read", "data_analysis.run.read")); - html = DataAnalysisHome.render(now); - assertThat(html).contains("Mine: 1", "Shared (visible to you): 1", "Starred by me: 0", "Failed", "you need data_analysis.watch.read"); - } - - - - /******************************************************************************* - ** Latest five remain owner-scoped even with administrator bypass and escape HTML. - *******************************************************************************/ - @Test - void recentRunDetailsAndLimit() throws Exception - { - QInstance instance = TestSupport.instance(); - instance.getTable(DataAnalysisTables.SAVED).withLabel("Saved & review"); - Instant now = Instant.parse("2026-10-13T23:10:00Z"); - for(int i = 0; i < 7; i++) + var counts = DataAnalysisHome.statistics(Instant.now()); + assertThat(counts.getStatisticsGroupData()).flatExtracting(g -> g.getStatisticList()).allSatisfy(s -> assertThat(s.getValue()).isZero()); + for(var widget : DataAnalysisHome.widgets()) { - TestSupport.insert(DataAnalysisTables.RUN, new QRecord().withValue("table_name", DataAnalysisTables.SAVED) - .withValue("operation", "COLUMN_PROFILE").withValue("user_id", "admin").withValue("status", "COMPLETE") - .withValue("started_at", now.minusSeconds(i * 60L)).withValue("finished_at", now.minusSeconds(i * 60L))); + assertThat(widget.getType()).isIn("multiStatistics", "table"); + var data = new DataAnalysisHome().render(new RenderWidgetInput().withWidgetMetaData(widget)).getWidgetData(); + if(data instanceof TableData table) + { + assertThat(table.getColumns()).hasSizeLessThanOrEqualTo(4); + assertThat(table.getRows()).isEmpty(); + assertThat(table.getNoRowsFoundHTML()).startsWith("No "); + } } - TestSupport.insert(DataAnalysisTables.RUN, new QRecord().withValue("table_name", "private_bob_table") - .withValue("operation", "TABLE_PROFILE").withValue("user_id", "bob").withValue("status", "FAILED").withValue("started_at", now)); - QContext.init(instance, TestSupport.admin().withValue(QSession.VALUE_KEY_USER_TIMEZONE, "America/Chicago")); - String html = DataAnalysisHome.render(now); - assertThat(html).contains("Saved <analysis> & review", "Column Profile", "2026-10-13 18:10", "2026-10-13 18:06", ""); - assertThat(html.split("
  • ", -1)).hasSize(6); - assertThat(html).contains("href=\"" + instance.getTablePath(DataAnalysisTables.RUN) + "/1\""); - assertThat(html).contains("href=\"" + instance.getTablePath(DataAnalysisTables.RUN) + "\">View all"); + QContext.init(instance, TestSupport.user("alice")); + assertThat(DataAnalysisHome.statistics(Instant.now()).getStatisticsGroupData()).allSatisfy(g -> + { + assertThat(g.getStatisticList()).isEmpty(); + assertThat(g.getSubheader()).contains("Unavailable"); + }); + assertThat(((TableData) DataAnalysisHome.render(DataAnalysisHome.FIRING, Instant.now())).getNoRowsFoundHTML()).contains("Unavailable"); } + @Test + void countsRowsLinksAndDateBoundaries() throws Exception + { + var instance = TestSupport.instance(); + Instant now = Instant.parse("2026-10-13T12:00:00Z"); + var mine = TestSupport.insert(DataAnalysisTables.SAVED, TestSupport.saved("alice", false).withValue("name", "