From 261d0213add0d3a348609514bc74e9644abd7d08 Mon Sep 17 00:00:00 2001 From: James Maes Date: Tue, 13 Oct 2026 16:26:15 -0500 Subject: [PATCH] feat(engine): column and table profiles, drill-through, privacy guard and exact runs The phase (b) analysis engine: typed results (metrics with drill filters, charts as data), column and table profiles, distinct values and duplicates, AnalysisSecurity (analyze permission, table READ, personalized table and field security, analyzable/excluded fields), the effective per-table config, the minimum group size guard, data versions, async exact runs cached in data_analysis_run, and DataAnalysisService as the facade. The AnalysisBackend port has an RC.5 stand-in that pushes down what core supports today and streams the rest with row limits. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 + README.md | 70 +- pom.xml | 5 + .../DataAnalysisQBitProducer.java | 1 + .../dataanalysis/DataAnalysisRuntime.java | 73 ++ .../dataanalysis/DataAnalysisService.java | 236 ++++++ .../dataanalysis/app/DataAnalysisHome.java | 12 +- .../dataanalysis/engine/AnalysisBackend.java | 145 ++++ .../dataanalysis/engine/AnalysisLabels.java | 179 +++++ .../dataanalysis/engine/AnalysisSecurity.java | 94 +++ .../dataanalysis/engine/ColumnProfiler.java | 219 ++++++ .../dataanalysis/engine/DistinctValues.java | 74 ++ .../dataanalysis/engine/DrillFilters.java | 239 ++++++ .../dataanalysis/engine/DuplicateFinder.java | 56 ++ .../dataanalysis/engine/EffectiveConfig.java | 211 ++++++ .../engine/EffectiveConfigResolver.java | 171 +++++ .../engine/MinGroupSizeGuard.java | 146 ++++ .../dataanalysis/engine/ProfileSupport.java | 150 ++++ .../engine/StandInAnalysisBackend.java | 689 ++++++++++++++++++ .../dataanalysis/engine/TableProfiler.java | 116 +++ .../dataanalysis/result/AnalysisRequest.java | 48 ++ .../qbits/dataanalysis/result/Axis.java | 100 +++ .../dataanalysis/result/CandidateKey.java | 101 +++ .../dataanalysis/result/Capabilities.java | 123 ++++ .../qbits/dataanalysis/result/Chart.java | 167 +++++ .../qbits/dataanalysis/result/ChartKind.java | 30 + .../dataanalysis/result/ColumnProfile.java | 455 ++++++++++++ .../result/ColumnProfileRequest.java | 122 ++++ .../dataanalysis/result/DateProfile.java | 167 +++++ .../result/DistinctValuesPage.java | 367 ++++++++++ .../result/DistinctValuesRequest.java | 188 +++++ .../qbits/dataanalysis/result/Drill.java | 79 ++ .../result/DuplicatesRequest.java | 123 ++++ .../dataanalysis/result/DuplicatesResult.java | 279 +++++++ .../dataanalysis/result/FieldSummary.java | 255 +++++++ .../qbits/dataanalysis/result/Metric.java | 210 ++++++ .../dataanalysis/result/NumericProfile.java | 101 +++ .../qbits/dataanalysis/result/Point.java | 123 ++++ .../qbits/dataanalysis/result/RunStatus.java | 189 +++++ .../qbits/dataanalysis/result/Series.java | 79 ++ .../dataanalysis/result/TableProfile.java | 323 ++++++++ .../result/TableProfileRequest.java | 123 ++++ .../dataanalysis/result/TextProfile.java | 123 ++++ .../qbits/dataanalysis/result/ValueCount.java | 145 ++++ .../qbits/dataanalysis/result/ValueType.java | 30 + .../dataanalysis/run/AnalysisRunService.java | 499 +++++++++++++ .../qbits/dataanalysis/run/DataVersion.java | 69 ++ .../dataanalysis/DataAnalysisHomeTest.java | 2 +- .../dataanalysis/EngineContractTest.java | 656 +++++++++++++++++ .../qbits/dataanalysis/EngineFixture.java | 145 ++++ .../qbits/dataanalysis/H2EngineTest.java | 40 + .../qbits/dataanalysis/MemoryEngineTest.java | 28 + .../dataanalysis/PostgresEngineTest.java | 73 ++ 53 files changed, 8437 insertions(+), 17 deletions(-) create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisRuntime.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisService.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisBackend.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisLabels.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisSecurity.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/ColumnProfiler.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/DistinctValues.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/DrillFilters.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/DuplicateFinder.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/EffectiveConfig.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/EffectiveConfigResolver.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/MinGroupSizeGuard.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/ProfileSupport.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/StandInAnalysisBackend.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/TableProfiler.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/AnalysisRequest.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/Axis.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/CandidateKey.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/Capabilities.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/Chart.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/ChartKind.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/ColumnProfile.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/ColumnProfileRequest.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/DateProfile.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/DistinctValuesPage.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/DistinctValuesRequest.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/Drill.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/DuplicatesRequest.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/DuplicatesResult.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/FieldSummary.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/Metric.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/NumericProfile.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/Point.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/RunStatus.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/Series.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/TableProfile.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/TableProfileRequest.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/TextProfile.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/ValueCount.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/result/ValueType.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/run/AnalysisRunService.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/run/DataVersion.java create mode 100644 src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java create mode 100644 src/test/java/com/kingsrook/qbits/dataanalysis/EngineFixture.java create mode 100644 src/test/java/com/kingsrook/qbits/dataanalysis/H2EngineTest.java create mode 100644 src/test/java/com/kingsrook/qbits/dataanalysis/MemoryEngineTest.java create mode 100644 src/test/java/com/kingsrook/qbits/dataanalysis/PostgresEngineTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 863e729..8a69132 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,4 +6,10 @@ Following [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and [Semantic ### Added +- Phase b host engine: typed column/table profiles, distinct pages, duplicates, count-equivalent drills, chart data, candidate keys and completeness heatmaps. +- Internal `AnalysisBackend` port and RC.5 stand-in with aggregate pushdown, bounded query pages, deterministic prefix estimates and honest truncation metadata. +- Personalized field/table authorization, enforced FK label lookup, runtime configuration overrides and minimum-group-size privacy before caching. +- Owner-scoped exact runs, asynchronous session propagation, normalized SHA-256 request identities and data-version/TTL cache reuse. +- Shared memory/H2/PostgreSQL contract tests for metric references, drill equivalence, permissions, locks, privacy, estimates, jobs and cache behavior. PostgreSQL tests skip when Docker is unavailable. + - Repository scaffold; analysis settings, saved analyses, schedules, stars, watches and runs tables; permissions; Analyses app home. diff --git a/README.md b/README.md index 5c5c0d7..ba9f9ba 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,17 @@ Statistical analysis and data dives on any QQQ table, filter or saved view: profiles, pivots, comparisons, anomalies, and saved and scheduled analyses. The approved [design is QRun-IO/qqq#1137](https://github.com/QRun-IO/qqq/pull/1137), `docs/DATA-ANALYSIS.md` in QQQ; [DESIGN.md](docs/DESIGN.md) includes its phase table. -Implemented now: repository scaffold, producer and configuration, six visible tables, enum sources, joins and child lists, permissions and owner locks, PostgreSQL DDL generation, and the Analyses home with real counts. No analysis engine, Analyze action, processes, scheduling, alert evaluation, exports or UI API endpoints ship yet. +## What works now + +The host API provides column and table profiles, distinct-value pages, duplicate groups, drill-through filters, effective per-table configuration, privacy suppression, and asynchronous exact runs with a persistent per-user cache. Metadata, six visible tables, owner locks, PostgreSQL DDL generation and the Analyses home are also available. + +HTTP endpoints require the QQQ RC.6 extension point and are next, along with the Analyze UI. Pivots, comparisons, anomalies, saved-analysis execution, scheduling and exports remain later phases. ## Requirements - Java 21; QQQ **4.1.0-RC.5+**. - A configured QQQ backend for the qbit tables and authenticated sessions with permissions and security keys. -- Later phases require the core aggregate extensions in DATA-ANALYSIS.md §3 and the UI API extension point ([qqq#1128](https://github.com/QRun-IO/qqq/pull/1128), RC.6). +- The engine works on RC.5 through an internal stand-in. A future `CoreAnalysisBackend` can replace it when the core aggregate extensions ship. HTTP endpoints require the UI API extension point ([qqq#1128](https://github.com/QRun-IO/qqq/pull/1128), RC.6). ## Install @@ -41,7 +45,7 @@ QQQ names and physical SQL table names are identical. `backendName` in configura | `data_analysis_watch` | Watches: metric, detector, parameters and observed state; links to saved analysis | | `data_analysis_run` | Analysis Runs: exact-run/cache result and history, scoped by user | -Only Saved Analyses, Watches, Analysis Settings and Analysis Runs appear in navigation. Schedules, stars and linked watches appear on saved records. Enum PVS cover analysis kind, chart kind, detector, watch state, run status, schedule frequency and format. JSON definitions, requests and results are stored as text; typed request validation arrives with the relevant execution phase. +Only Saved Analyses, Watches, Analysis Settings and Analysis Runs appear in navigation. Schedules, stars and linked watches appear on saved records. Enum PVS cover analysis kind, chart kind, detector, watch state, run status, schedule frequency and format. JSON definitions and results are stored as text. Profile, distinct-value and duplicate requests have typed Java contracts and are validated by the facade. ## Permissions @@ -54,11 +58,11 @@ Only Saved Analyses, Watches, Analysis Settings and Analysis Runs appear in navi | `data_analysis.admin` | The all-access key of the `data_analysis_user` record locks (see Security) | | `.analyze` | Analyze a source table, also requiring its READ permission | -`DataAnalysisPermissions.analyzePermissionName(table)` honors a custom permission base name and instance defaults. `canAnalyze(table)` requires both READ and the analyze permission; if instance core permission enforcement is off, only the READ check is waived. The feature check is a stand-in for core `TableFeaturePermissionsMetaData` (§3.6); hosts must grant these analyze names explicitly until that hook registers them in the catalog. All qbit tables use standard permission rules (no custom checker), so every name above appears in the core permission catalog and the qbit-authentik policy editor. +`DataAnalysisPermissions.analyzePermissionName(table)` honors a custom permission base name and instance defaults. `canAnalyze(table)` is the legacy helper; its READ check follows the instance setting. The engine adds an explicit READ check and always uses `PermissionEnforcement.ENFORCE`, even when the instance default is off. The feature check is a stand-in for core `TableFeaturePermissionsMetaData` (§3.6); hosts must grant these analyze names explicitly until that hook registers them in the catalog. All qbit tables use standard permission rules (no custom checker), so their CRUD permissions appear in the core permission catalog and the qbit-authentik policy editor. Hosts explicitly grant source-table analyze permissions until the core feature hook ships. ## Security -Every home count runs under the current `QContext` session with `PermissionEnforcement.ENFORCE`. A denied section explains the missing permission instead of showing zero. Mine and starred counts explicitly filter the current user even for administrators. Shared counts include only readable shared rows. Stars pointing at unreadable or deleted saved analyses do not count. Run counts cover starts within the last seven days, excluding future rows. +Every engine operation and home count runs under the current `QContext` session with `PermissionEnforcement.ENFORCE`. A denied section explains the missing permission instead of showing zero. Mine and starred counts explicitly filter the current user even for administrators. Shared counts include only readable shared rows. Stars pointing at unreadable or deleted saved analyses do not count. Run counts cover starts within the last seven days, excluding future rows. Saved analyses have a READ_AND_WRITE lock on `owner_user_id`; stars and runs have locks on `user_id`. Schedules inherit the saved analysis owner's lock through their join. Null owners are denied. The host's authentication/session adapter must supply: @@ -73,7 +77,11 @@ QQQ permissions and security-key values are separate. An admin permission alone **Shared READ is deliberately restricted in this scaffold.** The ordinary owner lock does not express owner OR shared while keeping writes owner-only, so setting `shared=true` currently does not expose a saved analysis to other users. Phase f will introduce separate read and write policies: reads permit owner OR shared only after checking `data_analysis.saved.read`; writes retain owner/admin checks. Query, get, count, joins, child lists and exports will receive cross-user regression tests before enabling shared READ. Collaboration sharing and favorites (§7.3, qqq#1136) will then replace the local sharing/star contracts. Shared analyses must always execute as their viewer. -Minimum group size defaults to 0 (off). Hosts should configure it for personal-data tables. Privacy suppression and outlier restrictions are planned with the analysis engine; the scaffold does not compute or cache analysis results. +Minimum group size defaults to 0 (off). Hosts should set it for personal-data tables. Groups, bins and distinct listings below N are merged into an Other row with no value or drill. Min/max, percentiles, mean and standard deviation are suppressed when the assessed population is below N. Outlier listings and their count's drill are removed whenever the guard is enabled. The guard runs before chart construction and before `result_json` is stored. + +Source fields must be present in personalized metadata, pass `FieldSecurityLockHelper`, and pass the configured allow/exclude lists. FK labels are queried with enforcement before core PVS formatting; unreadable related records or label fields fall back to the source id. This avoids RC.5's ordinary table-PVS translator's trusted permission bypass. + +Cached results are scoped to the authenticated user, normalized request, effective policy, session permissions/security keys/backend variants/time zone, and data version. Access and policy are checked again when retrieving runs. Hosts must grant `data_analysis.run.read`, `.insert` and `.edit` to analysis users for persisted profiles and exact jobs. Run errors contain safe text rather than backend data or query details. ## Configuration @@ -92,17 +100,59 @@ Minimum group size defaults to 0 (off). Hosts should configure it for personal-d | `excludeTables` | Empty set | | `tableDefaults` | Empty map of table names to `TableAnalysisDefaults` | -All properties have fluent `withX` setters. Global numeric defaults must be non-null and non-negative. Nullable per-table numeric overrides inherit global defaults; configured values must be non-negative. `TableAnalysisDefaults` also supports enabled, analyzable fields and excluded fields. Runtime table-config rows will take precedence over code defaults through the specified 60-second lookup in a later phase; the repository phase defines the configuration contract only. +All properties have fluent `withX` setters. Global numeric defaults must be non-null and non-negative. Nullable per-table numeric overrides inherit global defaults; configured values must be non-negative. `TableAnalysisDefaults` also supports enabled, analyzable fields and excluded fields. Runtime `data_analysis_table_config` values override per-table code defaults, which override global defaults. Null row values inherit; blank field lists mean all readable fields or no exclusions. Include/exclude table lists remain an outer restriction, even if a runtime row enables the table. Only the config-row lookup uses a temporary system session, restored in `finally`. It selects configuration fields only and is memoized for 60 seconds. Settings editors can call `service.getConfigResolver().invalidate()` for immediate refresh. + +`max_rows` bounds the stand-in's selected population and retained records. An exact request disables sampling but still respects this bound and reports `truncated: true` when necessary; increase the limit to obtain a complete larger profile. Zero selects no rows. Above `sample_threshold_rows`, estimates use the first `min(sample_rows, max_rows)` rows ordered by primary key. Counts describe that prefix, are not extrapolated, and carry `estimate`, `sampleRows`, `rowsScanned`, warnings and an `asOf` timestamp. This is deterministic prefix sampling, not a random sample. `cache_ttl_minutes: 0` disables cache reuse. + +## Host API + +Call the facade with the authenticated user's `QContext` installed. The producer registers an instance-scoped runtime: + +```java +import com.kingsrook.qbits.dataanalysis.DataAnalysisRuntime; +import com.kingsrook.qbits.dataanalysis.result.ColumnProfileRequest; +import com.kingsrook.qbits.dataanalysis.result.TableProfileRequest; +import com.kingsrook.qbits.dataanalysis.result.DistinctValuesRequest; +import com.kingsrook.qbits.dataanalysis.result.DuplicatesRequest; + +var service = DataAnalysisRuntime.service(); +var capabilities = service.capabilities("orders"); +var column = service.columnProfile(new ColumnProfileRequest() + .withTableName("orders").withFieldName("amount").withFilter(filter)); +var table = service.tableProfile(new TableProfileRequest() + .withTableName("orders").withFilter(filter)); +var values = service.distinctValues(new DistinctValuesRequest() + .withTableName("orders").withFieldName("status").withPage(0).withPageSize(100)); +var duplicates = service.duplicates(new DuplicatesRequest() + .withTableName("orders").withFields(java.util.List.of("customer", "status"))); +String runId = service.startExactRun(new ColumnProfileRequest() + .withTableName("orders").withFieldName("amount").withFilter(filter)); +var status = service.getRun(runId); // RUNNING, COMPLETE or FAILED; typed result on COMPLETE +``` + +`startExactRun` also accepts `TableProfileRequest`. Identical fresh exact requests reuse the existing COMPLETE row and id. A host may construct `DataAnalysisService(config, executor)` to use its managed executor; the default is the common pool. Workers receive a cloned session and instance context, with no inherited transaction or privileged action scope. + +The `result` package contains Serializable, Jackson-friendly request/result POJOs, semantic metrics, charts and drills. There are no frontend-specific fields. Distinct pages are zero-based with a maximum size of 500; search matches stored values. Guarded distinct groups are merged before pagination. Shares are fractions from 0 to 1. Numeric standard deviation is the sample statistic; percentiles match `percentile_cont` interpolation at `p * (n - 1)`. Date buckets use the session time zone (including DST), ISO Monday weeks and the specified 92-day/two-year/ten-year thresholds. Date range metrics are in days. Text shapes use at most the first 10,000 rows and state that sample size. + +Count drills combine an untouched copy of the request filter with their producing condition. Cardinality drills select one representative record per distinct non-null value; the privacy guard removes those representative drills. Truncated profiles retain their primary-key boundary in drills, so their counts remain reproducible. Serialized drill getters restore temporal operand types from the current instance metadata before core evaluates them. + +## RC.5 stand-in + +`engine.AnalysisBackend` isolates profilers from missing core operators. `StandInAnalysisBackend` pushes down COUNT, COUNT_DISTINCT, MIN, MAX, AVG, SUM, grouped counts, aggregate ordering and limits through core actions. RC.5 has no aggregate OFFSET, so pages over-fetch through the requested end and slice locally. Memory grouping pagination and scale-sensitive decimal equality have isolated compatibility handling. + +Variance, interpolated percentiles, string lengths/shapes, date classification, whitespace detection and representative-row drills use bounded primary-key query pages. Histogram/date membership counts use enforced range/count queries after their boundaries have been calculated. Numeric histograms have 20 interior bins between p1 and p99 plus below/above bins, and the last interior bin is closed. Profiles include Tukey outliers, date gaps, date-time weekday/hour patterns, chart alternatives, a seven-bucket moving average, table completeness and candidate keys (single fields and pairs among the eight most unique). + +Data versions use filtered COUNT plus MAX of a readable modification-date-like field, otherwise MAX(primary key). This is the specified inexpensive invalidation heuristic, not snapshot isolation; updates on a table without a modification timestamp can remain cached until TTL expiry. Concurrent writes can also change drill counts after a profile is produced. Results always retain `asOf`. ## Adopting in another app 1. Add a backend, install the producer and run the usual `QInstanceValidator`. 2. For PostgreSQL, obtain `DataAnalysisSchema.postgresScript(config)` and apply it through your application's migration system. It derives fields and unique indexes from table metadata. It does not execute DDL or upgrade existing schemas. 3. Map user-id security keys and permissions in the authentication adapter. Configure core audit persistence if field audit history is needed. -4. Review per-table include/exclude sets, limits and minimum group sizes before enabling future computation. Integrate Saved Analyses and Watches with host ESB/Quick Search when those optional host features are installed; this scaffold does not add those dependencies. +4. Review per-table include/exclude sets, limits and minimum group sizes before enabling analysis. Integrate Saved Analyses and Watches with host ESB/Quick Search when those optional host features are installed; this scaffold does not add those dependencies. 5. Use the Analyses home as the owner of analysis data. Other dashboards should link there until saved-analysis widgets are implemented. -Planned phases: **a** core aggregate extensions; **b** profiles, drill-through, runs/cache and endpoints; **c** pivots; **d** comparisons and relationships; **e** watch evaluation and alerts; **f** saved-analysis execution, sharing, stars integration, schedules, chart widgets, full home and exports. The current home summarizes stored rows; it does not run analyses or send deliveries. +Phase **b** profiles, drill-through and runs/cache are implemented through the host facade. Remaining work: **a** core aggregate extensions; **b** HTTP endpoints after QQQ RC.6 and analysis UI; **c** pivots; **d** comparisons and relationships; **e** watch evaluation and alerts; **f** saved-analysis execution, sharing, stars integration, schedules, chart widgets, full home and exports. The current home summarizes stored rows; the host facade executes profiles. The home does not send deliveries. ## Build @@ -110,7 +160,7 @@ Planned phases: **a** core aggregate extensions; **b** profiles, drill-through, ./mvnw verify ``` -The wrapper pins Maven 3.9.11. On Windows use `mvnw.cmd`. Verification runs Checkstyle, JUnit 5/AssertJ tests against the memory backend and JaCoCo gates of 70% instructions and 90% classes. For a populated local dependency cache use `./mvnw -o -q verify`. RDBMS, H2, Testcontainers PostgreSQL and the PostgreSQL driver remain test-scoped for later pushdown tests; middleware Javalin is optional. +The wrapper pins Maven 3.9.11. On Windows use `mvnw.cmd`. Verification runs Checkstyle, a shared JUnit 5/AssertJ contract suite on memory and H2, PostgreSQL via Testcontainers when Docker is available, and JaCoCo gates of 70% instructions and 90% classes. The suite checks numerical references, every emitted count/drill pair, security, guarded cache JSON, prefix sampling, asynchronous jobs and cache invalidation. For a populated local dependency cache use `./mvnw -o -q verify`. RDBMS, H2, Testcontainers PostgreSQL and the PostgreSQL driver are test-scoped; middleware Javalin is optional. ## License diff --git a/pom.xml b/pom.xml index dd2affe..4028e33 100644 --- a/pom.xml +++ b/pom.xml @@ -85,6 +85,11 @@ qqq-backend-module-rdbms test + + com.kingsrook.qqq + qqq-backend-module-postgres + test + org.postgresql postgresql diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitProducer.java b/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitProducer.java index 231dada..bbc5caf 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitProducer.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/DataAnalysisQBitProducer.java @@ -128,6 +128,7 @@ public class DataAnalysisQBitProducer implements QBitMetaDataProducer fields = security.analyzableFields(request.getTableName()).stream() + .filter(f -> request.getFields() == null || request.getFields().contains(f.getName())).toList(); + return (new TableProfiler(backend).profile(request, fields, scope)); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/app/DataAnalysisHome.java b/src/main/java/com/kingsrook/qbits/dataanalysis/app/DataAnalysisHome.java index a4ea48c..b8d0bb8 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/app/DataAnalysisHome.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/app/DataAnalysisHome.java @@ -119,7 +119,7 @@ public class DataAnalysisHome extends AbstractWidgetRenderer { int mine = count(DataAnalysisTables.SAVED, equal("owner_user_id", userId)); int shared = count(DataAnalysisTables.SAVED, equal("shared", true)); - html.append("

Mine: ").append(mine).append(" · Shared (visible to you): ").append(shared).append("

"); + html.append("

Mine: ").append(mine).append(" · Shared (visible to you): ").append(shared).append("

"); if(canRead(DataAnalysisTables.SAVED_STAR)) { CountInput stars = new CountInput(DataAnalysisTables.SAVED).withIncludeDistinctCount(true) @@ -131,9 +131,9 @@ public class DataAnalysisHome extends AbstractWidgetRenderer } if(mine == 0 && shared == 0) { - html.append("

No saved analyses yet: open any table and choose Analyze on a column header.

"); + html.append("

No saved analyses yet. Saving analyses from the Analyze action is planned for a later release.

"); } - html.append("

The Analyze action is planned for phase b; this scaffold currently stores definitions.

"); + html.append("

The Analyze action will be available in a later UI release.

"); link(html, DataAnalysisTables.SAVED); } else @@ -145,7 +145,7 @@ public class DataAnalysisHome extends AbstractWidgetRenderer { int enabled = count(DataAnalysisTables.WATCH, equal("enabled", true)); int firing = count(DataAnalysisTables.WATCH, equal("state", "FIRING")); - html.append("

Enabled: ").append(enabled).append(" · Firing: ").append(firing).append("

"); + html.append("

Enabled: ").append(enabled).append(" · Firing: ").append(firing).append("

"); if(enabled == 0 && firing == 0) { html.append("

No active watches: watch evaluation is planned for phase e.

"); @@ -161,10 +161,10 @@ public class DataAnalysisHome extends AbstractWidgetRenderer { int complete = count(DataAnalysisTables.RUN, recentRuns("COMPLETE", now)); int failed = count(DataAnalysisTables.RUN, recentRuns("FAILED", now)); - html.append("

Complete: ").append(complete).append(" · Failed: ").append(failed).append("

"); + html.append("

Complete: ").append(complete).append(" · Failed: ").append(failed).append("

"); if(complete == 0 && failed == 0) { - html.append("

No completed or failed runs in this period: exact analysis execution is planned for phase b.

"); + html.append("

No completed or failed runs in this period.

"); } link(html, DataAnalysisTables.RUN); } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisBackend.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisBackend.java new file mode 100644 index 0000000..54ec58a --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisBackend.java @@ -0,0 +1,145 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.io.Serializable; +import java.math.BigDecimal; +import java.time.ZoneId; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; + +/******************************************************************************* + ** Primitive port. All operations use the current session with ENFORCE. + *******************************************************************************/ +public interface AnalysisBackend +{ + /******************************************************************************* + ** Date units use the session zone and ISO Monday weeks. + *******************************************************************************/ + enum DateUnit + { + DAY, WEEK, MONTH, YEAR, DAY_OF_WEEK, HOUR_OF_DAY + } + + + + /******************************************************************************* + ** Actual row coverage and deterministic selected population. + *******************************************************************************/ + record Scope(QQueryFilter filter, long total, long rowsScanned, boolean truncated, boolean estimate) + { + } + + + + /******************************************************************************* + ** Blank includes null and whitespace; distinct excludes null. + *******************************************************************************/ + record FieldCounts(long nonNull, long nulls, long blanks, long distinct, QQueryFilter blankFilter, QQueryFilter distinctFilter) + { + } + + + + /******************************************************************************* + ** Group and exact predicate producing its members. + *******************************************************************************/ + record GroupCount(Serializable value, String label, long count, QQueryFilter condition) + { + } + + + + /******************************************************************************* + ** Numeric moments, percentile_cont and sample standard deviation. + *******************************************************************************/ + record NumericSummary(Map metrics, List outliers, long outlierCount, QQueryFilter outlierFilter, List counts) + { + } + + + + /******************************************************************************* + ** Text lengths exclude null but include empty strings. + *******************************************************************************/ + record LengthSummary(Map metrics, List histogram) + { + } + + + + /******************************************************************************* + ** Exact filtered count. + *******************************************************************************/ + long count(String table, QQueryFilter filter) throws QException; + + /******************************************************************************* + ** Select the bounded population used consistently by all operations. + *******************************************************************************/ + Scope scope(String table, QQueryFilter filter, boolean exact) throws QException; + + /******************************************************************************* + ** Batch field counts in groups of twenty. + *******************************************************************************/ + Map fieldCounts(String table, QQueryFilter filter, List fields) throws QException; + + /******************************************************************************* + ** Ordered groups with search on the stored value. + *******************************************************************************/ + List topValues(String table, QQueryFilter filter, String field, int limit, int offset, String search) throws QException; + + /******************************************************************************* + ** Numeric summary and ten extreme-value groups. + *******************************************************************************/ + NumericSummary numericSummary(String table, QQueryFilter filter, String field, List percentiles) throws QException; + + /******************************************************************************* + ** Half-open interior bins, closed final bin, and exterior bins. + *******************************************************************************/ + List numericHistogram(String table, QQueryFilter filter, String field, BigDecimal low, BigDecimal high, int bins) throws QException; + + /******************************************************************************* + ** Date buckets include interior zero intervals. + *******************************************************************************/ + List dateBuckets(String table, QQueryFilter filter, String field, DateUnit unit, ZoneId zone) throws QException; + + /******************************************************************************* + ** Typed minimum and maximum. + *******************************************************************************/ + Map extrema(String table, QQueryFilter filter, String field) throws QException; + + /******************************************************************************* + ** Length moments and histogram. + *******************************************************************************/ + LengthSummary lengthSummary(String table, QQueryFilter filter, String field) throws QException; + + /******************************************************************************* + ** Top shapes over a bounded primary-key prefix. + *******************************************************************************/ + List patterns(String table, QQueryFilter filter, String field, int sampleRows) throws QException; + + /******************************************************************************* + ** Top groups with count greater than one. + *******************************************************************************/ + List duplicates(String table, QQueryFilter filter, List fields, int limit) throws QException; +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisLabels.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisLabels.java new file mode 100644 index 0000000..4372566 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisLabels.java @@ -0,0 +1,179 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.io.Serializable; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.metadata.personalization.TableMetaDataPersonalizerAction; +import com.kingsrook.qqq.backend.core.actions.permissions.FieldSecurityLockHelper; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator; +import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + +/******************************************************************************* + ** Translate through core while enforcing access to every referenced label. + ** RC.5's direct table-PVS translation bypasses permissions, so table values are + ** read with ENFORCE and handed to the translator as an ephemeral enum source. + ** No shared metadata or session is mutated. + *******************************************************************************/ +public final class AnalysisLabels +{ + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private AnalysisLabels() + { + } + + + + /******************************************************************************* + ** Missing or denied labels fall back to the already-readable source id. + *******************************************************************************/ + public static String translate(QFieldMetaData field, Serializable value) throws QException + { + return (translate(field, value, new LinkedHashSet<>())); + } + + + + /******************************************************************************* + ** Resolve nested PVS label fields without invoking core's trusted lookup path. + *******************************************************************************/ + private static String translate(QFieldMetaData field, Serializable value, Set visited) throws QException + { + if(value == null) + { + return (null); + } + String fallback = value.toString(); + QPossibleValueSource source = QContext.getQInstance().getPossibleValueSource(field.getPossibleValueSourceName()); + if(source == null || visited.size() >= 8 || !visited.add(source.getName())) + { + return (fallback); + } + try + { + QPossibleValueTranslator translator = new QPossibleValueTranslator(); + if(source.getType() != QPossibleValueSourceType.TABLE) + { + return (translator.translatePossibleValue(field, value)); + } + QueryInput input = new QueryInput(source.getTableName()); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + QTableMetaData table = TableMetaDataPersonalizerAction.execute(input, true); + QTableMetaData original = QContext.getQInstance().getTable(source.getTableName()); + List labelFields = original.getRecordLabelFields() == null ? List.of() : original.getRecordLabelFields(); + for(String labelField : labelFields) + { + if(!table.getFields().containsKey(labelField) || !FieldSecurityLockHelper.isFieldAccessible(table.getField(labelField))) + { + return (fallback); + } + } + String id = source.getOverrideIdField() == null ? table.getPrimaryKeyField() : source.getOverrideIdField(); + Set selection = new LinkedHashSet<>(labelFields); + selection.add(id); + input.withFilter(DrillFilters.condition(id, QCriteriaOperator.EQUALS, value).withLimit(1)).withFieldNamesToInclude(selection) + .withShouldFetchHeavyFields(true).withShouldGenerateDisplayValues(true); + List records = new QueryAction().execute(input).getRecords(); + if(records.isEmpty()) + { + return (fallback); + } + QRecord record = records.getFirst(); + for(String labelField : labelFields) + { + QFieldMetaData labelMetadata = table.getField(labelField); + if(labelMetadata.getPossibleValueSourceName() != null) + { + record.withDisplayValue(labelField, translate(labelMetadata, record.getValue(labelField), visited)); + } + } + String label = QValueFormatter.formatRecordLabel(table, record); + QPossibleValueSource permitted = new QPossibleValueSource().withName(source.getName()).withType(QPossibleValueSourceType.ENUM) + .withValueFormat(source.getValueFormat()).withValueFields(source.getValueFields()).withEnumValues(List.of(new QPossibleValue<>(value, label))); + return (translator.buildTranslatedPossibleValueList(permitted, List.of(value)).getFirst().getLabel()); + } + catch(QPermissionDeniedException e) + { + return (fallback); + } + finally + { + visited.remove(source.getName()); + } + } + + + + /******************************************************************************* + ** Cache identity includes personalized field and related-label policies, so a + ** host changing field/record locks cannot release older translated labels. + *******************************************************************************/ + public static java.util.Map policy(String tableName) throws QException + { + java.util.Map result = new java.util.TreeMap<>(); + java.util.ArrayDeque pending = new java.util.ArrayDeque<>(); + pending.add(tableName); + while(!pending.isEmpty()) + { + String name = pending.removeFirst(); + if(result.containsKey(name)) + { + continue; + } + QueryInput input = new QueryInput(name); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + QTableMetaData table = TableMetaDataPersonalizerAction.execute(input, true); + java.util.Map entry = new java.util.TreeMap<>(); + entry.put("fields", table.getFields()); + entry.put("locks", table.getRecordSecurityLocks()); + entry.put("labelFields", table.getRecordLabelFields()); + entry.put("labelFormat", table.getRecordLabelFormat()); + entry.put("permissions", table.getPermissionRules()); + result.put(name, entry); + for(QFieldMetaData field : table.getFields().values()) + { + QPossibleValueSource source = QContext.getQInstance().getPossibleValueSource(field.getPossibleValueSourceName()); + if(source != null && source.getType() == QPossibleValueSourceType.TABLE) + { + pending.add(source.getTableName()); + } + } + } + return (result); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisSecurity.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisSecurity.java new file mode 100644 index 0000000..dbf399a --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisSecurity.java @@ -0,0 +1,94 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.util.List; +import com.kingsrook.qbits.dataanalysis.permissions.DataAnalysisPermissions; +import com.kingsrook.qqq.backend.core.actions.metadata.personalization.TableMetaDataPersonalizerAction; +import com.kingsrook.qqq.backend.core.actions.permissions.FieldSecurityLockHelper; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; +import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + +/******************************************************************************* + ** Fail closed before any source-table computation, including cache retrieval. + *******************************************************************************/ +public class AnalysisSecurity +{ + private final EffectiveConfigResolver configs; + + /******************************************************************************* + ** Use the runtime's configuration policy. + *******************************************************************************/ + public AnalysisSecurity(EffectiveConfigResolver configs) + { + this.configs = configs; + } + + + + /******************************************************************************* + ** Explicit enforcement is independent of the instance's default switch. + *******************************************************************************/ + public QTableMetaData table(String name) throws QException + { + QTableMetaData table = QContext.getQInstance().getTable(name); + QueryInput input = new QueryInput(name); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + if(table == null || !configs.resolve(name).getEnabled() || !DataAnalysisPermissions.canAnalyze(table) + || !PermissionsHelper.hasTablePermission(input, name, TablePermissionSubType.READ)) + { + throw new QPermissionDeniedException("Analysis is unavailable for this table."); + } + return (TableMetaDataPersonalizerAction.execute(input, true)); + } + + + + /******************************************************************************* + ** Use personalized fields and locks; excluded fields never appear in capabilities. + *******************************************************************************/ + public List analyzableFields(String name) throws QException + { + QTableMetaData table = table(name); + EffectiveConfig c = configs.resolve(name); + return (table.getFields().values().stream().filter(FieldSecurityLockHelper::isFieldAccessible) + .filter(f -> c.getAnalyzableFields().isEmpty() || c.getAnalyzableFields().contains(f.getName())) + .filter(f -> !c.getExcludedFields().contains(f.getName())).toList()); + } + + + + /******************************************************************************* + ** A common denial does not disclose a hidden field's existence or values. + *******************************************************************************/ + public QFieldMetaData field(String table, String field) throws QException + { + return (analyzableFields(table).stream().filter(f -> f.getName().equals(field)).findFirst() + .orElseThrow(() -> new QPermissionDeniedException("Analysis is unavailable for this field."))); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ColumnProfiler.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ColumnProfiler.java new file mode 100644 index 0000000..7710dcf --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ColumnProfiler.java @@ -0,0 +1,219 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.io.Serializable; +import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.dataanalysis.result.ChartKind; +import com.kingsrook.qbits.dataanalysis.result.ColumnProfile; +import com.kingsrook.qbits.dataanalysis.result.ColumnProfileRequest; +import com.kingsrook.qbits.dataanalysis.result.DateProfile; +import com.kingsrook.qbits.dataanalysis.result.NumericProfile; +import com.kingsrook.qbits.dataanalysis.result.TextProfile; +import com.kingsrook.qbits.dataanalysis.result.ValueCount; +import com.kingsrook.qbits.dataanalysis.result.ValueType; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + +/******************************************************************************* + ** Column profiles depend only on primitive backend operations and typed results. + *******************************************************************************/ +public class ColumnProfiler +{ + private final AnalysisBackend backend; + + /******************************************************************************* + ** The backend can be replaced when core's new operators ship. + *******************************************************************************/ + public ColumnProfiler(AnalysisBackend backend) + { + this.backend = backend; + } + + + + /******************************************************************************* + ** Produce a guarded profile for a validated field and selected population. + *******************************************************************************/ + public ColumnProfile profile(ColumnProfileRequest request, QFieldMetaData field, AnalysisBackend.Scope scope, EffectiveConfig config) throws QException + { + String table = request.getTableName(); + String name = field.getName(); + QQueryFilter filter = scope.filter(); + long rows = backend.count(table, filter); + AnalysisBackend.FieldCounts counts = backend.fieldCounts(table, filter, List.of(name)).get(name); + ColumnProfile result = new ColumnProfile().withTableName(table).withFieldName(name).withFieldLabel(field.getLabel()).withFieldType(field.getType().name()) + .withFilter(DrillFilters.restoreTypes(request.getTableName(), request.getFilter())).withEstimate(scope.estimate()).withSampleRows(scope.estimate() ? rows : 0L) + .withRowsScanned(scope.rowsScanned()).withTruncated(scope.truncated()).withAsOf(Instant.now()); + result.getCounts().add(ProfileSupport.count("rows", rows, table, filter, null)); + result.getCounts().add(ProfileSupport.count("nonNull", counts.nonNull(), table, filter, DrillFilters.nonNull(name, field.getType().isStringLike()))); + result.getCounts().add(ProfileSupport.count("nulls", counts.nulls(), table, filter, DrillFilters.nulls(name, field.getType().isStringLike()))); + if(field.getType().isStringLike()) + { + result.getCounts().add(ProfileSupport.count("blanks", counts.blanks(), table, filter, counts.blankFilter())); + } + result.getCounts().add(ProfileSupport.count("distinct", counts.distinct(), table, filter, counts.distinctFilter())); + result.getCounts().add(ProfileSupport.metric("uniqueness", ProfileSupport.share(counts.distinct(), counts.nonNull()), ValueType.PERCENT)); + result.withTopValues(ProfileSupport.values(table, filter, field, backend.topValues(table, filter, name, 20, 0, null), rows)); + QFieldType type = field.getType(); + if(type == QFieldType.INTEGER || type == QFieldType.LONG || type == QFieldType.DECIMAL) + { + numeric(result, filter, rows); + } + else if(type == QFieldType.DATE || type == QFieldType.DATE_TIME) + { + dates(result, filter, rows, type); + } + else if(type.isStringLike()) + { + AnalysisBackend.LengthSummary summary = backend.lengthSummary(table, filter, name); + TextProfile text = new TextProfile().withHistogram(ProfileSupport.values(table, filter, null, summary.histogram(), rows)) + .withPatterns(ProfileSupport.values(table, filter, null, backend.patterns(table, filter, name, 10_000), Math.min(rows, 10_000))) + .withSampleRows(Math.min(rows, 10_000)); + summary.metrics().forEach((key, value) -> text.getMetrics().add(ProfileSupport.metric(key, value, ValueType.DECIMAL))); + result.withText(text); + result.getWarnings().add("Patterns use the first " + Math.min(rows, 10_000) + " rows by primary key."); + } + else if(type == QFieldType.BOOLEAN) + { + result.withBooleans(new java.util.ArrayList<>()); + for(boolean value : List.of(true, false)) + { + QQueryFilter condition = DrillFilters.value(name, value, false); + result.getBooleans().add(ProfileSupport.count(Boolean.toString(value), backend.count(table, DrillFilters.and(filter, condition)), table, filter, condition)); + } + result.getBooleans().add(ProfileSupport.count("nulls", counts.nulls(), table, filter, DrillFilters.nulls(name, false))); + } + new MinGroupSizeGuard(config.getMinGroupSize()).column(result, rows); + if(field.getPossibleValueSourceName() != null || result.getNumeric() == null && result.getDates() == null) + { + result.withChart(ProfileSupport.chart(ChartKind.BAR, ChartKind.PARETO, name, result.getTopValues())); + } + else if(result.getNumeric() != null) + { + result.withChart(ProfileSupport.chart(ChartKind.HISTOGRAM, ChartKind.BOX, name, result.getNumeric().getHistogram())); + } + else + { + result.withChart(ProfileSupport.chart(ChartKind.TIME_SERIES, ChartKind.BAR, name, result.getDates().getBuckets())); + } + if(rows == 0) + { + result.getWarnings().add("No readable records match this analysis scope."); + } + else if(counts.nonNull() == 0) + { + result.getWarnings().add("All values in this analysis scope are null."); + } + if(scope.estimate()) + { + result.getWarnings().add("Estimate: first " + rows + " rows by primary key; not a random sample."); + } + if(scope.truncated()) + { + result.getWarnings().add("Partial result: max_rows reached; counts and drills cover only the selected prefix."); + } + return (result); + } + + + + /******************************************************************************* + ** Numeric moments, twenty interior bins, exterior bins and Tukey outliers. + *******************************************************************************/ + private void numeric(ColumnProfile result, QQueryFilter filter, long rows) throws QException + { + String table = result.getTableName(); + String field = result.getFieldName(); + AnalysisBackend.NumericSummary summary = backend.numericSummary(table, filter, field, + List.of("0.01", "0.05", "0.25", "0.50", "0.75", "0.95", "0.99").stream().map(BigDecimal::new).toList()); + NumericProfile numeric = new NumericProfile(); + summary.metrics().forEach((key, value) -> numeric.getMetrics().add(ProfileSupport.metric(key, value, ValueType.DECIMAL))); + for(AnalysisBackend.GroupCount count : summary.counts()) + { + numeric.getMetrics().add(ProfileSupport.count(count.label(), count.count(), table, filter, count.condition())); + } + numeric.getMetrics().add(ProfileSupport.count("outliers", summary.outlierCount(), table, filter, summary.outlierFilter())); + numeric.withHistogram(ProfileSupport.values(table, filter, null, + backend.numericHistogram(table, filter, field, summary.metrics().get("p1"), summary.metrics().get("p99"), 20), rows)); + numeric.withOutliers(ProfileSupport.values(table, filter, null, summary.outliers(), rows)); + result.withNumeric(numeric); + } + + + + /******************************************************************************* + ** Date extrema, range, calendar volume, gaps and cyclic time patterns. + *******************************************************************************/ + private void dates(ColumnProfile result, QQueryFilter filter, long rows, QFieldType type) throws QException + { + String table = result.getTableName(); + String field = result.getFieldName(); + ZoneId zone = ValueUtils.getSessionOrInstanceZoneId(); + Map extrema = backend.extrema(table, filter, field); + DateProfile dates = new DateProfile(); + ValueType valueType = type == QFieldType.DATE ? ValueType.DATE : ValueType.DATE_TIME; + extrema.forEach((key, value) -> dates.getMetrics().add(ProfileSupport.metric(key, value, valueType))); + if(extrema.get("min") != null && extrema.get("max") != null) + { + LocalDate first = ProfileSupport.day(extrema.get("min"), zone); + LocalDate last = ProfileSupport.day(extrema.get("max"), zone); + AnalysisBackend.DateUnit unit = autoBucket(first, last); + dates.withUnit(unit.name()); + dates.getMetrics().add(ProfileSupport.metric("range", ChronoUnit.DAYS.between(first, last), ValueType.DURATION)); + List buckets = ProfileSupport.values(table, filter, null, backend.dateBuckets(table, filter, field, unit, zone), rows); + dates.withBuckets(buckets).withGaps(buckets.stream().filter(b -> b.getCount() == 0).toList()); + if(type == QFieldType.DATE_TIME) + { + dates.withDayOfWeek(ProfileSupport.values(table, filter, null, backend.dateBuckets(table, filter, field, AnalysisBackend.DateUnit.DAY_OF_WEEK, zone), rows)) + .withHourOfDay(ProfileSupport.values(table, filter, null, backend.dateBuckets(table, filter, field, AnalysisBackend.DateUnit.HOUR_OF_DAY, zone), rows)); + } + } + result.withDates(dates); + } + + + + /******************************************************************************* + ** Inclusive calendar boundaries: 92 days, two years and ten years. + *******************************************************************************/ + public static AnalysisBackend.DateUnit autoBucket(LocalDate first, LocalDate last) + { + if(ChronoUnit.DAYS.between(first, last) <= 92) + { + return (AnalysisBackend.DateUnit.DAY); + } + if(!last.isAfter(first.plusYears(2))) + { + return (AnalysisBackend.DateUnit.WEEK); + } + return (!last.isAfter(first.plusYears(10)) ? AnalysisBackend.DateUnit.MONTH : AnalysisBackend.DateUnit.YEAR); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/DistinctValues.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/DistinctValues.java new file mode 100644 index 0000000..4ee6dce --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/DistinctValues.java @@ -0,0 +1,74 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.time.Instant; +import java.util.List; +import com.kingsrook.qbits.dataanalysis.result.DistinctValuesPage; +import com.kingsrook.qbits.dataanalysis.result.DistinctValuesRequest; +import com.kingsrook.qbits.dataanalysis.result.ValueCount; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; + +/******************************************************************************* + ** Distinct-value pages apply privacy before pagination, never page rare groups. + *******************************************************************************/ +public class DistinctValues +{ + private final AnalysisBackend backend; + + /******************************************************************************* + ** Bind primitive operations. + *******************************************************************************/ + public DistinctValues(AnalysisBackend backend) + { + this.backend = backend; + } + + + + /******************************************************************************* + ** Request validation caps the page size at 500 and rejects overflow. + *******************************************************************************/ + public DistinctValuesPage page(DistinctValuesRequest request, QFieldMetaData field, EffectiveConfig config) throws QException + { + if(request.getPage() == null || request.getPage() < 0 || request.getPageSize() == null || request.getPageSize() < 1 || request.getPageSize() > 500 + || (long) request.getPage() * request.getPageSize() > Integer.MAX_VALUE - 501) + { + throw new QException("Invalid distinct-values page; pageSize must be between 1 and 500."); + } + AnalysisBackend.Scope scope = backend.scope(request.getTableName(), request.getFilter(), true); + int offset = request.getPage() * request.getPageSize(); + boolean wholeListing = config.getMinGroupSize() > 0 || request.getSearch() != null && !request.getSearch().isBlank(); + int fetch = wholeListing ? Math.max(1, config.getMaxRows()) : request.getPageSize() + 1; + List values = ProfileSupport.values(request.getTableName(), scope.filter(), field, + backend.topValues(request.getTableName(), scope.filter(), field.getName(), fetch, wholeListing ? 0 : offset, request.getSearch()), scope.rowsScanned()); + long totalDistinct = wholeListing ? values.stream().filter(v -> v.getValue() != null).count() + : backend.fieldCounts(request.getTableName(), scope.filter(), List.of(field.getName())).get(field.getName()).distinct(); + values = new MinGroupSizeGuard(config.getMinGroupSize()).groups(values); + int start = wholeListing ? Math.min(offset, values.size()) : 0; + boolean more = values.size() > start + request.getPageSize(); + return (new DistinctValuesPage().withTableName(request.getTableName()).withFieldName(field.getName()).withFilter(DrillFilters.copy(request.getFilter())) + .withPage(request.getPage()).withPageSize(request.getPageSize()).withHasMore(more).withValues(values.subList(start, Math.min(values.size(), start + request.getPageSize()))) + .withTotalDistinct(totalDistinct) + .withRowsScanned(scope.rowsScanned()).withTruncated(scope.truncated()).withAsOf(Instant.now())); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/DrillFilters.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/DrillFilters.java new file mode 100644 index 0000000..d860959 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/DrillFilters.java @@ -0,0 +1,239 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.io.Serializable; +import java.time.LocalDate; +import java.time.ZoneId; +import java.util.List; +import com.kingsrook.qbits.dataanalysis.result.Drill; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; + +/******************************************************************************* + ** Immutable composition of the request filter and the producing condition. + *******************************************************************************/ +public final class DrillFilters +{ + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private DrillFilters() + { + } + + + + /******************************************************************************* + ** Copy criteria only; pagination and ordering are not analysis predicates. + *******************************************************************************/ + public static QQueryFilter copy(QQueryFilter filter) + { + QQueryFilter result = filter == null ? new QQueryFilter() : filter.clone(); + result.setLimit(null); + result.setSkip(null); + result.setOrderBys(null); + if(result.getSubFilters() != null) + { + result.setSubFilters(result.getSubFilters().stream().map(DrillFilters::copy).toList()); + } + return (result); + } + + + + /******************************************************************************* + ** Preserve OR semantics of both operands by nesting them under AND. + *******************************************************************************/ + public static QQueryFilter and(QQueryFilter original, QQueryFilter condition) + { + return (new QQueryFilter().withSubFilter(copy(original)).withSubFilter(copy(condition))); + } + + + + /******************************************************************************* + ** A drill always owns its filters. + *******************************************************************************/ + public static Drill drill(String table, QQueryFilter original, QQueryFilter condition) + { + return (new Drill().withTableName(table).withFilter(and(original, condition))); + } + + + + /******************************************************************************* + ** One criterion, including predicates without operands. + *******************************************************************************/ + public static QQueryFilter condition(String field, QCriteriaOperator operator, Serializable... values) + { + return (new QQueryFilter().withCriteria(field, operator, values)); + } + + + + /******************************************************************************* + ** RC.5 IS_BLANK includes empty strings. Applying it to StringLength + ** distinguishes null from length zero on both SQL and memory backends. + *******************************************************************************/ + public static QQueryFilter nulls(String field, boolean string) + { + QQueryFilter result = condition(field, QCriteriaOperator.IS_BLANK); + if(string) + { + result.withCriteria(new com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria(field, QCriteriaOperator.NOT_EQUALS_OR_IS_NULL, 0) + .withFieldFunction(new com.kingsrook.qqq.backend.core.model.metadata.fields.functions.FieldFunction().withFieldName(field) + .withFunctionTypeIdentifier(com.kingsrook.qqq.backend.core.model.metadata.fields.functions.implementations.StringLengthFunction.IDENTIFIER))); + } + return (result); + } + + + + /******************************************************************************* + ** Non-null includes an empty string, unlike core IS_NOT_BLANK. + *******************************************************************************/ + public static QQueryFilter nonNull(String field, boolean string) + { + QQueryFilter result = condition(field, QCriteriaOperator.IS_NOT_BLANK); + if(string) + { + result.withBooleanOperator(QQueryFilter.BooleanOperator.OR).withCriteria(field, QCriteriaOperator.EQUALS, ""); + } + return (result); + } + + + + /******************************************************************************* + ** A group equality uses a null-only filter for a null group. + *******************************************************************************/ + public static QQueryFilter value(String field, Serializable value, boolean string) + { + return (value == null ? nulls(field, string) : condition(field, QCriteriaOperator.EQUALS, value)); + } + + + + /******************************************************************************* + ** Half-open bounds, except that the final interior histogram bin is closed. + ** Explicit non-null avoids RC.5 memory treating null as less than a value. + *******************************************************************************/ + public static QQueryFilter range(String field, Serializable low, Serializable high, boolean closed) + { + QQueryFilter result = condition(field, QCriteriaOperator.IS_NOT_BLANK); + if(low != null) + { + result.withCriteria(field, QCriteriaOperator.GREATER_THAN_OR_EQUALS, low); + } + if(high != null) + { + result.withCriteria(field, closed ? QCriteriaOperator.LESS_THAN_OR_EQUALS : QCriteriaOperator.LESS_THAN, high); + } + return (result); + } + + + + /******************************************************************************* + ** Local midnight boundaries preserve DST and the user's time zone. + *******************************************************************************/ + public static QQueryFilter dateRange(String field, LocalDate low, LocalDate high, boolean dateTime, ZoneId zone) + { + return (range(field, dateTime ? low.atStartOfDay(zone).toInstant() : low, + dateTime ? high.atStartOfDay(zone).toInstant() : high, false)); + } + + + + /******************************************************************************* + ** Strict Tukey fences, excluding nulls on every backend. + *******************************************************************************/ + public static QQueryFilter outliers(String field, Serializable low, Serializable high) + { + QQueryFilter outside = condition(field, QCriteriaOperator.LESS_THAN, low) + .withBooleanOperator(QQueryFilter.BooleanOperator.OR).withCriteria(field, QCriteriaOperator.GREATER_THAN, high); + return (and(condition(field, QCriteriaOperator.IS_NOT_BLANK), outside)); + } + + + + /******************************************************************************* + ** Exact membership for streamed shapes, lengths and duplicate tuples. + *******************************************************************************/ + public static QQueryFilter in(String field, List values) + { + return (values.isEmpty() ? condition(field, QCriteriaOperator.FALSE) : new QQueryFilter().withCriteria(field, QCriteriaOperator.IN, values)); + } + + + + /******************************************************************************* + ** JSON has no temporal scalar types. Restore them from current metadata before + ** a cached drill is handed back to core, without mutating the stored filter. + *******************************************************************************/ + public static QQueryFilter restoreTypes(String table, QQueryFilter filter) + { + QQueryFilter result = copy(filter); + var instance = com.kingsrook.qqq.backend.core.context.QContext.getQInstance(); + if(instance == null || instance.getTable(table) == null) + { + return (result); + } + if(result.getCriteria() != null) + { + for(var criterion : result.getCriteria()) + { + var field = instance.getTable(table).getFields().get(criterion.getFieldName()); + if(field != null && criterion.getValues() != null) + { + var type = field.getType(); + if(type == com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType.DATE + || type == com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType.DATE_TIME + || type == com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType.DECIMAL + || criterion.getOperator() == QCriteriaOperator.EQUALS || criterion.getOperator() == QCriteriaOperator.IN) + { + criterion.setValues(criterion.getValues().stream().map(v -> restoreValue(type, v)).toList()); + } + } + } + } + if(result.getSubFilters() != null) + { + result.setSubFilters(result.getSubFilters().stream().map(f -> restoreTypes(table, f)).toList()); + } + return (result); + } + + + + /******************************************************************************* + ** String decimal operands preserve exact scale and precision across JSON round trips. + *******************************************************************************/ + private static Serializable restoreValue(com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType type, Serializable value) + { + if(type == com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType.DECIMAL) + { + return (value == null ? null : value.toString()); + } + return (com.kingsrook.qqq.backend.core.utils.ValueUtils.getValueAsFieldType(type, value)); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/DuplicateFinder.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/DuplicateFinder.java new file mode 100644 index 0000000..e8fe3e1 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/DuplicateFinder.java @@ -0,0 +1,56 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.time.Instant; +import com.kingsrook.qbits.dataanalysis.result.DuplicatesRequest; +import com.kingsrook.qbits.dataanalysis.result.DuplicatesResult; +import com.kingsrook.qqq.backend.core.exceptions.QException; + +/******************************************************************************* + ** Top fifty duplicate tuples with permission-filtered membership drills. + *******************************************************************************/ +public class DuplicateFinder +{ + private final AnalysisBackend backend; + + /******************************************************************************* + ** Bind primitive operations. + *******************************************************************************/ + public DuplicateFinder(AnalysisBackend backend) + { + this.backend = backend; + } + + + + /******************************************************************************* + ** Guard tuples before exposing or serializing them. + *******************************************************************************/ + public DuplicatesResult find(DuplicatesRequest request, EffectiveConfig config) throws QException + { + AnalysisBackend.Scope scope = backend.scope(request.getTableName(), request.getFilter(), true); + return (new DuplicatesResult().withTableName(request.getTableName()).withFields(request.getFields()).withFilter(DrillFilters.copy(request.getFilter())) + .withRowsScanned(scope.rowsScanned()).withTruncated(scope.truncated()).withAsOf(Instant.now()) + .withGroups(new MinGroupSizeGuard(config.getMinGroupSize()).groups(ProfileSupport.values(request.getTableName(), scope.filter(), null, + backend.duplicates(request.getTableName(), scope.filter(), request.getFields(), 50), scope.rowsScanned())))); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/EffectiveConfig.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/EffectiveConfig.java new file mode 100644 index 0000000..d7a0b74 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/EffectiveConfig.java @@ -0,0 +1,211 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + + +import java.io.Serializable; +import java.util.Set; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + + +/******************************************************************************* + ** Typed EffectiveConfig contract; independent of any presentation framework. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class EffectiveConfig implements Serializable +{ + private Boolean enabled = true; + private Integer maxRows; + private Integer sampleThresholdRows; + private Integer sampleRows; + private Integer minGroupSize; + private Integer cacheTtlMinutes; + private Set analyzableFields = Set.of(); + private Set excludedFields = Set.of(); + + + + /******************************************************************************* + ** Getter for enabled. + *******************************************************************************/ + public Boolean getEnabled() + { + return (enabled); + } + + + + /******************************************************************************* + ** Fluent setter for enabled. + *******************************************************************************/ + public EffectiveConfig withEnabled(Boolean enabled) + { + this.enabled = enabled; + return (this); + } + + + + /******************************************************************************* + ** Getter for maxRows. + *******************************************************************************/ + public Integer getMaxRows() + { + return (maxRows); + } + + + + /******************************************************************************* + ** Fluent setter for maxRows. + *******************************************************************************/ + public EffectiveConfig withMaxRows(Integer maxRows) + { + this.maxRows = maxRows; + return (this); + } + + + + /******************************************************************************* + ** Getter for sampleThresholdRows. + *******************************************************************************/ + public Integer getSampleThresholdRows() + { + return (sampleThresholdRows); + } + + + + /******************************************************************************* + ** Fluent setter for sampleThresholdRows. + *******************************************************************************/ + public EffectiveConfig withSampleThresholdRows(Integer sampleThresholdRows) + { + this.sampleThresholdRows = sampleThresholdRows; + return (this); + } + + + + /******************************************************************************* + ** Getter for sampleRows. + *******************************************************************************/ + public Integer getSampleRows() + { + return (sampleRows); + } + + + + /******************************************************************************* + ** Fluent setter for sampleRows. + *******************************************************************************/ + public EffectiveConfig withSampleRows(Integer sampleRows) + { + this.sampleRows = sampleRows; + return (this); + } + + + + /******************************************************************************* + ** Getter for minGroupSize. + *******************************************************************************/ + public Integer getMinGroupSize() + { + return (minGroupSize); + } + + + + /******************************************************************************* + ** Fluent setter for minGroupSize. + *******************************************************************************/ + public EffectiveConfig withMinGroupSize(Integer minGroupSize) + { + this.minGroupSize = minGroupSize; + return (this); + } + + + + /******************************************************************************* + ** Getter for cacheTtlMinutes. + *******************************************************************************/ + public Integer getCacheTtlMinutes() + { + return (cacheTtlMinutes); + } + + + + /******************************************************************************* + ** Fluent setter for cacheTtlMinutes. + *******************************************************************************/ + public EffectiveConfig withCacheTtlMinutes(Integer cacheTtlMinutes) + { + this.cacheTtlMinutes = cacheTtlMinutes; + return (this); + } + + + + /******************************************************************************* + ** Getter for analyzableFields. + *******************************************************************************/ + public Set getAnalyzableFields() + { + return (analyzableFields); + } + + + + /******************************************************************************* + ** Fluent setter for analyzableFields. + *******************************************************************************/ + public EffectiveConfig withAnalyzableFields(Set analyzableFields) + { + this.analyzableFields = analyzableFields; + return (this); + } + + + + /******************************************************************************* + ** Getter for excludedFields. + *******************************************************************************/ + public Set getExcludedFields() + { + return (excludedFields); + } + + + + /******************************************************************************* + ** Fluent setter for excludedFields. + *******************************************************************************/ + public EffectiveConfig withExcludedFields(Set excludedFields) + { + this.excludedFields = excludedFields; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/EffectiveConfigResolver.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/EffectiveConfigResolver.java new file mode 100644 index 0000000..e589d57 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/EffectiveConfigResolver.java @@ -0,0 +1,171 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; +import com.kingsrook.qbits.dataanalysis.DataAnalysisQBitConfig; +import com.kingsrook.qbits.dataanalysis.TableAnalysisDefaults; +import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession; + +/******************************************************************************* + ** Instance-owned configuration resolver; only the settings lookup is privileged. + *******************************************************************************/ +public class EffectiveConfigResolver +{ + private final DataAnalysisQBitConfig defaults; + private final ConcurrentHashMap cache = new ConcurrentHashMap<>(); + + /******************************************************************************* + ** A 60-second configuration snapshot, never a user-data cache. + *******************************************************************************/ + private record CachedConfig(long expires, EffectiveConfig config) + { + } + + + + /******************************************************************************* + ** Bind code defaults to one host runtime. + *******************************************************************************/ + public EffectiveConfigResolver(DataAnalysisQBitConfig defaults) + { + this.defaults = defaults; + } + + + + /******************************************************************************* + ** Return a detached snapshot so callers cannot change memoized policy. + *******************************************************************************/ + public EffectiveConfig resolve(String table) throws QException + { + CachedConfig cached = cache.get(table); + if(cached == null || cached.expires() <= System.currentTimeMillis()) + { + cached = new CachedConfig(System.currentTimeMillis() + 60_000, load(table)); + cache.put(table, cached); + } + EffectiveConfig c = cached.config(); + return (new EffectiveConfig().withEnabled(c.getEnabled()).withMaxRows(c.getMaxRows()).withSampleRows(c.getSampleRows()) + .withSampleThresholdRows(c.getSampleThresholdRows()).withMinGroupSize(c.getMinGroupSize()).withCacheTtlMinutes(c.getCacheTtlMinutes()) + .withAnalyzableFields(Set.copyOf(c.getAnalyzableFields())).withExcludedFields(Set.copyOf(c.getExcludedFields()))); + } + + + + /******************************************************************************* + ** Settings editors may invalidate immediately instead of waiting for expiry. + *******************************************************************************/ + public void invalidate() + { + cache.clear(); + } + + + + /******************************************************************************* + ** Runtime row overrides per-table code defaults, then global defaults. + *******************************************************************************/ + private EffectiveConfig load(String table) throws QException + { + TableAnalysisDefaults d = defaults.getTableDefaults().getOrDefault(table, new TableAnalysisDefaults()); + EffectiveConfig c = new EffectiveConfig().withEnabled(!Boolean.FALSE.equals(d.getEnabled())) + .withMaxRows(or(d.getMaxRows(), defaults.getDefaultMaxRows())).withSampleRows(or(d.getSampleRows(), defaults.getDefaultSampleRows())) + .withSampleThresholdRows(or(d.getSampleThresholdRows(), defaults.getDefaultSampleThresholdRows())) + .withMinGroupSize(or(d.getMinGroupSize(), defaults.getDefaultMinGroupSize())).withCacheTtlMinutes(or(d.getCacheTtlMinutes(), defaults.getDefaultCacheTtlMinutes())) + .withAnalyzableFields(d.getAnalyzableFields() == null ? Set.of() : Set.copyOf(d.getAnalyzableFields())) + .withExcludedFields(d.getExcludedFields() == null ? Set.of() : Set.copyOf(d.getExcludedFields())); + QSession original = QContext.getQSession(); + List rows; + try + { + QContext.setQSession(new QSystemUserSession()); + QueryInput input = new QueryInput(DataAnalysisTables.TABLE_CONFIG) + .withFilter(DrillFilters.condition("table_name", QCriteriaOperator.EQUALS, table).withLimit(1)) + .withFieldNamesToInclude(Set.of("enabled", "max_rows", "sample_rows", "sample_threshold_rows", "min_group_size", "cache_ttl_minutes", "analyzable_fields", "excluded_fields")) + .withShouldFetchHeavyFields(true); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + rows = new QueryAction().execute(input).getRecords(); + } + finally + { + QContext.setQSession(original); + } + if(!rows.isEmpty()) + { + QRecord r = rows.getFirst(); + if(r.getValue("enabled") != null) + { + c.withEnabled(r.getValueBoolean("enabled")); + } + c.withMaxRows(or(r.getValueInteger("max_rows"), c.getMaxRows())).withSampleRows(or(r.getValueInteger("sample_rows"), c.getSampleRows())) + .withSampleThresholdRows(or(r.getValueInteger("sample_threshold_rows"), c.getSampleThresholdRows())) + .withMinGroupSize(or(r.getValueInteger("min_group_size"), c.getMinGroupSize())).withCacheTtlMinutes(or(r.getValueInteger("cache_ttl_minutes"), c.getCacheTtlMinutes())); + if(r.getValue("analyzable_fields") != null) + { + c.withAnalyzableFields(csv(r.getValueString("analyzable_fields"))); + } + if(r.getValue("excluded_fields") != null) + { + c.withExcludedFields(csv(r.getValueString("excluded_fields"))); + } + } + c.withEnabled(c.getEnabled() && (defaults.getIncludeTables().isEmpty() || defaults.getIncludeTables().contains(table)) && !defaults.getExcludeTables().contains(table)); + if(List.of(c.getMaxRows(), c.getSampleRows(), c.getSampleThresholdRows(), c.getMinGroupSize(), c.getCacheTtlMinutes()).stream().anyMatch(n -> n < 0)) + { + throw new QException("Analysis settings contain an invalid limit."); + } + return (c); + } + + + + /******************************************************************************* + ** Null inherits, zero is a real configured limit. + *******************************************************************************/ + private static Integer or(Integer value, Integer fallback) + { + return (value == null ? fallback : value); + } + + + + /******************************************************************************* + ** Blank lists mean all readable fields or no exclusions. + *******************************************************************************/ + private static Set csv(String text) + { + return (Arrays.stream(text.split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toSet())); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MinGroupSizeGuard.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MinGroupSizeGuard.java new file mode 100644 index 0000000..c9a82a2 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MinGroupSizeGuard.java @@ -0,0 +1,146 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qbits.dataanalysis.result.ColumnProfile; +import com.kingsrook.qbits.dataanalysis.result.Metric; +import com.kingsrook.qbits.dataanalysis.result.ValueCount; + +/******************************************************************************* + ** Privacy transformation before chart construction, serialization and caching. + *******************************************************************************/ +public class MinGroupSizeGuard +{ + private final int minimum; + + /******************************************************************************* + ** Zero disables suppression. + *******************************************************************************/ + public MinGroupSizeGuard(int minimum) + { + this.minimum = minimum; + } + + + + /******************************************************************************* + ** Never carry original values or drills into a merged group. + *******************************************************************************/ + public List groups(List groups) + { + if(minimum == 0) + { + return (groups); + } + List result = new ArrayList<>(); + long count = 0; + int hidden = 0; + java.math.BigDecimal share = java.math.BigDecimal.ZERO; + for(ValueCount group : groups) + { + if(group.getCount() < minimum) + { + hidden++; + count += group.getCount(); + share = share.add(group.getShare()); + } + else + { + result.add(group); + } + } + if(hidden > 0) + { + result.add(new ValueCount().withLabel("Other (" + hidden + " groups below " + minimum + ")").withCount(count).withShare(share)); + } + return (result); + } + + + + /******************************************************************************* + ** Suppressed scalar values are removed, not just tagged as hidden. + *******************************************************************************/ + public void metrics(List metrics, long rows) + { + if(rows >= minimum) + { + return; + } + for(Metric metric : metrics) + { + String key = metric.getKey(); + if(key.startsWith("p") || key.startsWith("min") || key.startsWith("max") || key.startsWith("mean") || key.equals("stddev") || key.equals("range")) + { + metric.withValue(null).withFormatted(null).withDrill(null).withSuppressed(true) + .withSuppressedReason("Hidden: fewer than " + minimum + " records"); + } + } + } + + + + /******************************************************************************* + ** Remove extreme values and their drills whenever privacy is enabled. + *******************************************************************************/ + public void column(ColumnProfile profile, long rows) + { + profile.withTopValues(groups(profile.getTopValues())); + if(minimum > 0) + { + profile.getCounts().stream().filter(m -> m.getKey().equals("distinct")).forEach(m -> m.withDrill(null)); + if(profile.getBooleans() != null) + { + List booleanGroups = profile.getBooleans().stream().map(m -> new ValueCount().withValue(m.getKey()).withLabel(m.getLabel()) + .withCount(((Number) m.getValue()).longValue()).withShare(ProfileSupport.share(((Number) m.getValue()).longValue(), rows)).withDrill(m.getDrill())).toList(); + profile.withBooleans(groups(booleanGroups).stream().map(v -> ProfileSupport.metric(v.getValue() == null ? "other" : v.getValue().toString(), v.getCount(), + com.kingsrook.qbits.dataanalysis.result.ValueType.COUNT).withLabel(v.getLabel()).withDrill(v.getDrill())).toList()); + } + } + if(profile.getNumeric() != null) + { + metrics(profile.getNumeric().getMetrics(), rows); + profile.getNumeric().withHistogram(groups(profile.getNumeric().getHistogram())); + if(minimum > 0) + { + profile.getNumeric().withOutliers(List.of()); + profile.getNumeric().getMetrics().stream().filter(m -> m.getKey().equals("outliers")).forEach(m -> m.withDrill(null)); + } + } + if(profile.getDates() != null) + { + metrics(profile.getDates().getMetrics(), rows); + profile.getDates().withBuckets(groups(profile.getDates().getBuckets())).withDayOfWeek(groups(profile.getDates().getDayOfWeek())) + .withHourOfDay(groups(profile.getDates().getHourOfDay())); + if(minimum > 0) + { + profile.getDates().withGaps(List.of()); + } + } + if(profile.getText() != null) + { + metrics(profile.getText().getMetrics(), rows); + profile.getText().withHistogram(groups(profile.getText().getHistogram())).withPatterns(groups(profile.getText().getPatterns())); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ProfileSupport.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ProfileSupport.java new file mode 100644 index 0000000..0092e3f --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ProfileSupport.java @@ -0,0 +1,150 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.io.Serializable; +import java.math.BigDecimal; +import java.math.MathContext; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qbits.dataanalysis.result.Axis; +import com.kingsrook.qbits.dataanalysis.result.Chart; +import com.kingsrook.qbits.dataanalysis.result.ChartKind; +import com.kingsrook.qbits.dataanalysis.result.Metric; +import com.kingsrook.qbits.dataanalysis.result.Point; +import com.kingsrook.qbits.dataanalysis.result.Series; +import com.kingsrook.qbits.dataanalysis.result.ValueCount; +import com.kingsrook.qbits.dataanalysis.result.ValueType; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; + +/******************************************************************************* + ** Shared semantic result construction; profilers contain no rendering concerns. + *******************************************************************************/ +public final class ProfileSupport +{ + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private ProfileSupport() + { + } + + + + /******************************************************************************* + ** Empty populations have zero share, never NaN. + *******************************************************************************/ + public static BigDecimal share(long count, long rows) + { + return (rows == 0 ? BigDecimal.ZERO : BigDecimal.valueOf(count).divide(BigDecimal.valueOf(rows), MathContext.DECIMAL128)); + } + + + + /******************************************************************************* + ** Every ordinary count carries its exact membership predicate. + *******************************************************************************/ + public static Metric count(String key, long count, String table, QQueryFilter filter, QQueryFilter condition) + { + return (metric(key, count, ValueType.COUNT).withDrill(DrillFilters.drill(table, filter, condition))); + } + + + + /******************************************************************************* + ** Scalars have semantic types and locale-neutral formatted values. + *******************************************************************************/ + public static Metric metric(String key, Serializable value, ValueType type) + { + return (new Metric().withKey(key).withLabel(key).withValue(value).withValueType(type).withFormatted(value == null ? null : value.toString())); + } + + + + /******************************************************************************* + ** PVS translations run as the current user; drill values always remain ids. + *******************************************************************************/ + public static List values(String table, QQueryFilter filter, QFieldMetaData field, List groups, long total) throws QException + { + List result = new ArrayList<>(); + for(AnalysisBackend.GroupCount group : groups) + { + String label = group.label(); + if(field != null && field.getPossibleValueSourceName() != null && group.value() != null) + { + String translated = AnalysisLabels.translate(field, group.value()); + if(translated != null) + { + label = translated; + } + } + result.add(new ValueCount().withValue(group.value()).withLabel(label).withCount(group.count()).withShare(share(group.count(), total)) + .withDrill(DrillFilters.drill(table, filter, group.condition()))); + } + return (result); + } + + + + /******************************************************************************* + ** Charts are built only from privacy-guarded groups. + *******************************************************************************/ + public static Chart chart(ChartKind kind, ChartKind alternative, String field, List values) + { + Series series = new Series().withName("Count"); + for(ValueCount value : values) + { + series.getPoints().add(new Point().withX(value.getValue()).withY(BigDecimal.valueOf(value.getCount())) + .withLabel(value.getLabel()).withDrill(value.getDrill())); + } + Chart chart = new Chart().withKind(kind).withAlternatives(List.of(alternative)) + .withX(new Axis().withField(field).withLabel(field).withValueType(ValueType.TEXT)) + .withY(new Axis().withLabel("Count").withValueType(ValueType.COUNT)).withSeries(new ArrayList<>(List.of(series))); + if(kind == ChartKind.TIME_SERIES) + { + Series average = new Series().withName("7-bucket moving average"); + for(int i = 0; i < values.size(); i++) + { + long sum = 0; + for(int j = Math.max(0, i - 6); j <= i; j++) + { + sum += values.get(j).getCount(); + } + average.getPoints().add(new Point().withX(values.get(i).getValue()).withLabel(values.get(i).getLabel()) + .withY(BigDecimal.valueOf(sum).divide(BigDecimal.valueOf(Math.min(7, i + 1)), MathContext.DECIMAL128))); + } + chart.getSeries().add(average); + } + return (chart); + } + + + + /******************************************************************************* + ** Convert instants in the session zone; DATE values are already local dates. + *******************************************************************************/ + public static java.time.LocalDate day(Serializable value, java.time.ZoneId zone) + { + return (value instanceof java.time.LocalDate date ? date : ((java.time.Instant) value).atZone(zone).toLocalDate()); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/StandInAnalysisBackend.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/StandInAnalysisBackend.java new file mode 100644 index 0000000..121d9bd --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/StandInAnalysisBackend.java @@ -0,0 +1,689 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.io.Serializable; +import java.math.BigDecimal; +import java.math.MathContext; +import java.time.DayOfWeek; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.temporal.TemporalAdjusters; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Function; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; +import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction; +import com.kingsrook.qqq.backend.core.actions.tables.CountAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByGroupBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; + +/******************************************************************************* + ** Stand-in until QQQ core DATA-ANALYSIS.md §3 operators ship in an RC; then add + ** CoreAnalysisBackend that pushes these down. RC.5 estimates use the first + ** sample_rows rows by primary key, not random sampling. Streamed operations + ** page by primary key and retain at most max_rows records. + *******************************************************************************/ +public class StandInAnalysisBackend implements AnalysisBackend +{ + private static final MathContext MATH = MathContext.DECIMAL128; + private final EffectiveConfig config; + + /******************************************************************************* + ** Bind one request's detached configuration. + *******************************************************************************/ + public StandInAnalysisBackend(EffectiveConfig config) + { + this.config = config; + } + + + + /******************************************************************************* + ** CountAction is core's dedicated query-count operation. + *******************************************************************************/ + @Override + public long count(String table, QQueryFilter filter) throws QException + { + CountInput input = new CountInput(table).withFilter(DrillFilters.restoreTypes(table, filter)); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + return (new CountAction().execute(input).getCount().longValue()); + } + + + + /******************************************************************************* + ** A boundary query selects the prefix retained in every drill. + *******************************************************************************/ + @Override + public Scope scope(String table, QQueryFilter filter, boolean exact) throws QException + { + long total = count(table, filter); + boolean estimate = !exact && total > config.getSampleThresholdRows(); + int limit = estimate ? Math.min(config.getSampleRows(), config.getMaxRows()) : config.getMaxRows(); + QQueryFilter scoped = DrillFilters.copy(filter); + if(total > limit) + { + String pk = primaryKey(table); + if(limit == 0) + { + scoped = DrillFilters.and(scoped, DrillFilters.in(pk, List.of())); + } + else + { + List boundary = query(table, scoped, Set.of(pk), limit - 1, 1); + if(boundary.isEmpty()) + { + throw new QException("Data changed while selecting analysis rows; retry."); + } + scoped = DrillFilters.and(scoped, DrillFilters.condition(pk, QCriteriaOperator.LESS_THAN_OR_EQUALS, boundary.getFirst().getValue(pk))); + } + } + return (new Scope(scoped, total, Math.min(total, limit), total > limit && (!estimate || limit < config.getSampleRows()), estimate)); + } + + + + /******************************************************************************* + ** Fetch only fields used by the operation, including requested heavy text. + *******************************************************************************/ + private List query(String table, QQueryFilter filter, Set fields, int offset, int limit) throws QException + { + QueryInput input = new QueryInput(table).withFilter(DrillFilters.restoreTypes(table, filter).withOrderBy(new QFilterOrderBy(primaryKey(table))).withSkip(offset).withLimit(limit)) + .withFieldNamesToInclude(fields).withShouldFetchHeavyFields(true); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + return (new QueryAction().execute(input).getRecords()); + } + + + + /******************************************************************************* + ** Bound both page size and retained records. + *******************************************************************************/ + private List rows(String table, QQueryFilter filter, Set fields, int limit) throws QException + { + List records = new ArrayList<>(); + int bound = Math.min(limit, config.getMaxRows()); + while(records.size() < bound) + { + int size = Math.min(1000, bound - records.size()); + List page = query(table, filter, fields, records.size(), size); + records.addAll(page); + if(page.size() < size) + { + break; + } + } + return (records); + } + + + + /******************************************************************************* + ** Core applies table permissions, field locks and record locks. + *******************************************************************************/ + private List aggregate(AggregateInput input) throws QException + { + QQueryFilter restored = DrillFilters.restoreTypes(input.getTableName(), input.getFilter()); + restored.setOrderBys(input.getFilter().getOrderBys()); + input.setFilter(restored); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + return (new AggregateAction().execute(input).getResults()); + } + + + + /******************************************************************************* + ** Batch supported aggregates; nulls use core criteria. The bounded scan adds + ** whitespace blanks and representative records for cardinality drills. + *******************************************************************************/ + @Override + public Map fieldCounts(String table, QQueryFilter filter, List fields) throws QException + { + Map result = new LinkedHashMap<>(); + long total = count(table, filter); + for(int start = 0; start < fields.size(); start += 20) + { + List batch = fields.subList(start, Math.min(start + 20, fields.size())); + AggregateInput input = new AggregateInput(table).withFilter(DrillFilters.copy(filter)); + for(String field : batch) + { + input.withAggregate(countOf(field, AggregateOperator.COUNT)).withAggregate(countOf(field, AggregateOperator.COUNT_DISTINCT)); + } + List aggregates = aggregate(input); + Set selection = new LinkedHashSet<>(batch); + selection.add(primaryKey(table)); + List records = rows(table, filter, selection, config.getMaxRows()); + for(String field : batch) + { + boolean string = field(table, field).getType().isStringLike(); + long nulls = count(table, DrillFilters.and(filter, DrillFilters.nulls(field, string))); + Map representatives = new LinkedHashMap<>(); + Set blanks = new LinkedHashSet<>(); + for(QRecord record : records) + { + Serializable value = record.getValue(field); + if(value != null) + { + representatives.putIfAbsent(value instanceof BigDecimal d ? d.stripTrailingZeros() : value, record.getValue(primaryKey(table))); + if(string && value.toString().isBlank()) + { + blanks.add(value); + } + } + } + QQueryFilter blankFilter = DrillFilters.nulls(field, string); + if(!blanks.isEmpty()) + { + blankFilter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR).withSubFilter(blankFilter) + .withSubFilter(DrillFilters.in(field, new ArrayList<>(blanks))); + } + long distinct = aggregates.isEmpty() ? 0 : number(aggregates.getFirst(), field, AggregateOperator.COUNT_DISTINCT).longValue(); + if(isMemory(table) && field(table, field).getType() == QFieldType.DECIMAL) + { + distinct = representatives.size(); + } + result.put(field, new FieldCounts(total - nulls, nulls, string ? count(table, DrillFilters.and(filter, blankFilter)) : nulls, + distinct, blankFilter, DrillFilters.in(primaryKey(table), new ArrayList<>(representatives.values())))); + } + } + return (result); + } + + + + /******************************************************************************* + ** Push down group, count, ordering and pagination. + *******************************************************************************/ + @Override + public List topValues(String table, QQueryFilter filter, String field, int limit, int offset, String search) throws QException + { + QQueryFilter selected = DrillFilters.copy(filter); + if(search != null && !search.isBlank()) + { + if(!field(table, field).getType().isStringLike()) + { + return (groups(table, selected, List.of(field), Math.max(1, config.getMaxRows()), 0, false).stream() + .filter(g -> g.value() != null && g.value().toString().contains(search)).skip(offset).limit(limit).toList()); + } + selected = DrillFilters.and(selected, DrillFilters.condition(field, QCriteriaOperator.CONTAINS, search)); + } + return (groups(table, selected, List.of(field), limit, offset, false)); + } + + + + /******************************************************************************* + ** Tuple drills use IN for non-null values and null-only predicates otherwise. + *******************************************************************************/ + private List groups(String table, QQueryFilter filter, List fields, int limit, int offset, boolean duplicates) throws QException + { + Aggregate count = countOf(primaryKey(table), AggregateOperator.COUNT); + QQueryFilter ordered = DrillFilters.copy(filter).withOrderBy(new QFilterOrderByAggregate(count, false)); + AggregateInput input = new AggregateInput(table).withFilter(ordered).withAggregate(count).withLimit(Math.addExact(limit, offset)); + for(String field : fields) + { + input.withGroupBy(new GroupBy(field(table, field))); + ordered.withOrderBy(new QFilterOrderByGroupBy(new GroupBy(field(table, field)))); + } + Map merged = new LinkedHashMap<>(); + for(AggregateResult row : aggregate(input)) + { + long size = number(row, primaryKey(table), AggregateOperator.COUNT).longValue(); + LinkedHashMap tuple = new LinkedHashMap<>(); + QQueryFilter condition = new QQueryFilter(); + for(String field : fields) + { + Serializable value = row.getGroupByValues().entrySet().stream().filter(e -> field.equals(e.getKey().getFieldName())).findFirst().orElseThrow().getValue(); + tuple.put(field, value instanceof BigDecimal d ? new BigDecimal(d.stripTrailingZeros().toPlainString()) : value); + condition.withSubFilter(value == null ? DrillFilters.nulls(field, field(table, field).getType().isStringLike()) + : value instanceof BigDecimal ? DrillFilters.in(field, List.of(value.toString())) + : duplicates ? DrillFilters.in(field, List.of(value)) : DrillFilters.value(field, value, field(table, field).getType().isStringLike())); + } + Serializable value = duplicates ? tuple : tuple.get(fields.getFirst()); + GroupCount previous = merged.get(value); + if(previous != null) + { + size += previous.count(); + condition = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR).withSubFilter(previous.condition()).withSubFilter(condition); + } + merged.put(value, new GroupCount(value, value == null ? "(null)" : value.toString(), size, condition)); + } + List groups = new ArrayList<>(merged.values()); + if(isMemory(table)) + { + groups.sort(Comparator.comparingLong(GroupCount::count).reversed().thenComparing(GroupCount::label)); + } + return (groups.stream().filter(g -> !duplicates || g.count() > 1).skip(offset).limit(limit).toList()); + } + + + + /******************************************************************************* + ** Push down basic moments; stream ordered values for variance and percentiles. + *******************************************************************************/ + @Override + public NumericSummary numericSummary(String table, QQueryFilter filter, String field, List percentiles) throws QException + { + List values = rows(table, filter, Set.of(field), config.getMaxRows()).stream().map(r -> r.getValue(field)) + .filter(Objects::nonNull).map(v -> new BigDecimal(v.toString())).sorted().toList(); + Map metrics = new LinkedHashMap<>(); + AggregateInput input = new AggregateInput(table).withFilter(DrillFilters.copy(filter)); + for(AggregateOperator op : List.of(AggregateOperator.MIN, AggregateOperator.MAX, AggregateOperator.AVG, AggregateOperator.SUM)) + { + input.withAggregate(new Aggregate(field, op)); + } + List results = aggregate(input); + if(values.isEmpty()) + { + for(String key : List.of("min", "max", "mean", "stddev", "sum")) + { + metrics.put(key, null); + } + for(BigDecimal p : percentiles) + { + metrics.put("p" + p.multiply(BigDecimal.valueOf(100)).stripTrailingZeros().toPlainString(), null); + } + QQueryFilter empty = DrillFilters.in(primaryKey(table), List.of()); + return (new NumericSummary(metrics, List.of(), 0, empty, List.of(new GroupCount("zeros", "zeros", 0, empty), new GroupCount("negatives", "negatives", 0, empty)))); + } + AggregateResult result = results.getFirst(); + metrics.put("min", number(result, field, AggregateOperator.MIN)); + metrics.put("max", number(result, field, AggregateOperator.MAX)); + metrics.put("mean", number(result, field, AggregateOperator.AVG)); + metrics.put("sum", number(result, field, AggregateOperator.SUM)); + BigDecimal mean = values.stream().reduce(BigDecimal.ZERO, BigDecimal::add).divide(BigDecimal.valueOf(values.size()), MATH); + BigDecimal variance = values.stream().map(v -> v.subtract(mean).pow(2)).reduce(BigDecimal.ZERO, BigDecimal::add); + metrics.put("stddev", values.size() < 2 ? null : variance.divide(BigDecimal.valueOf(values.size() - 1), MATH).sqrt(MATH)); + for(BigDecimal p : percentiles) + { + metrics.put("p" + p.multiply(BigDecimal.valueOf(100)).stripTrailingZeros().toPlainString(), percentile(values, p)); + } + BigDecimal q1 = percentile(values, new BigDecimal("0.25")); + BigDecimal q3 = percentile(values, new BigDecimal("0.75")); + BigDecimal span = q3.subtract(q1).multiply(new BigDecimal("1.5")); + BigDecimal low = q1.subtract(span); + BigDecimal high = q3.add(span); + QQueryFilter outliers = isMemory(table) + ? new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withSubFilter(numericRange(table, field, values, null, low, false)) + .withSubFilter(DrillFilters.and(DrillFilters.condition(field, QCriteriaOperator.IS_NOT_BLANK), DrillFilters.condition(field, QCriteriaOperator.GREATER_THAN, high))) + : DrillFilters.outliers(field, low, high); + List extremes = values.stream().filter(v -> v.compareTo(low) < 0 || v.compareTo(high) > 0).map(v -> new BigDecimal(v.stripTrailingZeros().toPlainString())).distinct() + .sorted(Comparator.comparing((BigDecimal v) -> v.subtract(q1.add(q3).divide(BigDecimal.valueOf(2), MATH)).abs()).reversed()).limit(10).toList(); + List extremeGroups = new ArrayList<>(); + for(BigDecimal value : extremes) + { + QQueryFilter condition = DrillFilters.in(field, equalNumbers(values, value)); + extremeGroups.add(new GroupCount(value, value.toPlainString(), count(table, DrillFilters.and(filter, condition)), condition)); + } + QQueryFilter zeros = DrillFilters.in(field, equalNumbers(values, BigDecimal.ZERO)); + QQueryFilter negatives = numericRange(table, field, values, null, BigDecimal.ZERO, false); + return (new NumericSummary(metrics, extremeGroups, count(table, DrillFilters.and(filter, outliers)), outliers, + List.of(bucket(table, filter, "zeros", zeros), bucket(table, filter, "negatives", negatives)))); + } + + + + /******************************************************************************* + ** PostgreSQL percentile_cont interpolation at p * (n - 1). + *******************************************************************************/ + public static BigDecimal percentile(List sorted, BigDecimal p) + { + if(sorted.isEmpty()) + { + return (null); + } + BigDecimal index = p.multiply(BigDecimal.valueOf(sorted.size() - 1)); + int low = index.intValue(); + int high = Math.min(low + 1, sorted.size() - 1); + return (sorted.get(low).add(sorted.get(high).subtract(sorted.get(low)).multiply(index.subtract(BigDecimal.valueOf(low))))); + } + + + + /******************************************************************************* + ** Predicates share the exact boundary arithmetic used by drills. + *******************************************************************************/ + @Override + public List numericHistogram(String table, QQueryFilter filter, String field, BigDecimal low, BigDecimal high, int bins) throws QException + { + List result = new ArrayList<>(); + if(low == null || high == null) + { + return (result); + } + List values = rows(table, filter, Set.of(field), config.getMaxRows()).stream().map(r -> r.getValue(field)) + .filter(Objects::nonNull).map(v -> new BigDecimal(v.toString())).toList(); + result.add(bucket(table, filter, "Below " + low, numericRange(table, field, values, null, low, false))); + BigDecimal width = high.subtract(low).divide(BigDecimal.valueOf(bins), MATH); + for(int i = 0; i < bins; i++) + { + BigDecimal start = low.add(width.multiply(BigDecimal.valueOf(i))); + BigDecimal end = i == bins - 1 ? high : low.add(width.multiply(BigDecimal.valueOf(i + 1))); + result.add(bucket(table, filter, "[" + start + ", " + end + (i == bins - 1 ? "]" : ")"), numericRange(table, field, values, start, end, i == bins - 1))); + } + result.add(bucket(table, filter, "Above " + high, DrillFilters.and(DrillFilters.condition(field, QCriteriaOperator.IS_NOT_BLANK), DrillFilters.condition(field, QCriteriaOperator.GREATER_THAN, high)))); + return (result); + } + + + + /******************************************************************************* + ** A bucket count and its producing condition. + *******************************************************************************/ + private GroupCount bucket(String table, QQueryFilter filter, String label, QQueryFilter condition) throws QException + { + return (new GroupCount(label, label, count(table, DrillFilters.and(filter, condition)), condition)); + } + + + + /******************************************************************************* + ** Fill gaps inside the observed range, retaining DST-aware boundaries. + *******************************************************************************/ + @Override + public List dateBuckets(String table, QQueryFilter filter, String field, DateUnit unit, ZoneId zone) throws QException + { + List records = rows(table, filter, Set.of(field), config.getMaxRows()); + if(unit == DateUnit.DAY_OF_WEEK || unit == DateUnit.HOUR_OF_DAY) + { + return (mappedGroups(table, filter, field, records, v -> unit == DateUnit.DAY_OF_WEEK ? day(v, zone).getDayOfWeek().getValue() + : ((Instant) v).atZone(zone).getHour(), 24)); + } + List dates = records.stream().map(r -> r.getValue(field)).filter(Objects::nonNull).map(v -> day(v, zone)).sorted().toList(); + List result = new ArrayList<>(); + if(dates.isEmpty()) + { + return (result); + } + LocalDate start = floor(dates.getFirst(), unit); + LocalDate last = floor(dates.getLast(), unit); + while(!start.isAfter(last)) + { + LocalDate end = next(start, unit); + result.add(bucket(table, filter, start + " / " + end, DrillFilters.dateRange(field, start, end, field(table, field).getType() == QFieldType.DATE_TIME, zone))); + start = end; + } + return (result); + } + + + + /******************************************************************************* + ** Dates carry no time-zone conversion; instants do. + *******************************************************************************/ + public static LocalDate day(Serializable value, ZoneId zone) + { + return (ProfileSupport.day(value, zone)); + } + + + + /******************************************************************************* + ** ISO weeks begin Monday. + *******************************************************************************/ + private static LocalDate floor(LocalDate date, DateUnit unit) + { + return (switch(unit) + { + case WEEK -> date.with(TemporalAdjusters.previousOrSame(DayOfWeek.MONDAY)); + case MONTH -> date.withDayOfMonth(1); + case YEAR -> date.withDayOfYear(1); + default -> date; + }); + } + + + + /******************************************************************************* + ** Calendar arithmetic handles leap years and daylight savings. + *******************************************************************************/ + private static LocalDate next(LocalDate date, DateUnit unit) + { + return (switch(unit) + { + case WEEK -> date.plusWeeks(1); + case MONTH -> date.plusMonths(1); + case YEAR -> date.plusYears(1); + default -> date.plusDays(1); + }); + } + + + + /******************************************************************************* + ** Preserve types in minimum and maximum aggregate values. + *******************************************************************************/ + @Override + public Map extrema(String table, QQueryFilter filter, String field) throws QException + { + AggregateInput input = new AggregateInput(table).withFilter(DrillFilters.copy(filter)) + .withAggregate(new Aggregate(field, AggregateOperator.MIN)).withAggregate(new Aggregate(field, AggregateOperator.MAX)); + Map result = new LinkedHashMap<>(); + result.put("min", null); + result.put("max", null); + for(AggregateResult row : aggregate(input)) + { + row.getAggregateValues().forEach((key, value) -> result.put(key.getOperator().name().toLowerCase(java.util.Locale.ROOT), value)); + } + return (result); + } + + + + /******************************************************************************* + ** Stream string lengths; bucket members retain exact value-based drills. + *******************************************************************************/ + @Override + public LengthSummary lengthSummary(String table, QQueryFilter filter, String field) throws QException + { + List records = rows(table, filter, Set.of(field), config.getMaxRows()); + List lengths = records.stream().map(r -> r.getValueString(field)).filter(Objects::nonNull).map(String::length).sorted().toList(); + Map metrics = new LinkedHashMap<>(); + if(!lengths.isEmpty()) + { + metrics.put("minLength", BigDecimal.valueOf(lengths.getFirst())); + metrics.put("maxLength", BigDecimal.valueOf(lengths.getLast())); + metrics.put("meanLength", BigDecimal.valueOf(lengths.stream().mapToLong(Integer::longValue).sum()).divide(BigDecimal.valueOf(lengths.size()), MATH)); + } + int width = lengths.isEmpty() ? 1 : Math.max(1, (int) Math.ceil((lengths.getLast() + 1.0) / 20)); + return (new LengthSummary(metrics, mappedGroups(table, filter, field, records, v -> (v.toString().length() / width * width) + "–" + (v.toString().length() / width * width + width - 1), 20))); + } + + + + /******************************************************************************* + ** Shapes use a primary-key prefix of at most the requested sample size. + *******************************************************************************/ + @Override + public List patterns(String table, QQueryFilter filter, String field, int sampleRows) throws QException + { + List records = rows(table, filter, Set.of(field, primaryKey(table)), sampleRows); + QQueryFilter sampled = records.isEmpty() ? DrillFilters.in(primaryKey(table), List.of()) + : DrillFilters.condition(primaryKey(table), QCriteriaOperator.LESS_THAN_OR_EQUALS, records.getLast().getValue(primaryKey(table))); + return (mappedGroups(table, DrillFilters.and(filter, sampled), field, records, v -> shape(v.toString()), 10)); + } + + + + /******************************************************************************* + ** Collapse runs of letters and digits, preserving literal punctuation. + *******************************************************************************/ + public static String shape(String text) + { + return (text.replaceAll("\\p{L}+", "A").replaceAll("\\p{N}+", "9")); + } + + + + /******************************************************************************* + ** Derived groups do not require unsupported SQL functions. + *******************************************************************************/ + private List mappedGroups(String table, QQueryFilter filter, String field, List rows, Function mapper, int limit) throws QException + { + Map> members = new LinkedHashMap<>(); + for(QRecord row : rows) + { + Serializable value = row.getValue(field); + if(value != null) + { + members.computeIfAbsent(mapper.apply(value), ignored -> new LinkedHashSet<>()).add(value); + } + } + List groups = new ArrayList<>(); + for(Map.Entry> entry : members.entrySet()) + { + QQueryFilter condition = DrillFilters.and(filter, DrillFilters.in(field, new ArrayList<>(entry.getValue()))); + groups.add(new GroupCount(entry.getKey(), entry.getKey().toString(), count(table, condition), condition)); + } + return (groups.stream().sorted(Comparator.comparingLong(GroupCount::count).reversed().thenComparing(GroupCount::label)).limit(limit).toList()); + } + + + + /******************************************************************************* + ** Top duplicate groups are sorted before the count threshold is applied. + *******************************************************************************/ + @Override + public List duplicates(String table, QQueryFilter filter, List fields, int limit) throws QException + { + return (groups(table, filter, fields, limit, 0, true)); + } + + + + /******************************************************************************* + ** Metadata lookup follows facade security checks. + *******************************************************************************/ + private static QFieldMetaData field(String table, String field) + { + return (QContext.getQInstance().getTable(table).getField(field)); + } + + + + /******************************************************************************* + ** Deterministic paging requires an addressable primary key. + *******************************************************************************/ + private static String primaryKey(String table) + { + return (QContext.getQInstance().getTable(table).getPrimaryKeyField()); + } + + + + /******************************************************************************* + ** COUNT and COUNT_DISTINCT always read as integers: RC.5's RDBMS reader + ** otherwise reads them as the field's type (a boolean count fails). + *******************************************************************************/ + private static Aggregate countOf(String field, AggregateOperator op) + { + return (new Aggregate(field, op).withFieldType(QFieldType.INTEGER)); + } + + + + /******************************************************************************* + ** The value of one aggregate of a result row (0 when absent). + *******************************************************************************/ + private static BigDecimal number(AggregateResult row, String field, AggregateOperator op) + { + Serializable value = row.getAggregateValues().entrySet().stream().filter(e -> field.equals(e.getKey().getFieldName()) && op == e.getKey().getOperator()) + .map(Map.Entry::getValue).filter(Objects::nonNull).findFirst().orElse(0); + return (new BigDecimal(value.toString())); + } + + + + /******************************************************************************* + ** RC.5 memory equality is scale-sensitive (BigDecimal.equals), unlike SQL. + ** Use strict comparison plus actual boundary spellings to preserve half-open + ** membership. SQL backends use ordinary range predicates. + *******************************************************************************/ + private static QQueryFilter numericRange(String table, String field, List values, BigDecimal low, BigDecimal high, boolean closed) + { + if(!isMemory(table)) + { + return (DrillFilters.range(field, low, high, closed)); + } + QQueryFilter result = DrillFilters.condition(field, QCriteriaOperator.IS_NOT_BLANK); + if(low != null) + { + result.withSubFilter(new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withSubFilter(DrillFilters.condition(field, QCriteriaOperator.GREATER_THAN, low)) + .withSubFilter(DrillFilters.in(field, equalNumbers(values, low)))); + } + if(high != null) + { + result.withCriteria(field, QCriteriaOperator.LESS_THAN_OR_EQUALS, high); + List boundary = equalNumbers(values, high); + if(!closed && !boundary.isEmpty()) + { + result.withCriteria(field, QCriteriaOperator.NOT_IN, boundary); + } + } + return (result); + } + + + + /******************************************************************************* + ** Preserve scale through JSON using string operands, accepted by core and SQL. + *******************************************************************************/ + private static List equalNumbers(List values, BigDecimal target) + { + return (values.stream().filter(v -> v.compareTo(target) == 0).map(BigDecimal::toString).distinct().toList()); + } + + + + /******************************************************************************* + ** Isolate compatibility fixes to the RC.5 memory implementation. + *******************************************************************************/ + private static boolean isMemory(String table) + { + return ("memory".equals(QContext.getQInstance().getBackend(QContext.getQInstance().getTable(table).getBackendName()).getBackendType())); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/TableProfiler.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/TableProfiler.java new file mode 100644 index 0000000..fd55ba8 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/TableProfiler.java @@ -0,0 +1,116 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.dataanalysis.result.CandidateKey; +import com.kingsrook.qbits.dataanalysis.result.ChartKind; +import com.kingsrook.qbits.dataanalysis.result.FieldSummary; +import com.kingsrook.qbits.dataanalysis.result.Point; +import com.kingsrook.qbits.dataanalysis.result.TableProfile; +import com.kingsrook.qbits.dataanalysis.result.TableProfileRequest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; + +/******************************************************************************* + ** Batched table completeness and candidate-key discovery over the selected scope. + *******************************************************************************/ +public class TableProfiler +{ + private final AnalysisBackend backend; + + /******************************************************************************* + ** Bind the primitive operation port. + *******************************************************************************/ + public TableProfiler(AnalysisBackend backend) + { + this.backend = backend; + } + + + + /******************************************************************************* + ** Candidate pairs consider the eight most unique readable selected fields. + *******************************************************************************/ + public TableProfile profile(TableProfileRequest request, List fields, AnalysisBackend.Scope scope) throws QException + { + String table = request.getTableName(); + QQueryFilter filter = scope.filter(); + long rows = backend.count(table, filter); + TableProfile result = new TableProfile().withTableName(table).withFilter(DrillFilters.restoreTypes(request.getTableName(), request.getFilter())).withEstimate(scope.estimate()) + .withSampleRows(scope.estimate() ? rows : 0L).withRowsScanned(scope.rowsScanned()).withTruncated(scope.truncated()).withAsOf(Instant.now()) + .withRowCount(ProfileSupport.count("rows", rows, table, filter, null)); + Map counts = backend.fieldCounts(table, filter, fields.stream().map(QFieldMetaData::getName).toList()); + for(QFieldMetaData field : fields) + { + String name = field.getName(); + AnalysisBackend.FieldCounts c = counts.get(name); + result.getFields().add(new FieldSummary().withField(name).withLabel(field.getLabel()).withType(field.getType().name()) + .withCompleteness(ProfileSupport.share(rows - c.blanks(), rows)).withDistinct(c.distinct()).withUniqueness(ProfileSupport.share(c.distinct(), c.nonNull())) + .withConstant(c.distinct() == 1).withEmpty(c.blanks() == rows) + .withNulls(ProfileSupport.count("nulls", c.nulls(), table, filter, DrillFilters.nulls(name, field.getType().isStringLike()))) + .withBlanks(ProfileSupport.count("blanks", c.blanks(), table, filter, c.blankFilter()))); + if(rows > 0 && c.distinct() == rows && c.nulls() == 0) + { + result.getCandidateKeys().add(key(table, filter, List.of(name), rows)); + } + } + List ranked = result.getFields().stream().sorted(Comparator.comparing(FieldSummary::getUniqueness).reversed().thenComparing(FieldSummary::getField)).limit(8).toList(); + for(int i = 0; i < ranked.size(); i++) + { + for(int j = i + 1; j < ranked.size(); j++) + { + List pair = List.of(ranked.get(i).getField(), ranked.get(j).getField()); + if(rows > 0 && pair.stream().allMatch(f -> counts.get(f).nulls() == 0) && backend.duplicates(table, filter, pair, 1).isEmpty()) + { + result.getCandidateKeys().add(key(table, filter, pair, rows)); + } + } + } + result.withHeatmap(ProfileSupport.chart(ChartKind.HEATMAP, ChartKind.BAR, "field", List.of())); + result.getHeatmap().getY().withLabel("Completeness").withValueType(com.kingsrook.qbits.dataanalysis.result.ValueType.PERCENT); + result.getHeatmap().getSeries().getFirst().withName("Completeness").withPoints(new ArrayList<>()); + for(FieldSummary field : result.getFields()) + { + result.getHeatmap().getSeries().getFirst().getPoints().add(new Point().withX(field.getField()).withLabel(field.getLabel()).withY(field.getCompleteness())); + } + if(scope.estimate() || scope.truncated()) + { + result.getWarnings().add("Profiles and candidate keys describe only the first " + rows + " rows by primary key."); + } + return (result); + } + + + + /******************************************************************************* + ** Candidate-key counts drill to the whole assessed population. + *******************************************************************************/ + private static CandidateKey key(String table, QQueryFilter filter, List fields, long rows) + { + return (new CandidateKey().withFields(fields).withRowCount(rows).withDrill(DrillFilters.drill(table, filter, null))); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/AnalysisRequest.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/AnalysisRequest.java new file mode 100644 index 0000000..5ff6e31 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/AnalysisRequest.java @@ -0,0 +1,48 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + + +import java.io.Serializable; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; + + +/******************************************************************************* + ** Common analysis request contract. + *******************************************************************************/ +public interface AnalysisRequest extends Serializable +{ + + /******************************************************************************* + ** Request getTableName. + *******************************************************************************/ + String getTableName(); + + /******************************************************************************* + ** Request getFilter. + *******************************************************************************/ + QQueryFilter getFilter(); + + /******************************************************************************* + ** Request getExact. + *******************************************************************************/ + Boolean getExact(); +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/Axis.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/Axis.java new file mode 100644 index 0000000..8e09534 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/Axis.java @@ -0,0 +1,100 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + + +import java.io.Serializable; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + + +/******************************************************************************* + ** Typed Axis contract; independent of any presentation framework. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class Axis implements Serializable +{ + private String field; + private String label; + private ValueType valueType; + + + + /******************************************************************************* + ** Getter for field. + *******************************************************************************/ + public String getField() + { + return (field); + } + + + + /******************************************************************************* + ** Fluent setter for field. + *******************************************************************************/ + public Axis withField(String field) + { + this.field = field; + return (this); + } + + + + /******************************************************************************* + ** Getter for label. + *******************************************************************************/ + public String getLabel() + { + return (label); + } + + + + /******************************************************************************* + ** Fluent setter for label. + *******************************************************************************/ + public Axis withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for valueType. + *******************************************************************************/ + public ValueType getValueType() + { + return (valueType); + } + + + + /******************************************************************************* + ** Fluent setter for valueType. + *******************************************************************************/ + public Axis withValueType(ValueType valueType) + { + this.valueType = valueType; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/CandidateKey.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/CandidateKey.java new file mode 100644 index 0000000..a6d980d --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/CandidateKey.java @@ -0,0 +1,101 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + + +import java.io.Serializable; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + + +/******************************************************************************* + ** Typed CandidateKey contract; independent of any presentation framework. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class CandidateKey implements Serializable +{ + private List fields; + private Long rowCount; + private Drill drill; + + + + /******************************************************************************* + ** Getter for fields. + *******************************************************************************/ + public List getFields() + { + return (fields); + } + + + + /******************************************************************************* + ** Fluent setter for fields. + *******************************************************************************/ + public CandidateKey withFields(List fields) + { + this.fields = fields; + return (this); + } + + + + /******************************************************************************* + ** Getter for rowCount. + *******************************************************************************/ + public Long getRowCount() + { + return (rowCount); + } + + + + /******************************************************************************* + ** Fluent setter for rowCount. + *******************************************************************************/ + public CandidateKey withRowCount(Long rowCount) + { + this.rowCount = rowCount; + return (this); + } + + + + /******************************************************************************* + ** Getter for drill. + *******************************************************************************/ + public Drill getDrill() + { + return (drill); + } + + + + /******************************************************************************* + ** Fluent setter for drill. + *******************************************************************************/ + public CandidateKey withDrill(Drill drill) + { + this.drill = drill; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/Capabilities.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/Capabilities.java new file mode 100644 index 0000000..719cd5d --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/Capabilities.java @@ -0,0 +1,123 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + + +import java.io.Serializable; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + + +/******************************************************************************* + ** Typed Capabilities contract; independent of any presentation framework. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class Capabilities implements Serializable +{ + private String tableName; + private List fields; + private com.kingsrook.qbits.dataanalysis.engine.EffectiveConfig config; + private String backend; + + + + /******************************************************************************* + ** Getter for tableName. + *******************************************************************************/ + public String getTableName() + { + return (tableName); + } + + + + /******************************************************************************* + ** Fluent setter for tableName. + *******************************************************************************/ + public Capabilities withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for fields. + *******************************************************************************/ + public List getFields() + { + return (fields); + } + + + + /******************************************************************************* + ** Fluent setter for fields. + *******************************************************************************/ + public Capabilities withFields(List fields) + { + this.fields = fields; + return (this); + } + + + + /******************************************************************************* + ** Getter for config. + *******************************************************************************/ + public com.kingsrook.qbits.dataanalysis.engine.EffectiveConfig getConfig() + { + return (config); + } + + + + /******************************************************************************* + ** Fluent setter for config. + *******************************************************************************/ + public Capabilities withConfig(com.kingsrook.qbits.dataanalysis.engine.EffectiveConfig config) + { + this.config = config; + return (this); + } + + + + /******************************************************************************* + ** Getter for backend. + *******************************************************************************/ + public String getBackend() + { + return (backend); + } + + + + /******************************************************************************* + ** Fluent setter for backend. + *******************************************************************************/ + public Capabilities withBackend(String backend) + { + this.backend = backend; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/Chart.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/Chart.java new file mode 100644 index 0000000..e0dee58 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/Chart.java @@ -0,0 +1,167 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + + +import java.io.Serializable; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + + +/******************************************************************************* + ** Typed Chart contract; independent of any presentation framework. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class Chart implements Serializable +{ + private ChartKind kind; + private List alternatives = List.of(); + private Axis x; + private Axis y; + private List series = List.of(); + private List annotations = List.of(); + + + + /******************************************************************************* + ** Getter for kind. + *******************************************************************************/ + public ChartKind getKind() + { + return (kind); + } + + + + /******************************************************************************* + ** Fluent setter for kind. + *******************************************************************************/ + public Chart withKind(ChartKind kind) + { + this.kind = kind; + return (this); + } + + + + /******************************************************************************* + ** Getter for alternatives. + *******************************************************************************/ + public List getAlternatives() + { + return (alternatives); + } + + + + /******************************************************************************* + ** Fluent setter for alternatives. + *******************************************************************************/ + public Chart withAlternatives(List alternatives) + { + this.alternatives = alternatives; + return (this); + } + + + + /******************************************************************************* + ** Getter for x. + *******************************************************************************/ + public Axis getX() + { + return (x); + } + + + + /******************************************************************************* + ** Fluent setter for x. + *******************************************************************************/ + public Chart withX(Axis x) + { + this.x = x; + return (this); + } + + + + /******************************************************************************* + ** Getter for y. + *******************************************************************************/ + public Axis getY() + { + return (y); + } + + + + /******************************************************************************* + ** Fluent setter for y. + *******************************************************************************/ + public Chart withY(Axis y) + { + this.y = y; + return (this); + } + + + + /******************************************************************************* + ** Getter for series. + *******************************************************************************/ + public List getSeries() + { + return (series); + } + + + + /******************************************************************************* + ** Fluent setter for series. + *******************************************************************************/ + public Chart withSeries(List series) + { + this.series = series; + return (this); + } + + + + /******************************************************************************* + ** Getter for annotations. + *******************************************************************************/ + public List getAnnotations() + { + return (annotations); + } + + + + /******************************************************************************* + ** Fluent setter for annotations. + *******************************************************************************/ + public Chart withAnnotations(List annotations) + { + this.annotations = annotations; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/ChartKind.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/ChartKind.java new file mode 100644 index 0000000..49b3742 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/ChartKind.java @@ -0,0 +1,30 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + + +/******************************************************************************* + ** Semantic ChartKind values. + *******************************************************************************/ +public enum ChartKind +{ + HISTOGRAM, BOX, BAR, PARETO, TIME_SERIES, SCATTER, HEATMAP +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/ColumnProfile.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/ColumnProfile.java new file mode 100644 index 0000000..fecd7e0 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/ColumnProfile.java @@ -0,0 +1,455 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + + +import java.io.Serializable; +import java.time.Instant; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; + + +/******************************************************************************* + ** Typed ColumnProfile contract; independent of any presentation framework. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class ColumnProfile implements Serializable +{ + private String tableName; + private QQueryFilter filter; + private Boolean estimate = false; + private Long sampleRows = 0L; + private Boolean truncated = false; + private Long rowsScanned = 0L; + private Instant asOf; + private String dataVersion; + private List warnings = new java.util.ArrayList<>(); + private String fieldName; + private String fieldLabel; + private String fieldType; + private List counts = new java.util.ArrayList<>(); + private NumericProfile numeric; + private DateProfile dates; + private TextProfile text; + private List booleans; + private List topValues = List.of(); + private Chart chart; + + + + /******************************************************************************* + ** Getter for tableName. + *******************************************************************************/ + public String getTableName() + { + return (tableName); + } + + + + /******************************************************************************* + ** Fluent setter for tableName. + *******************************************************************************/ + public ColumnProfile withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for filter. + *******************************************************************************/ + public QQueryFilter getFilter() + { + return (filter); + } + + + + /******************************************************************************* + ** Fluent setter for filter. + *******************************************************************************/ + public ColumnProfile withFilter(QQueryFilter filter) + { + this.filter = filter; + return (this); + } + + + + /******************************************************************************* + ** Getter for estimate. + *******************************************************************************/ + public Boolean getEstimate() + { + return (estimate); + } + + + + /******************************************************************************* + ** Fluent setter for estimate. + *******************************************************************************/ + public ColumnProfile withEstimate(Boolean estimate) + { + this.estimate = estimate; + return (this); + } + + + + /******************************************************************************* + ** Getter for sampleRows. + *******************************************************************************/ + public Long getSampleRows() + { + return (sampleRows); + } + + + + /******************************************************************************* + ** Fluent setter for sampleRows. + *******************************************************************************/ + public ColumnProfile withSampleRows(Long sampleRows) + { + this.sampleRows = sampleRows; + return (this); + } + + + + /******************************************************************************* + ** Getter for truncated. + *******************************************************************************/ + public Boolean getTruncated() + { + return (truncated); + } + + + + /******************************************************************************* + ** Fluent setter for truncated. + *******************************************************************************/ + public ColumnProfile withTruncated(Boolean truncated) + { + this.truncated = truncated; + return (this); + } + + + + /******************************************************************************* + ** Getter for rowsScanned. + *******************************************************************************/ + public Long getRowsScanned() + { + return (rowsScanned); + } + + + + /******************************************************************************* + ** Fluent setter for rowsScanned. + *******************************************************************************/ + public ColumnProfile withRowsScanned(Long rowsScanned) + { + this.rowsScanned = rowsScanned; + return (this); + } + + + + /******************************************************************************* + ** Getter for asOf. + *******************************************************************************/ + public Instant getAsOf() + { + return (asOf); + } + + + + /******************************************************************************* + ** Fluent setter for asOf. + *******************************************************************************/ + public ColumnProfile withAsOf(Instant asOf) + { + this.asOf = asOf; + return (this); + } + + + + /******************************************************************************* + ** Getter for dataVersion. + *******************************************************************************/ + public String getDataVersion() + { + return (dataVersion); + } + + + + /******************************************************************************* + ** Fluent setter for dataVersion. + *******************************************************************************/ + public ColumnProfile withDataVersion(String dataVersion) + { + this.dataVersion = dataVersion; + return (this); + } + + + + /******************************************************************************* + ** Getter for warnings. + *******************************************************************************/ + public List getWarnings() + { + return (warnings); + } + + + + /******************************************************************************* + ** Fluent setter for warnings. + *******************************************************************************/ + public ColumnProfile withWarnings(List warnings) + { + this.warnings = warnings; + return (this); + } + + + + /******************************************************************************* + ** Getter for fieldName. + *******************************************************************************/ + public String getFieldName() + { + return (fieldName); + } + + + + /******************************************************************************* + ** Fluent setter for fieldName. + *******************************************************************************/ + public ColumnProfile withFieldName(String fieldName) + { + this.fieldName = fieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for fieldLabel. + *******************************************************************************/ + public String getFieldLabel() + { + return (fieldLabel); + } + + + + /******************************************************************************* + ** Fluent setter for fieldLabel. + *******************************************************************************/ + public ColumnProfile withFieldLabel(String fieldLabel) + { + this.fieldLabel = fieldLabel; + return (this); + } + + + + /******************************************************************************* + ** Getter for fieldType. + *******************************************************************************/ + public String getFieldType() + { + return (fieldType); + } + + + + /******************************************************************************* + ** Fluent setter for fieldType. + *******************************************************************************/ + public ColumnProfile withFieldType(String fieldType) + { + this.fieldType = fieldType; + return (this); + } + + + + /******************************************************************************* + ** Getter for counts. + *******************************************************************************/ + public List getCounts() + { + return (counts); + } + + + + /******************************************************************************* + ** Fluent setter for counts. + *******************************************************************************/ + public ColumnProfile withCounts(List counts) + { + this.counts = counts; + return (this); + } + + + + /******************************************************************************* + ** Getter for numeric. + *******************************************************************************/ + public NumericProfile getNumeric() + { + return (numeric); + } + + + + /******************************************************************************* + ** Fluent setter for numeric. + *******************************************************************************/ + public ColumnProfile withNumeric(NumericProfile numeric) + { + this.numeric = numeric; + return (this); + } + + + + /******************************************************************************* + ** Getter for dates. + *******************************************************************************/ + public DateProfile getDates() + { + return (dates); + } + + + + /******************************************************************************* + ** Fluent setter for dates. + *******************************************************************************/ + public ColumnProfile withDates(DateProfile dates) + { + this.dates = dates; + return (this); + } + + + + /******************************************************************************* + ** Getter for text. + *******************************************************************************/ + public TextProfile getText() + { + return (text); + } + + + + /******************************************************************************* + ** Fluent setter for text. + *******************************************************************************/ + public ColumnProfile withText(TextProfile text) + { + this.text = text; + return (this); + } + + + + /******************************************************************************* + ** Getter for booleans. + *******************************************************************************/ + public List getBooleans() + { + return (booleans); + } + + + + /******************************************************************************* + ** Fluent setter for booleans. + *******************************************************************************/ + public ColumnProfile withBooleans(List booleans) + { + this.booleans = booleans; + return (this); + } + + + + /******************************************************************************* + ** Getter for topValues. + *******************************************************************************/ + public List getTopValues() + { + return (topValues); + } + + + + /******************************************************************************* + ** Fluent setter for topValues. + *******************************************************************************/ + public ColumnProfile withTopValues(List topValues) + { + this.topValues = topValues; + return (this); + } + + + + /******************************************************************************* + ** Getter for chart. + *******************************************************************************/ + public Chart getChart() + { + return (chart); + } + + + + /******************************************************************************* + ** Fluent setter for chart. + *******************************************************************************/ + public ColumnProfile withChart(Chart chart) + { + this.chart = chart; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/ColumnProfileRequest.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/ColumnProfileRequest.java new file mode 100644 index 0000000..3260aaf --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/ColumnProfileRequest.java @@ -0,0 +1,122 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; + + +/******************************************************************************* + ** Typed ColumnProfileRequest contract; independent of any presentation framework. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class ColumnProfileRequest implements AnalysisRequest +{ + private String tableName; + private QQueryFilter filter; + private Boolean exact = false; + private String fieldName; + + + + /******************************************************************************* + ** Getter for tableName. + *******************************************************************************/ + public String getTableName() + { + return (tableName); + } + + + + /******************************************************************************* + ** Fluent setter for tableName. + *******************************************************************************/ + public ColumnProfileRequest withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for filter. + *******************************************************************************/ + public QQueryFilter getFilter() + { + return (filter); + } + + + + /******************************************************************************* + ** Fluent setter for filter. + *******************************************************************************/ + public ColumnProfileRequest withFilter(QQueryFilter filter) + { + this.filter = filter; + return (this); + } + + + + /******************************************************************************* + ** Getter for exact. + *******************************************************************************/ + public Boolean getExact() + { + return (exact); + } + + + + /******************************************************************************* + ** Fluent setter for exact. + *******************************************************************************/ + public ColumnProfileRequest withExact(Boolean exact) + { + this.exact = exact; + return (this); + } + + + + /******************************************************************************* + ** Getter for fieldName. + *******************************************************************************/ + public String getFieldName() + { + return (fieldName); + } + + + + /******************************************************************************* + ** Fluent setter for fieldName. + *******************************************************************************/ + public ColumnProfileRequest withFieldName(String fieldName) + { + this.fieldName = fieldName; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/DateProfile.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/DateProfile.java new file mode 100644 index 0000000..5a1b254 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/DateProfile.java @@ -0,0 +1,167 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + + +import java.io.Serializable; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + + +/******************************************************************************* + ** Typed DateProfile contract; independent of any presentation framework. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class DateProfile implements Serializable +{ + private List metrics = new java.util.ArrayList<>(); + private String unit; + private List buckets = List.of(); + private List gaps = List.of(); + private List dayOfWeek = List.of(); + private List hourOfDay = List.of(); + + + + /******************************************************************************* + ** Getter for metrics. + *******************************************************************************/ + public List getMetrics() + { + return (metrics); + } + + + + /******************************************************************************* + ** Fluent setter for metrics. + *******************************************************************************/ + public DateProfile withMetrics(List metrics) + { + this.metrics = metrics; + return (this); + } + + + + /******************************************************************************* + ** Getter for unit. + *******************************************************************************/ + public String getUnit() + { + return (unit); + } + + + + /******************************************************************************* + ** Fluent setter for unit. + *******************************************************************************/ + public DateProfile withUnit(String unit) + { + this.unit = unit; + return (this); + } + + + + /******************************************************************************* + ** Getter for buckets. + *******************************************************************************/ + public List getBuckets() + { + return (buckets); + } + + + + /******************************************************************************* + ** Fluent setter for buckets. + *******************************************************************************/ + public DateProfile withBuckets(List buckets) + { + this.buckets = buckets; + return (this); + } + + + + /******************************************************************************* + ** Getter for gaps. + *******************************************************************************/ + public List getGaps() + { + return (gaps); + } + + + + /******************************************************************************* + ** Fluent setter for gaps. + *******************************************************************************/ + public DateProfile withGaps(List gaps) + { + this.gaps = gaps; + return (this); + } + + + + /******************************************************************************* + ** Getter for dayOfWeek. + *******************************************************************************/ + public List getDayOfWeek() + { + return (dayOfWeek); + } + + + + /******************************************************************************* + ** Fluent setter for dayOfWeek. + *******************************************************************************/ + public DateProfile withDayOfWeek(List dayOfWeek) + { + this.dayOfWeek = dayOfWeek; + return (this); + } + + + + /******************************************************************************* + ** Getter for hourOfDay. + *******************************************************************************/ + public List getHourOfDay() + { + return (hourOfDay); + } + + + + /******************************************************************************* + ** Fluent setter for hourOfDay. + *******************************************************************************/ + public DateProfile withHourOfDay(List hourOfDay) + { + this.hourOfDay = hourOfDay; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/DistinctValuesPage.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/DistinctValuesPage.java new file mode 100644 index 0000000..d8d8924 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/DistinctValuesPage.java @@ -0,0 +1,367 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + + +import java.io.Serializable; +import java.time.Instant; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; + + +/******************************************************************************* + ** Typed DistinctValuesPage contract; independent of any presentation framework. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class DistinctValuesPage implements Serializable +{ + private String tableName; + private QQueryFilter filter; + private Boolean estimate = false; + private Long sampleRows = 0L; + private Boolean truncated = false; + private Long rowsScanned = 0L; + private Instant asOf; + private String dataVersion; + private List warnings = new java.util.ArrayList<>(); + private String fieldName; + private Integer page; + private Integer pageSize; + private Boolean hasMore; + private Long totalDistinct; + private List values = List.of(); + + + + /******************************************************************************* + ** Getter for tableName. + *******************************************************************************/ + public String getTableName() + { + return (tableName); + } + + + + /******************************************************************************* + ** Fluent setter for tableName. + *******************************************************************************/ + public DistinctValuesPage withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for filter. + *******************************************************************************/ + public QQueryFilter getFilter() + { + return (filter); + } + + + + /******************************************************************************* + ** Fluent setter for filter. + *******************************************************************************/ + public DistinctValuesPage withFilter(QQueryFilter filter) + { + this.filter = filter; + return (this); + } + + + + /******************************************************************************* + ** Getter for estimate. + *******************************************************************************/ + public Boolean getEstimate() + { + return (estimate); + } + + + + /******************************************************************************* + ** Fluent setter for estimate. + *******************************************************************************/ + public DistinctValuesPage withEstimate(Boolean estimate) + { + this.estimate = estimate; + return (this); + } + + + + /******************************************************************************* + ** Getter for sampleRows. + *******************************************************************************/ + public Long getSampleRows() + { + return (sampleRows); + } + + + + /******************************************************************************* + ** Fluent setter for sampleRows. + *******************************************************************************/ + public DistinctValuesPage withSampleRows(Long sampleRows) + { + this.sampleRows = sampleRows; + return (this); + } + + + + /******************************************************************************* + ** Getter for truncated. + *******************************************************************************/ + public Boolean getTruncated() + { + return (truncated); + } + + + + /******************************************************************************* + ** Fluent setter for truncated. + *******************************************************************************/ + public DistinctValuesPage withTruncated(Boolean truncated) + { + this.truncated = truncated; + return (this); + } + + + + /******************************************************************************* + ** Getter for rowsScanned. + *******************************************************************************/ + public Long getRowsScanned() + { + return (rowsScanned); + } + + + + /******************************************************************************* + ** Fluent setter for rowsScanned. + *******************************************************************************/ + public DistinctValuesPage withRowsScanned(Long rowsScanned) + { + this.rowsScanned = rowsScanned; + return (this); + } + + + + /******************************************************************************* + ** Getter for asOf. + *******************************************************************************/ + public Instant getAsOf() + { + return (asOf); + } + + + + /******************************************************************************* + ** Fluent setter for asOf. + *******************************************************************************/ + public DistinctValuesPage withAsOf(Instant asOf) + { + this.asOf = asOf; + return (this); + } + + + + /******************************************************************************* + ** Getter for dataVersion. + *******************************************************************************/ + public String getDataVersion() + { + return (dataVersion); + } + + + + /******************************************************************************* + ** Fluent setter for dataVersion. + *******************************************************************************/ + public DistinctValuesPage withDataVersion(String dataVersion) + { + this.dataVersion = dataVersion; + return (this); + } + + + + /******************************************************************************* + ** Getter for warnings. + *******************************************************************************/ + public List getWarnings() + { + return (warnings); + } + + + + /******************************************************************************* + ** Fluent setter for warnings. + *******************************************************************************/ + public DistinctValuesPage withWarnings(List warnings) + { + this.warnings = warnings; + return (this); + } + + + + /******************************************************************************* + ** Getter for fieldName. + *******************************************************************************/ + public String getFieldName() + { + return (fieldName); + } + + + + /******************************************************************************* + ** Fluent setter for fieldName. + *******************************************************************************/ + public DistinctValuesPage withFieldName(String fieldName) + { + this.fieldName = fieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for page. + *******************************************************************************/ + public Integer getPage() + { + return (page); + } + + + + /******************************************************************************* + ** Fluent setter for page. + *******************************************************************************/ + public DistinctValuesPage withPage(Integer page) + { + this.page = page; + return (this); + } + + + + /******************************************************************************* + ** Getter for pageSize. + *******************************************************************************/ + public Integer getPageSize() + { + return (pageSize); + } + + + + /******************************************************************************* + ** Fluent setter for pageSize. + *******************************************************************************/ + public DistinctValuesPage withPageSize(Integer pageSize) + { + this.pageSize = pageSize; + return (this); + } + + + + /******************************************************************************* + ** Getter for hasMore. + *******************************************************************************/ + public Boolean getHasMore() + { + return (hasMore); + } + + + + /******************************************************************************* + ** Fluent setter for hasMore. + *******************************************************************************/ + public DistinctValuesPage withHasMore(Boolean hasMore) + { + this.hasMore = hasMore; + return (this); + } + + + + /******************************************************************************* + ** Getter for totalDistinct. + *******************************************************************************/ + public Long getTotalDistinct() + { + return (totalDistinct); + } + + + + /******************************************************************************* + ** Fluent setter for totalDistinct. + *******************************************************************************/ + public DistinctValuesPage withTotalDistinct(Long totalDistinct) + { + this.totalDistinct = totalDistinct; + return (this); + } + + + + /******************************************************************************* + ** Getter for values. + *******************************************************************************/ + public List getValues() + { + return (values); + } + + + + /******************************************************************************* + ** Fluent setter for values. + *******************************************************************************/ + public DistinctValuesPage withValues(List values) + { + this.values = values; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/DistinctValuesRequest.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/DistinctValuesRequest.java new file mode 100644 index 0000000..82124c0 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/DistinctValuesRequest.java @@ -0,0 +1,188 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; + + +/******************************************************************************* + ** Typed DistinctValuesRequest contract; independent of any presentation framework. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class DistinctValuesRequest implements AnalysisRequest +{ + private String tableName; + private QQueryFilter filter; + private Boolean exact = false; + private String fieldName; + private Integer page = 0; + private Integer pageSize = 100; + private String search; + + + + /******************************************************************************* + ** Getter for tableName. + *******************************************************************************/ + public String getTableName() + { + return (tableName); + } + + + + /******************************************************************************* + ** Fluent setter for tableName. + *******************************************************************************/ + public DistinctValuesRequest withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for filter. + *******************************************************************************/ + public QQueryFilter getFilter() + { + return (filter); + } + + + + /******************************************************************************* + ** Fluent setter for filter. + *******************************************************************************/ + public DistinctValuesRequest withFilter(QQueryFilter filter) + { + this.filter = filter; + return (this); + } + + + + /******************************************************************************* + ** Getter for exact. + *******************************************************************************/ + public Boolean getExact() + { + return (exact); + } + + + + /******************************************************************************* + ** Fluent setter for exact. + *******************************************************************************/ + public DistinctValuesRequest withExact(Boolean exact) + { + this.exact = exact; + return (this); + } + + + + /******************************************************************************* + ** Getter for fieldName. + *******************************************************************************/ + public String getFieldName() + { + return (fieldName); + } + + + + /******************************************************************************* + ** Fluent setter for fieldName. + *******************************************************************************/ + public DistinctValuesRequest withFieldName(String fieldName) + { + this.fieldName = fieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for page. + *******************************************************************************/ + public Integer getPage() + { + return (page); + } + + + + /******************************************************************************* + ** Fluent setter for page. + *******************************************************************************/ + public DistinctValuesRequest withPage(Integer page) + { + this.page = page; + return (this); + } + + + + /******************************************************************************* + ** Getter for pageSize. + *******************************************************************************/ + public Integer getPageSize() + { + return (pageSize); + } + + + + /******************************************************************************* + ** Fluent setter for pageSize. + *******************************************************************************/ + public DistinctValuesRequest withPageSize(Integer pageSize) + { + this.pageSize = pageSize; + return (this); + } + + + + /******************************************************************************* + ** Getter for search. + *******************************************************************************/ + public String getSearch() + { + return (search); + } + + + + /******************************************************************************* + ** Fluent setter for search. + *******************************************************************************/ + public DistinctValuesRequest withSearch(String search) + { + this.search = search; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/Drill.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/Drill.java new file mode 100644 index 0000000..328b347 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/Drill.java @@ -0,0 +1,79 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + + +import java.io.Serializable; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; + + +/******************************************************************************* + ** Typed Drill contract; independent of any presentation framework. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class Drill implements Serializable +{ + private String tableName; + private QQueryFilter filter; + + + + /******************************************************************************* + ** Getter for tableName. + *******************************************************************************/ + public String getTableName() + { + return (tableName); + } + + + + /******************************************************************************* + ** Fluent setter for tableName. + *******************************************************************************/ + public Drill withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for filter. + *******************************************************************************/ + public QQueryFilter getFilter() + { + return (com.kingsrook.qbits.dataanalysis.engine.DrillFilters.restoreTypes(tableName, filter)); + } + + + + /******************************************************************************* + ** Fluent setter for filter. + *******************************************************************************/ + public Drill withFilter(QQueryFilter filter) + { + this.filter = filter; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/DuplicatesRequest.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/DuplicatesRequest.java new file mode 100644 index 0000000..c1bae39 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/DuplicatesRequest.java @@ -0,0 +1,123 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + + +import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; + + +/******************************************************************************* + ** Typed DuplicatesRequest contract; independent of any presentation framework. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class DuplicatesRequest implements AnalysisRequest +{ + private String tableName; + private QQueryFilter filter; + private Boolean exact = false; + private List fields; + + + + /******************************************************************************* + ** Getter for tableName. + *******************************************************************************/ + public String getTableName() + { + return (tableName); + } + + + + /******************************************************************************* + ** Fluent setter for tableName. + *******************************************************************************/ + public DuplicatesRequest withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for filter. + *******************************************************************************/ + public QQueryFilter getFilter() + { + return (filter); + } + + + + /******************************************************************************* + ** Fluent setter for filter. + *******************************************************************************/ + public DuplicatesRequest withFilter(QQueryFilter filter) + { + this.filter = filter; + return (this); + } + + + + /******************************************************************************* + ** Getter for exact. + *******************************************************************************/ + public Boolean getExact() + { + return (exact); + } + + + + /******************************************************************************* + ** Fluent setter for exact. + *******************************************************************************/ + public DuplicatesRequest withExact(Boolean exact) + { + this.exact = exact; + return (this); + } + + + + /******************************************************************************* + ** Getter for fields. + *******************************************************************************/ + public List getFields() + { + return (fields); + } + + + + /******************************************************************************* + ** Fluent setter for fields. + *******************************************************************************/ + public DuplicatesRequest withFields(List fields) + { + this.fields = fields; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/DuplicatesResult.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/DuplicatesResult.java new file mode 100644 index 0000000..6cecfc7 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/DuplicatesResult.java @@ -0,0 +1,279 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + + +import java.io.Serializable; +import java.time.Instant; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; + + +/******************************************************************************* + ** Typed DuplicatesResult contract; independent of any presentation framework. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class DuplicatesResult implements Serializable +{ + private String tableName; + private QQueryFilter filter; + private Boolean estimate = false; + private Long sampleRows = 0L; + private Boolean truncated = false; + private Long rowsScanned = 0L; + private Instant asOf; + private String dataVersion; + private List warnings = new java.util.ArrayList<>(); + private List fields; + private List groups = List.of(); + + + + /******************************************************************************* + ** Getter for tableName. + *******************************************************************************/ + public String getTableName() + { + return (tableName); + } + + + + /******************************************************************************* + ** Fluent setter for tableName. + *******************************************************************************/ + public DuplicatesResult withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for filter. + *******************************************************************************/ + public QQueryFilter getFilter() + { + return (filter); + } + + + + /******************************************************************************* + ** Fluent setter for filter. + *******************************************************************************/ + public DuplicatesResult withFilter(QQueryFilter filter) + { + this.filter = filter; + return (this); + } + + + + /******************************************************************************* + ** Getter for estimate. + *******************************************************************************/ + public Boolean getEstimate() + { + return (estimate); + } + + + + /******************************************************************************* + ** Fluent setter for estimate. + *******************************************************************************/ + public DuplicatesResult withEstimate(Boolean estimate) + { + this.estimate = estimate; + return (this); + } + + + + /******************************************************************************* + ** Getter for sampleRows. + *******************************************************************************/ + public Long getSampleRows() + { + return (sampleRows); + } + + + + /******************************************************************************* + ** Fluent setter for sampleRows. + *******************************************************************************/ + public DuplicatesResult withSampleRows(Long sampleRows) + { + this.sampleRows = sampleRows; + return (this); + } + + + + /******************************************************************************* + ** Getter for truncated. + *******************************************************************************/ + public Boolean getTruncated() + { + return (truncated); + } + + + + /******************************************************************************* + ** Fluent setter for truncated. + *******************************************************************************/ + public DuplicatesResult withTruncated(Boolean truncated) + { + this.truncated = truncated; + return (this); + } + + + + /******************************************************************************* + ** Getter for rowsScanned. + *******************************************************************************/ + public Long getRowsScanned() + { + return (rowsScanned); + } + + + + /******************************************************************************* + ** Fluent setter for rowsScanned. + *******************************************************************************/ + public DuplicatesResult withRowsScanned(Long rowsScanned) + { + this.rowsScanned = rowsScanned; + return (this); + } + + + + /******************************************************************************* + ** Getter for asOf. + *******************************************************************************/ + public Instant getAsOf() + { + return (asOf); + } + + + + /******************************************************************************* + ** Fluent setter for asOf. + *******************************************************************************/ + public DuplicatesResult withAsOf(Instant asOf) + { + this.asOf = asOf; + return (this); + } + + + + /******************************************************************************* + ** Getter for dataVersion. + *******************************************************************************/ + public String getDataVersion() + { + return (dataVersion); + } + + + + /******************************************************************************* + ** Fluent setter for dataVersion. + *******************************************************************************/ + public DuplicatesResult withDataVersion(String dataVersion) + { + this.dataVersion = dataVersion; + return (this); + } + + + + /******************************************************************************* + ** Getter for warnings. + *******************************************************************************/ + public List getWarnings() + { + return (warnings); + } + + + + /******************************************************************************* + ** Fluent setter for warnings. + *******************************************************************************/ + public DuplicatesResult withWarnings(List warnings) + { + this.warnings = warnings; + return (this); + } + + + + /******************************************************************************* + ** Getter for fields. + *******************************************************************************/ + public List getFields() + { + return (fields); + } + + + + /******************************************************************************* + ** Fluent setter for fields. + *******************************************************************************/ + public DuplicatesResult withFields(List fields) + { + this.fields = fields; + return (this); + } + + + + /******************************************************************************* + ** Getter for groups. + *******************************************************************************/ + public List getGroups() + { + return (groups); + } + + + + /******************************************************************************* + ** Fluent setter for groups. + *******************************************************************************/ + public DuplicatesResult withGroups(List groups) + { + this.groups = groups; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/FieldSummary.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/FieldSummary.java new file mode 100644 index 0000000..7acfd9e --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/FieldSummary.java @@ -0,0 +1,255 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + + +import java.io.Serializable; +import java.math.BigDecimal; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + + +/******************************************************************************* + ** Typed FieldSummary contract; independent of any presentation framework. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class FieldSummary implements Serializable +{ + private String field; + private String label; + private String type; + private BigDecimal completeness; + private Long distinct; + private BigDecimal uniqueness; + private Boolean constant; + private Boolean empty; + private Metric nulls; + private Metric blanks; + + + + /******************************************************************************* + ** Getter for field. + *******************************************************************************/ + public String getField() + { + return (field); + } + + + + /******************************************************************************* + ** Fluent setter for field. + *******************************************************************************/ + public FieldSummary withField(String field) + { + this.field = field; + return (this); + } + + + + /******************************************************************************* + ** Getter for label. + *******************************************************************************/ + public String getLabel() + { + return (label); + } + + + + /******************************************************************************* + ** Fluent setter for label. + *******************************************************************************/ + public FieldSummary withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for type. + *******************************************************************************/ + public String getType() + { + return (type); + } + + + + /******************************************************************************* + ** Fluent setter for type. + *******************************************************************************/ + public FieldSummary withType(String type) + { + this.type = type; + return (this); + } + + + + /******************************************************************************* + ** Getter for completeness. + *******************************************************************************/ + public BigDecimal getCompleteness() + { + return (completeness); + } + + + + /******************************************************************************* + ** Fluent setter for completeness. + *******************************************************************************/ + public FieldSummary withCompleteness(BigDecimal completeness) + { + this.completeness = completeness; + return (this); + } + + + + /******************************************************************************* + ** Getter for distinct. + *******************************************************************************/ + public Long getDistinct() + { + return (distinct); + } + + + + /******************************************************************************* + ** Fluent setter for distinct. + *******************************************************************************/ + public FieldSummary withDistinct(Long distinct) + { + this.distinct = distinct; + return (this); + } + + + + /******************************************************************************* + ** Getter for uniqueness. + *******************************************************************************/ + public BigDecimal getUniqueness() + { + return (uniqueness); + } + + + + /******************************************************************************* + ** Fluent setter for uniqueness. + *******************************************************************************/ + public FieldSummary withUniqueness(BigDecimal uniqueness) + { + this.uniqueness = uniqueness; + return (this); + } + + + + /******************************************************************************* + ** Getter for constant. + *******************************************************************************/ + public Boolean getConstant() + { + return (constant); + } + + + + /******************************************************************************* + ** Fluent setter for constant. + *******************************************************************************/ + public FieldSummary withConstant(Boolean constant) + { + this.constant = constant; + return (this); + } + + + + /******************************************************************************* + ** Getter for empty. + *******************************************************************************/ + public Boolean getEmpty() + { + return (empty); + } + + + + /******************************************************************************* + ** Fluent setter for empty. + *******************************************************************************/ + public FieldSummary withEmpty(Boolean empty) + { + this.empty = empty; + return (this); + } + + + + /******************************************************************************* + ** Getter for nulls. + *******************************************************************************/ + public Metric getNulls() + { + return (nulls); + } + + + + /******************************************************************************* + ** Fluent setter for nulls. + *******************************************************************************/ + public FieldSummary withNulls(Metric nulls) + { + this.nulls = nulls; + return (this); + } + + + + /******************************************************************************* + ** Getter for blanks. + *******************************************************************************/ + public Metric getBlanks() + { + return (blanks); + } + + + + /******************************************************************************* + ** Fluent setter for blanks. + *******************************************************************************/ + public FieldSummary withBlanks(Metric blanks) + { + this.blanks = blanks; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/Metric.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/Metric.java new file mode 100644 index 0000000..456f77c --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/Metric.java @@ -0,0 +1,210 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + + +import java.io.Serializable; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + + +/******************************************************************************* + ** Typed Metric contract; independent of any presentation framework. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class Metric implements Serializable +{ + private String key; + private String label; + private Serializable value; + private ValueType valueType; + private String formatted; + private Drill drill; + private Boolean suppressed = false; + private String suppressedReason; + + + + /******************************************************************************* + ** Getter for key. + *******************************************************************************/ + public String getKey() + { + return (key); + } + + + + /******************************************************************************* + ** Fluent setter for key. + *******************************************************************************/ + public Metric withKey(String key) + { + this.key = key; + return (this); + } + + + + /******************************************************************************* + ** Getter for label. + *******************************************************************************/ + public String getLabel() + { + return (label); + } + + + + /******************************************************************************* + ** Fluent setter for label. + *******************************************************************************/ + public Metric withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for value. + *******************************************************************************/ + public Serializable getValue() + { + return (value); + } + + + + /******************************************************************************* + ** Fluent setter for value. + *******************************************************************************/ + public Metric withValue(Serializable value) + { + this.value = value; + return (this); + } + + + + /******************************************************************************* + ** Getter for valueType. + *******************************************************************************/ + public ValueType getValueType() + { + return (valueType); + } + + + + /******************************************************************************* + ** Fluent setter for valueType. + *******************************************************************************/ + public Metric withValueType(ValueType valueType) + { + this.valueType = valueType; + return (this); + } + + + + /******************************************************************************* + ** Getter for formatted. + *******************************************************************************/ + public String getFormatted() + { + return (formatted); + } + + + + /******************************************************************************* + ** Fluent setter for formatted. + *******************************************************************************/ + public Metric withFormatted(String formatted) + { + this.formatted = formatted; + return (this); + } + + + + /******************************************************************************* + ** Getter for drill. + *******************************************************************************/ + public Drill getDrill() + { + return (drill); + } + + + + /******************************************************************************* + ** Fluent setter for drill. + *******************************************************************************/ + public Metric withDrill(Drill drill) + { + this.drill = drill; + return (this); + } + + + + /******************************************************************************* + ** Getter for suppressed. + *******************************************************************************/ + public Boolean getSuppressed() + { + return (suppressed); + } + + + + /******************************************************************************* + ** Fluent setter for suppressed. + *******************************************************************************/ + public Metric withSuppressed(Boolean suppressed) + { + this.suppressed = suppressed; + return (this); + } + + + + /******************************************************************************* + ** Getter for suppressedReason. + *******************************************************************************/ + public String getSuppressedReason() + { + return (suppressedReason); + } + + + + /******************************************************************************* + ** Fluent setter for suppressedReason. + *******************************************************************************/ + public Metric withSuppressedReason(String suppressedReason) + { + this.suppressedReason = suppressedReason; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/NumericProfile.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/NumericProfile.java new file mode 100644 index 0000000..5591abe --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/NumericProfile.java @@ -0,0 +1,101 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + + +import java.io.Serializable; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + + +/******************************************************************************* + ** Typed NumericProfile contract; independent of any presentation framework. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class NumericProfile implements Serializable +{ + private List metrics = new java.util.ArrayList<>(); + private List histogram = List.of(); + private List outliers = List.of(); + + + + /******************************************************************************* + ** Getter for metrics. + *******************************************************************************/ + public List getMetrics() + { + return (metrics); + } + + + + /******************************************************************************* + ** Fluent setter for metrics. + *******************************************************************************/ + public NumericProfile withMetrics(List metrics) + { + this.metrics = metrics; + return (this); + } + + + + /******************************************************************************* + ** Getter for histogram. + *******************************************************************************/ + public List getHistogram() + { + return (histogram); + } + + + + /******************************************************************************* + ** Fluent setter for histogram. + *******************************************************************************/ + public NumericProfile withHistogram(List histogram) + { + this.histogram = histogram; + return (this); + } + + + + /******************************************************************************* + ** Getter for outliers. + *******************************************************************************/ + public List getOutliers() + { + return (outliers); + } + + + + /******************************************************************************* + ** Fluent setter for outliers. + *******************************************************************************/ + public NumericProfile withOutliers(List outliers) + { + this.outliers = outliers; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/Point.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/Point.java new file mode 100644 index 0000000..ab5be5c --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/Point.java @@ -0,0 +1,123 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + + +import java.io.Serializable; +import java.math.BigDecimal; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + + +/******************************************************************************* + ** Typed Point contract; independent of any presentation framework. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class Point implements Serializable +{ + private Serializable x; + private BigDecimal y; + private String label; + private Drill drill; + + + + /******************************************************************************* + ** Getter for x. + *******************************************************************************/ + public Serializable getX() + { + return (x); + } + + + + /******************************************************************************* + ** Fluent setter for x. + *******************************************************************************/ + public Point withX(Serializable x) + { + this.x = x; + return (this); + } + + + + /******************************************************************************* + ** Getter for y. + *******************************************************************************/ + public BigDecimal getY() + { + return (y); + } + + + + /******************************************************************************* + ** Fluent setter for y. + *******************************************************************************/ + public Point withY(BigDecimal y) + { + this.y = y; + return (this); + } + + + + /******************************************************************************* + ** Getter for label. + *******************************************************************************/ + public String getLabel() + { + return (label); + } + + + + /******************************************************************************* + ** Fluent setter for label. + *******************************************************************************/ + public Point withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for drill. + *******************************************************************************/ + public Drill getDrill() + { + return (drill); + } + + + + /******************************************************************************* + ** Fluent setter for drill. + *******************************************************************************/ + public Point withDrill(Drill drill) + { + this.drill = drill; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/RunStatus.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/RunStatus.java new file mode 100644 index 0000000..52fa928 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/RunStatus.java @@ -0,0 +1,189 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + + +import java.io.Serializable; +import java.time.Instant; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + + +/******************************************************************************* + ** Typed RunStatus contract; independent of any presentation framework. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class RunStatus implements Serializable +{ + private String runId; + private String status; + private String error; + private ColumnProfile columnProfile; + private TableProfile tableProfile; + private Instant startedAt; + private Instant finishedAt; + + + + /******************************************************************************* + ** Getter for runId. + *******************************************************************************/ + public String getRunId() + { + return (runId); + } + + + + /******************************************************************************* + ** Fluent setter for runId. + *******************************************************************************/ + public RunStatus withRunId(String runId) + { + this.runId = runId; + return (this); + } + + + + /******************************************************************************* + ** Getter for status. + *******************************************************************************/ + public String getStatus() + { + return (status); + } + + + + /******************************************************************************* + ** Fluent setter for status. + *******************************************************************************/ + public RunStatus withStatus(String status) + { + this.status = status; + return (this); + } + + + + /******************************************************************************* + ** Getter for error. + *******************************************************************************/ + public String getError() + { + return (error); + } + + + + /******************************************************************************* + ** Fluent setter for error. + *******************************************************************************/ + public RunStatus withError(String error) + { + this.error = error; + return (this); + } + + + + /******************************************************************************* + ** Getter for columnProfile. + *******************************************************************************/ + public ColumnProfile getColumnProfile() + { + return (columnProfile); + } + + + + /******************************************************************************* + ** Fluent setter for columnProfile. + *******************************************************************************/ + public RunStatus withColumnProfile(ColumnProfile columnProfile) + { + this.columnProfile = columnProfile; + return (this); + } + + + + /******************************************************************************* + ** Getter for tableProfile. + *******************************************************************************/ + public TableProfile getTableProfile() + { + return (tableProfile); + } + + + + /******************************************************************************* + ** Fluent setter for tableProfile. + *******************************************************************************/ + public RunStatus withTableProfile(TableProfile tableProfile) + { + this.tableProfile = tableProfile; + return (this); + } + + + + /******************************************************************************* + ** Getter for startedAt. + *******************************************************************************/ + public Instant getStartedAt() + { + return (startedAt); + } + + + + /******************************************************************************* + ** Fluent setter for startedAt. + *******************************************************************************/ + public RunStatus withStartedAt(Instant startedAt) + { + this.startedAt = startedAt; + return (this); + } + + + + /******************************************************************************* + ** Getter for finishedAt. + *******************************************************************************/ + public Instant getFinishedAt() + { + return (finishedAt); + } + + + + /******************************************************************************* + ** Fluent setter for finishedAt. + *******************************************************************************/ + public RunStatus withFinishedAt(Instant finishedAt) + { + this.finishedAt = finishedAt; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/Series.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/Series.java new file mode 100644 index 0000000..5a089b4 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/Series.java @@ -0,0 +1,79 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + + +import java.io.Serializable; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + + +/******************************************************************************* + ** Typed Series contract; independent of any presentation framework. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class Series implements Serializable +{ + private String name; + private List points = new java.util.ArrayList<>(); + + + + /******************************************************************************* + ** Getter for name. + *******************************************************************************/ + public String getName() + { + return (name); + } + + + + /******************************************************************************* + ** Fluent setter for name. + *******************************************************************************/ + public Series withName(String name) + { + this.name = name; + return (this); + } + + + + /******************************************************************************* + ** Getter for points. + *******************************************************************************/ + public List getPoints() + { + return (points); + } + + + + /******************************************************************************* + ** Fluent setter for points. + *******************************************************************************/ + public Series withPoints(List points) + { + this.points = points; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/TableProfile.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/TableProfile.java new file mode 100644 index 0000000..33ece55 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/TableProfile.java @@ -0,0 +1,323 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + + +import java.io.Serializable; +import java.time.Instant; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; + + +/******************************************************************************* + ** Typed TableProfile contract; independent of any presentation framework. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class TableProfile implements Serializable +{ + private String tableName; + private QQueryFilter filter; + private Boolean estimate = false; + private Long sampleRows = 0L; + private Boolean truncated = false; + private Long rowsScanned = 0L; + private Instant asOf; + private String dataVersion; + private List warnings = new java.util.ArrayList<>(); + private Metric rowCount; + private List fields = new java.util.ArrayList<>(); + private Chart heatmap; + private List candidateKeys = new java.util.ArrayList<>(); + + + + /******************************************************************************* + ** Getter for tableName. + *******************************************************************************/ + public String getTableName() + { + return (tableName); + } + + + + /******************************************************************************* + ** Fluent setter for tableName. + *******************************************************************************/ + public TableProfile withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for filter. + *******************************************************************************/ + public QQueryFilter getFilter() + { + return (filter); + } + + + + /******************************************************************************* + ** Fluent setter for filter. + *******************************************************************************/ + public TableProfile withFilter(QQueryFilter filter) + { + this.filter = filter; + return (this); + } + + + + /******************************************************************************* + ** Getter for estimate. + *******************************************************************************/ + public Boolean getEstimate() + { + return (estimate); + } + + + + /******************************************************************************* + ** Fluent setter for estimate. + *******************************************************************************/ + public TableProfile withEstimate(Boolean estimate) + { + this.estimate = estimate; + return (this); + } + + + + /******************************************************************************* + ** Getter for sampleRows. + *******************************************************************************/ + public Long getSampleRows() + { + return (sampleRows); + } + + + + /******************************************************************************* + ** Fluent setter for sampleRows. + *******************************************************************************/ + public TableProfile withSampleRows(Long sampleRows) + { + this.sampleRows = sampleRows; + return (this); + } + + + + /******************************************************************************* + ** Getter for truncated. + *******************************************************************************/ + public Boolean getTruncated() + { + return (truncated); + } + + + + /******************************************************************************* + ** Fluent setter for truncated. + *******************************************************************************/ + public TableProfile withTruncated(Boolean truncated) + { + this.truncated = truncated; + return (this); + } + + + + /******************************************************************************* + ** Getter for rowsScanned. + *******************************************************************************/ + public Long getRowsScanned() + { + return (rowsScanned); + } + + + + /******************************************************************************* + ** Fluent setter for rowsScanned. + *******************************************************************************/ + public TableProfile withRowsScanned(Long rowsScanned) + { + this.rowsScanned = rowsScanned; + return (this); + } + + + + /******************************************************************************* + ** Getter for asOf. + *******************************************************************************/ + public Instant getAsOf() + { + return (asOf); + } + + + + /******************************************************************************* + ** Fluent setter for asOf. + *******************************************************************************/ + public TableProfile withAsOf(Instant asOf) + { + this.asOf = asOf; + return (this); + } + + + + /******************************************************************************* + ** Getter for dataVersion. + *******************************************************************************/ + public String getDataVersion() + { + return (dataVersion); + } + + + + /******************************************************************************* + ** Fluent setter for dataVersion. + *******************************************************************************/ + public TableProfile withDataVersion(String dataVersion) + { + this.dataVersion = dataVersion; + return (this); + } + + + + /******************************************************************************* + ** Getter for warnings. + *******************************************************************************/ + public List getWarnings() + { + return (warnings); + } + + + + /******************************************************************************* + ** Fluent setter for warnings. + *******************************************************************************/ + public TableProfile withWarnings(List warnings) + { + this.warnings = warnings; + return (this); + } + + + + /******************************************************************************* + ** Getter for rowCount. + *******************************************************************************/ + public Metric getRowCount() + { + return (rowCount); + } + + + + /******************************************************************************* + ** Fluent setter for rowCount. + *******************************************************************************/ + public TableProfile withRowCount(Metric rowCount) + { + this.rowCount = rowCount; + return (this); + } + + + + /******************************************************************************* + ** Getter for fields. + *******************************************************************************/ + public List getFields() + { + return (fields); + } + + + + /******************************************************************************* + ** Fluent setter for fields. + *******************************************************************************/ + public TableProfile withFields(List fields) + { + this.fields = fields; + return (this); + } + + + + /******************************************************************************* + ** Getter for heatmap. + *******************************************************************************/ + public Chart getHeatmap() + { + return (heatmap); + } + + + + /******************************************************************************* + ** Fluent setter for heatmap. + *******************************************************************************/ + public TableProfile withHeatmap(Chart heatmap) + { + this.heatmap = heatmap; + return (this); + } + + + + /******************************************************************************* + ** Getter for candidateKeys. + *******************************************************************************/ + public List getCandidateKeys() + { + return (candidateKeys); + } + + + + /******************************************************************************* + ** Fluent setter for candidateKeys. + *******************************************************************************/ + public TableProfile withCandidateKeys(List candidateKeys) + { + this.candidateKeys = candidateKeys; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/TableProfileRequest.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/TableProfileRequest.java new file mode 100644 index 0000000..9a7565f --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/TableProfileRequest.java @@ -0,0 +1,123 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + + +import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; + + +/******************************************************************************* + ** Typed TableProfileRequest contract; independent of any presentation framework. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class TableProfileRequest implements AnalysisRequest +{ + private String tableName; + private QQueryFilter filter; + private Boolean exact = false; + private List fields; + + + + /******************************************************************************* + ** Getter for tableName. + *******************************************************************************/ + public String getTableName() + { + return (tableName); + } + + + + /******************************************************************************* + ** Fluent setter for tableName. + *******************************************************************************/ + public TableProfileRequest withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for filter. + *******************************************************************************/ + public QQueryFilter getFilter() + { + return (filter); + } + + + + /******************************************************************************* + ** Fluent setter for filter. + *******************************************************************************/ + public TableProfileRequest withFilter(QQueryFilter filter) + { + this.filter = filter; + return (this); + } + + + + /******************************************************************************* + ** Getter for exact. + *******************************************************************************/ + public Boolean getExact() + { + return (exact); + } + + + + /******************************************************************************* + ** Fluent setter for exact. + *******************************************************************************/ + public TableProfileRequest withExact(Boolean exact) + { + this.exact = exact; + return (this); + } + + + + /******************************************************************************* + ** Getter for fields. + *******************************************************************************/ + public List getFields() + { + return (fields); + } + + + + /******************************************************************************* + ** Fluent setter for fields. + *******************************************************************************/ + public TableProfileRequest withFields(List fields) + { + this.fields = fields; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/TextProfile.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/TextProfile.java new file mode 100644 index 0000000..6f9ad34 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/TextProfile.java @@ -0,0 +1,123 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + + +import java.io.Serializable; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + + +/******************************************************************************* + ** Typed TextProfile contract; independent of any presentation framework. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class TextProfile implements Serializable +{ + private List metrics = new java.util.ArrayList<>(); + private List histogram = List.of(); + private List patterns = List.of(); + private Long sampleRows; + + + + /******************************************************************************* + ** Getter for metrics. + *******************************************************************************/ + public List getMetrics() + { + return (metrics); + } + + + + /******************************************************************************* + ** Fluent setter for metrics. + *******************************************************************************/ + public TextProfile withMetrics(List metrics) + { + this.metrics = metrics; + return (this); + } + + + + /******************************************************************************* + ** Getter for histogram. + *******************************************************************************/ + public List getHistogram() + { + return (histogram); + } + + + + /******************************************************************************* + ** Fluent setter for histogram. + *******************************************************************************/ + public TextProfile withHistogram(List histogram) + { + this.histogram = histogram; + return (this); + } + + + + /******************************************************************************* + ** Getter for patterns. + *******************************************************************************/ + public List getPatterns() + { + return (patterns); + } + + + + /******************************************************************************* + ** Fluent setter for patterns. + *******************************************************************************/ + public TextProfile withPatterns(List patterns) + { + this.patterns = patterns; + return (this); + } + + + + /******************************************************************************* + ** Getter for sampleRows. + *******************************************************************************/ + public Long getSampleRows() + { + return (sampleRows); + } + + + + /******************************************************************************* + ** Fluent setter for sampleRows. + *******************************************************************************/ + public TextProfile withSampleRows(Long sampleRows) + { + this.sampleRows = sampleRows; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/ValueCount.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/ValueCount.java new file mode 100644 index 0000000..29e5271 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/ValueCount.java @@ -0,0 +1,145 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + + +import java.io.Serializable; +import java.math.BigDecimal; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + + +/******************************************************************************* + ** Typed ValueCount contract; independent of any presentation framework. + *******************************************************************************/ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class ValueCount implements Serializable +{ + private Serializable value; + private String label; + private Long count; + private BigDecimal share; + private Drill drill; + + + + /******************************************************************************* + ** Getter for value. + *******************************************************************************/ + public Serializable getValue() + { + return (value); + } + + + + /******************************************************************************* + ** Fluent setter for value. + *******************************************************************************/ + public ValueCount withValue(Serializable value) + { + this.value = value; + return (this); + } + + + + /******************************************************************************* + ** Getter for label. + *******************************************************************************/ + public String getLabel() + { + return (label); + } + + + + /******************************************************************************* + ** Fluent setter for label. + *******************************************************************************/ + public ValueCount withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for count. + *******************************************************************************/ + public Long getCount() + { + return (count); + } + + + + /******************************************************************************* + ** Fluent setter for count. + *******************************************************************************/ + public ValueCount withCount(Long count) + { + this.count = count; + return (this); + } + + + + /******************************************************************************* + ** Getter for share. + *******************************************************************************/ + public BigDecimal getShare() + { + return (share); + } + + + + /******************************************************************************* + ** Fluent setter for share. + *******************************************************************************/ + public ValueCount withShare(BigDecimal share) + { + this.share = share; + return (this); + } + + + + /******************************************************************************* + ** Getter for drill. + *******************************************************************************/ + public Drill getDrill() + { + return (drill); + } + + + + /******************************************************************************* + ** Fluent setter for drill. + *******************************************************************************/ + public ValueCount withDrill(Drill drill) + { + this.drill = drill; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/result/ValueType.java b/src/main/java/com/kingsrook/qbits/dataanalysis/result/ValueType.java new file mode 100644 index 0000000..17f0b89 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/result/ValueType.java @@ -0,0 +1,30 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.result; + + +/******************************************************************************* + ** Semantic ValueType values. + *******************************************************************************/ +public enum ValueType +{ + COUNT, DECIMAL, PERCENT, DATE, DATE_TIME, DURATION, TEXT +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/run/AnalysisRunService.java b/src/main/java/com/kingsrook/qbits/dataanalysis/run/AnalysisRunService.java new file mode 100644 index 0000000..ddecca4 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/run/AnalysisRunService.java @@ -0,0 +1,499 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.run; + +import java.io.Serializable; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.Executor; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.kingsrook.qbits.dataanalysis.DataAnalysisService; +import com.kingsrook.qbits.dataanalysis.engine.AnalysisBackend; +import com.kingsrook.qbits.dataanalysis.engine.DrillFilters; +import com.kingsrook.qbits.dataanalysis.engine.EffectiveConfig; +import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables; +import com.kingsrook.qbits.dataanalysis.result.AnalysisRequest; +import com.kingsrook.qbits.dataanalysis.result.ColumnProfile; +import com.kingsrook.qbits.dataanalysis.result.ColumnProfileRequest; +import com.kingsrook.qbits.dataanalysis.result.RunStatus; +import com.kingsrook.qbits.dataanalysis.result.TableProfile; +import com.kingsrook.qbits.dataanalysis.result.TableProfileRequest; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.CapturedContext; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + +/******************************************************************************* + ** Persistent owner-scoped runs. Results are guarded before being serialized. + ** Cache identity includes session policy, effective config and data version. + *******************************************************************************/ +public class AnalysisRunService +{ + private static final QLogger LOG = QLogger.getLogger(AnalysisRunService.class); + private final DataAnalysisService service; + private final Executor executor; + + /******************************************************************************* + ** Use a host-managed executor without inheriting a caller transaction. + *******************************************************************************/ + public AnalysisRunService(DataAnalysisService service, Executor executor) + { + this.service = service; + this.executor = executor; + } + + + + /******************************************************************************* + ** Estimate is transient; exact results are persisted and reused. + *******************************************************************************/ + public ColumnProfile columnProfile(ColumnProfileRequest request) throws QException + { + service.validate(request); + AnalysisBackend backend = service.backend(request.getTableName()); + AnalysisBackend.Scope scope = backend.scope(request.getTableName(), request.getFilter(), Boolean.TRUE.equals(request.getExact())); + if(scope.estimate()) + { + return (service.compute(request, backend, scope).withDataVersion(DataVersion.of(backend, request.getTableName(), request.getFilter()))); + } + return (exact(request).getColumnProfile()); + } + + + + /******************************************************************************* + ** Table profiles share the same lifecycle and cache isolation. + *******************************************************************************/ + public TableProfile tableProfile(TableProfileRequest request) throws QException + { + service.validate(request); + AnalysisBackend backend = service.backend(request.getTableName()); + AnalysisBackend.Scope scope = backend.scope(request.getTableName(), request.getFilter(), Boolean.TRUE.equals(request.getExact())); + if(scope.estimate()) + { + return (service.compute(request, backend, scope).withDataVersion(DataVersion.of(backend, request.getTableName(), request.getFilter()))); + } + return (exact(request).getTableProfile()); + } + + + + /******************************************************************************* + ** Complete synchronously for already-small populations and explicit exact calls. + *******************************************************************************/ + private RunStatus exact(AnalysisRequest request) throws QException + { + AnalysisRequest snapshot = snapshot(request); + QRecord cached = cached(snapshot); + if(cached != null) + { + return (decode(cached)); + } + QRecord run = insert(snapshot); + execute(snapshot, run); + RunStatus result = getRun(run.getValueString("id")); + if(!"COMPLETE".equals(result.getStatus())) + { + throw new QException("Analysis failed; inspect the run status and retry."); + } + return (result); + } + + + + /******************************************************************************* + ** Snapshot both request and session before dispatching work. No ambient + ** transaction or privileged action stack is copied into the worker. + *******************************************************************************/ + public String startExactRun(AnalysisRequest request) throws QException + { + service.validate(request); + AnalysisRequest snapshot = snapshot(request); + service.validate(snapshot); + QRecord cached = cached(snapshot); + if(cached != null) + { + return (cached.getValueString("id")); + } + QRecord run = insert(snapshot); + CapturedContext context; + try + { + context = new CapturedContext(QContext.getQInstance(), QContext.getQSession().clone()); + } + catch(CloneNotSupportedException e) + { + fail(run); + throw new QException("Could not capture the analysis session.", e); + } + try + { + executor.execute(() -> QContext.withTemporaryContext(context, () -> execute(snapshot, run))); + } + catch(RuntimeException e) + { + fail(run); + throw new QException("The analysis executor could not accept this run.", e); + } + return (run.getValueString("id")); + } + + + + /******************************************************************************* + ** Only completed, fresh, owner-and-policy-matching rows qualify as cache hits. + *******************************************************************************/ + private QRecord cached(AnalysisRequest request) throws QException + { + service.validate(request); + EffectiveConfig config = service.getConfigResolver().resolve(request.getTableName()); + if(config.getCacheTtlMinutes() == 0) + { + return (null); + } + QQueryFilter filter = DrillFilters.condition("user_id", QCriteriaOperator.EQUALS, userId()) + .withCriteria("request_hash", QCriteriaOperator.EQUALS, hash(request)) + .withCriteria("data_version", QCriteriaOperator.EQUALS, DataVersion.of(service.backend(request.getTableName()), request.getTableName(), request.getFilter())) + .withCriteria("status", QCriteriaOperator.EQUALS, "COMPLETE") + .withCriteria("finished_at", QCriteriaOperator.GREATER_THAN, Instant.now().minus(config.getCacheTtlMinutes(), ChronoUnit.MINUTES)) + .withCriteria("finished_at", QCriteriaOperator.LESS_THAN_OR_EQUALS, Instant.now()) + .withOrderBy(new QFilterOrderBy("id", false)).withLimit(1); + List rows = query(filter); + return (rows.isEmpty() ? null : rows.getFirst()); + } + + + + /******************************************************************************* + ** Run inserts and updates are ordinary user actions under owner locks. + *******************************************************************************/ + private QRecord insert(AnalysisRequest request) throws QException + { + QRecord run = new QRecord().withValue("table_name", request.getTableName()).withValue("operation", operation(request)) + .withValue("request_hash", hash(request)).withValue("filter_hash", digest(JsonUtils.toJson(DrillFilters.copy(request.getFilter())))) + .withValue("data_version", DataVersion.of(service.backend(request.getTableName()), request.getTableName(), request.getFilter())) + .withValue("status", "RUNNING").withValue("started_at", Instant.now()).withValue("user_id", userId()); + InsertInput input = new InsertInput(DataAnalysisTables.RUN).withRecords(List.of(run)); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + QRecord saved = new InsertAction().execute(input).getRecords().getFirst(); + check(saved); + return (saved); + } + + + + /******************************************************************************* + ** Failure text is intentionally generic; backend errors may contain user data. + *******************************************************************************/ + private void execute(AnalysisRequest request, QRecord run) + { + try + { + service.validate(request); + AnalysisBackend backend = service.backend(request.getTableName()); + AnalysisBackend.Scope scope = backend.scope(request.getTableName(), request.getFilter(), true); + Serializable result; + if(request instanceof ColumnProfileRequest column) + { + result = service.compute(column, backend, scope).withDataVersion(run.getValueString("data_version")); + } + else + { + result = service.compute((TableProfileRequest) request, backend, scope).withDataVersion(run.getValueString("data_version")); + } + if(!hash(request).equals(run.getValueString("request_hash"))) + { + throw new QException("Analysis policy changed during the run."); + } + update(new QRecord().withValue("id", run.getValue("id")).withValue("status", "COMPLETE").withValue("finished_at", Instant.now()) + .withValue("rows_scanned", scope.rowsScanned()).withValue("sampled", false).withValue("result_json", JsonUtils.toJson(result))); + } + catch(Exception e) + { + LOG.warn("Analysis execution failed; recording a safe failure status.", e); + fail(run); + } + } + + + + /******************************************************************************* + ** Persist safe failure status; persistence failures are logged separately. + *******************************************************************************/ + private void fail(QRecord run) + { + try + { + update(new QRecord().withValue("id", run.getValue("id")).withValue("status", "FAILED").withValue("finished_at", Instant.now()) + .withValue("result_json", null).withValue("error", "Analysis could not be completed. Check permissions and settings, then retry.")); + } + catch(QException e) + { + LOG.warn("Could not persist analysis failure status.", e); + } + } + + + + /******************************************************************************* + ** Recheck source-table access and policy before releasing historical data. + *******************************************************************************/ + public RunStatus getRun(String id) throws QException + { + Integer key; + try + { + key = Integer.valueOf(id); + } + catch(RuntimeException e) + { + throw new QException("Invalid analysis run id."); + } + List rows = query(DrillFilters.condition("id", QCriteriaOperator.EQUALS, key).withCriteria("user_id", QCriteriaOperator.EQUALS, userId()).withLimit(1)); + if(rows.isEmpty()) + { + throw new QPermissionDeniedException("Analysis run is unavailable."); + } + QRecord row = rows.getFirst(); + service.capabilities(row.getValueString("table_name")); + RunStatus status = decode(row); + if("COMPLETE".equals(status.getStatus())) + { + AnalysisRequest request = status.getColumnProfile() != null + ? new ColumnProfileRequest().withTableName(status.getColumnProfile().getTableName()).withFieldName(status.getColumnProfile().getFieldName()).withFilter(status.getColumnProfile().getFilter()).withExact(true) + : new TableProfileRequest().withTableName(status.getTableProfile().getTableName()).withFields(status.getTableProfile().getFields().stream().map(f -> f.getField()).toList()).withFilter(status.getTableProfile().getFilter()).withExact(true); + service.validate(request); + if(!hash(request).equals(row.getValueString("request_hash"))) + { + throw new QPermissionDeniedException("Analysis policy changed; request a new run."); + } + } + return (status); + } + + + + /******************************************************************************* + ** Deserialize typed payloads through core JsonUtils. + *******************************************************************************/ + private RunStatus decode(QRecord row) throws QException + { + RunStatus status = new RunStatus().withRunId(row.getValueString("id")).withStatus(row.getValueString("status")).withError(row.getValueString("error")) + .withStartedAt(row.getValueInstant("started_at")).withFinishedAt(row.getValueInstant("finished_at")); + try + { + if("COMPLETE".equals(status.getStatus())) + { + if("COLUMN_PROFILE".equals(row.getValueString("operation"))) + { + status.withColumnProfile(JsonUtils.toObject(row.getValueString("result_json"), ColumnProfile.class)); + } + else + { + status.withTableProfile(JsonUtils.toObject(row.getValueString("result_json"), TableProfile.class)); + } + } + } + catch(java.io.IOException e) + { + throw new QException("Stored analysis result could not be read.", e); + } + return (status); + } + + + + /******************************************************************************* + ** Read only through enforced core actions, including heavy result JSON. + *******************************************************************************/ + private static List query(QQueryFilter filter) throws QException + { + QueryInput input = new QueryInput(DataAnalysisTables.RUN).withFilter(filter).withShouldFetchHeavyFields(true); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + return (new QueryAction().execute(input).getRecords()); + } + + + + /******************************************************************************* + ** Write only supplied run fields and check core record-level failures. + *******************************************************************************/ + private static void update(QRecord record) throws QException + { + UpdateInput input = new UpdateInput(DataAnalysisTables.RUN).withRecords(List.of(record)); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + check(new UpdateAction().execute(input).getRecords().getFirst()); + } + + + + /******************************************************************************* + ** Core can return record-level errors without throwing. + *******************************************************************************/ + private static void check(QRecord record) throws QException + { + if(record.getErrors() != null && !record.getErrors().isEmpty()) + { + throw new QException("Analysis run could not be stored."); + } + } + + + + /******************************************************************************* + ** An authenticated identity is mandatory even on unprotected host tables. + *******************************************************************************/ + private static String userId() throws QException + { + if(QContext.getQSession() == null || QContext.getQSession().getUser() == null || QContext.getQSession().getUser().getIdReference() == null) + { + throw new QPermissionDeniedException("Sign in to run an analysis."); + } + return (QContext.getQSession().getUser().getIdReference()); + } + + + + /******************************************************************************* + ** Exact requests have canonical selected fields and no pagination metadata. + *******************************************************************************/ + private AnalysisRequest snapshot(AnalysisRequest request) throws QException + { + if(request instanceof ColumnProfileRequest column) + { + return (new ColumnProfileRequest().withTableName(column.getTableName()).withFieldName(column.getFieldName()).withFilter(DrillFilters.restoreTypes(column.getTableName(), column.getFilter())).withExact(true)); + } + if(request instanceof TableProfileRequest table) + { + List fields = service.capabilities(table.getTableName()).getFields().stream() + .filter(f -> table.getFields() == null || table.getFields().contains(f)).sorted().toList(); + return (new TableProfileRequest().withTableName(table.getTableName()).withFields(fields).withFilter(DrillFilters.restoreTypes(table.getTableName(), table.getFilter())).withExact(true)); + } + throw new QException("Exact runs support column and table profiles only."); + } + + + + /******************************************************************************* + ** Stable operation identity. + *******************************************************************************/ + private static String operation(AnalysisRequest request) + { + return (request instanceof ColumnProfileRequest ? "COLUMN_PROFILE" : "TABLE_PROFILE"); + } + + + + /******************************************************************************* + ** Normalize JSON objects and commutative filter lists, then SHA-256. Policy + ** values are hashed, never persisted as plaintext in the cache identity. + *******************************************************************************/ + private String hash(AnalysisRequest request) throws QException + { + Map identity = new LinkedHashMap<>(); + identity.put("request", snapshot(request)); + identity.put("operation", operation(request)); + identity.put("user", userId()); + identity.put("permissions", QContext.getQSession().getPermissions().stream().sorted().toList()); + identity.put("keys", QContext.getQSession().getSecurityKeyValues()); + identity.put("zone", ValueUtils.getSessionOrInstanceZoneId().getId()); + identity.put("config", service.getConfigResolver().resolve(request.getTableName())); + identity.put("fields", service.capabilities(request.getTableName()).getFields().stream().sorted().toList()); + identity.put("policy", com.kingsrook.qbits.dataanalysis.engine.AnalysisLabels.policy(request.getTableName())); + identity.put("backendVariants", QContext.getQSession().getBackendVariants()); + try + { + JsonNode json = new ObjectMapper().readTree(JsonUtils.toJson(identity)); + return (digest(JsonUtils.toJson(normalize(json, "")))); + } + catch(java.io.IOException e) + { + throw new QException("Could not normalize analysis request.", e); + } + } + + + + /******************************************************************************* + ** Preserve ordered parameter arrays, but sort commutative filter clauses. + *******************************************************************************/ + private static Object normalize(JsonNode node, String key) + { + if(node.isObject()) + { + Map result = new TreeMap<>(); + node.fields().forEachRemaining(e -> result.put(e.getKey(), normalize(e.getValue(), e.getKey()))); + return (result); + } + if(node.isArray()) + { + List result = new ArrayList<>(); + node.forEach(value -> result.add(normalize(value, key))); + if(List.of("criteria", "subFilters", "analyzableFields", "excludedFields").contains(key)) + { + result.sort(Comparator.comparing(JsonUtils::toJson)); + } + return (result); + } + return (node); + } + + + + /******************************************************************************* + ** SHA-256 is guaranteed by the Java platform. + *******************************************************************************/ + private static String digest(String value) throws QException + { + try + { + return (HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8)))); + } + catch(java.security.NoSuchAlgorithmException e) + { + throw new QException("SHA-256 unavailable.", e); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/run/DataVersion.java b/src/main/java/com/kingsrook/qbits/dataanalysis/run/DataVersion.java new file mode 100644 index 0000000..9e4a325 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/run/DataVersion.java @@ -0,0 +1,69 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.run; + +import java.io.Serializable; +import java.util.Locale; +import com.kingsrook.qbits.dataanalysis.engine.AnalysisBackend; +import com.kingsrook.qqq.backend.core.actions.permissions.FieldSecurityLockHelper; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + +/******************************************************************************* + ** Cheap filtered data version; deliberately not a transactional snapshot guarantee. + *******************************************************************************/ +public final class DataVersion +{ + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private DataVersion() + { + } + + + + /******************************************************************************* + ** Never bypass a lock on a modification field to construct a cache key. + *******************************************************************************/ + public static String of(AnalysisBackend backend, String table, QQueryFilter filter) throws QException + { + QTableMetaData metadata = QContext.getQInstance().getTable(table); + String versionField = metadata.getFields().values().stream().filter(FieldSecurityLockHelper::isFieldAccessible) + .filter(f -> isModificationField(f.getName())).map(QFieldMetaData::getName).findFirst().orElse(metadata.getPrimaryKeyField()); + Serializable maximum = backend.extrema(table, filter, versionField).get("max"); + return (backend.count(table, filter) + ":" + versionField + ":" + maximum); + } + + + + /******************************************************************************* + ** Conventional host modification timestamps, in camel or snake case. + *******************************************************************************/ + private static boolean isModificationField(String field) + { + String name = field.replace("_", "").toLowerCase(Locale.ROOT); + return (name.equals("modifydate") || name.equals("modifiedat") || name.equals("updatedat") || name.equals("modifieddate")); + } +} diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisHomeTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisHomeTest.java index 0cfa1cd..68c7fd9 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisHomeTest.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisHomeTest.java @@ -61,7 +61,7 @@ class DataAnalysisHomeTest QInstance instance = TestSupport.instance(); QContext.init(instance, TestSupport.user("alice").withPermissions("data_analysis.saved.read", "data_analysis.watch.read", "data_analysis.run.read")); String html = DataAnalysisHome.render(Instant.now()); - assertThat(html).contains("Mine: 0", "Starred by me: 0", "No saved analyses yet: open any table and choose Analyze on a column header", + assertThat(html).contains("Mine: 0", "Starred by me: 0", "No saved analyses yet", "No active watches", "No completed or failed runs", "View all"); assertThat(new DataAnalysisHome().render(new RenderWidgetInput())).isNotNull(); QContext.init(instance, TestSupport.user("alice")); diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java new file mode 100644 index 0000000..a2421cf --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java @@ -0,0 +1,656 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.util.List; +import java.util.Map; +import java.util.Set; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.kingsrook.qbits.dataanalysis.engine.AnalysisBackend; +import com.kingsrook.qbits.dataanalysis.engine.ColumnProfiler; +import com.kingsrook.qbits.dataanalysis.engine.DrillFilters; +import com.kingsrook.qbits.dataanalysis.engine.StandInAnalysisBackend; +import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables; +import com.kingsrook.qbits.dataanalysis.result.ColumnProfile; +import com.kingsrook.qbits.dataanalysis.result.ColumnProfileRequest; +import com.kingsrook.qbits.dataanalysis.result.DistinctValuesRequest; +import com.kingsrook.qbits.dataanalysis.result.Drill; +import com.kingsrook.qbits.dataanalysis.result.DuplicatesRequest; +import com.kingsrook.qbits.dataanalysis.result.Metric; +import com.kingsrook.qbits.dataanalysis.result.RunStatus; +import com.kingsrook.qbits.dataanalysis.result.TableProfileRequest; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; +import com.kingsrook.qqq.backend.core.actions.tables.CountAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.within; + +/******************************************************************************* + ** Shared backend acceptance tests, including a generic drill equivalence walker. + *******************************************************************************/ +abstract class EngineContractTest extends EngineFixture +{ + /******************************************************************************* + ** All numeric moments and percentiles agree to 1e-9. + *******************************************************************************/ + @Test + void numericProfile() throws Exception + { + ColumnProfile result = service.columnProfile(column("amount")); + assertMetric(result.getCounts(), "rows", "12"); + assertMetric(result.getCounts(), "nonNull", "11"); + assertMetric(result.getCounts(), "nulls", "1"); + assertMetric(result.getCounts(), "distinct", "8"); + Map expected = Map.ofEntries(Map.entry("min", "-10"), Map.entry("max", "1000"), Map.entry("sum", "1170"), + Map.entry("mean", "106.36363636363636"), Map.entry("p1", "-9"), Map.entry("p5", "-5"), Map.entry("p25", "5"), + Map.entry("p50", "20"), Map.entry("p75", "35"), Map.entry("p95", "525"), Map.entry("p99", "905"), + Map.entry("zeros", "2"), Map.entry("negatives", "1"), Map.entry("outliers", "1")); + for(var entry : expected.entrySet()) + { + assertMetric(result.getNumeric().getMetrics(), entry.getKey(), entry.getValue()); + } + double[] values = {-10, 0, 0, 10, 10, 20, 20, 30, 40, 50, 1000}; + double mean = 1170.0 / 11; + double variance = java.util.Arrays.stream(values).map(v -> Math.pow(v - mean, 2)).sum() / 10; + assertMetric(result.getNumeric().getMetrics(), "stddev", Double.toString(Math.sqrt(variance))); + assertThat(result.getNumeric().getHistogram()).hasSize(22); + assertThat(result.getNumeric().getHistogram().stream().mapToLong(v -> v.getCount()).sum()).isEqualTo(11); + assertThat(result.getNumeric().getOutliers()).singleElement().satisfies(v -> assertThat(new BigDecimal(v.getValue().toString())).isEqualByComparingTo("1000")); + assertThat(result.getChart().getKind().name()).isEqualTo("HISTOGRAM"); + assertDrills(result); + assertDrills(service.columnProfile(column("quantity"))); + } + + /******************************************************************************* + ** Text nulls, whitespace, lengths, shapes, enum labels and FK labels. + *******************************************************************************/ + @Test + void textAndLabels() throws Exception + { + ColumnProfile text = service.columnProfile(column("note")); + assertMetric(text.getCounts(), "nulls", "1"); + assertMetric(text.getCounts(), "blanks", "4"); + assertMetric(text.getCounts(), "distinct", "10"); + assertMetric(text.getText().getMetrics(), "minLength", "0"); + assertMetric(text.getText().getMetrics(), "maxLength", "5"); + assertMetric(text.getText().getMetrics(), "meanLength", Double.toString(28.0 / 11)); + assertThat(text.getText().getPatterns()).anySatisfy(v -> + { + assertThat(v.getLabel()).isEqualTo("A-9"); + assertThat(v.getCount()).isEqualTo(3); + }); + assertDrills(text); + ColumnProfile customer = service.columnProfile(column("customer")); + assertThat(customer.getTopValues()).extracting(v -> v.getLabel()).containsExactly("Alice Co", "Bob Co"); + assertThat(customer.getChart().getKind().name()).isEqualTo("BAR"); + assertDrills(customer); + ColumnProfile status = service.columnProfile(column("status")); + assertThat(status.getTopValues()).extracting(v -> v.getLabel()).containsExactlyInAnyOrder("New order", "Paid order"); + assertDrills(status); + } + + /******************************************************************************* + ** Date gaps, timezone buckets, moving average and boolean counts. + *******************************************************************************/ + @Test + void datesAndBooleans() throws Exception + { + ColumnProfile date = service.columnProfile(column("order_date")); + assertThat(date.getDates().getUnit()).isEqualTo("DAY"); + assertThat(date.getDates().getBuckets()).extracting(v -> v.getCount()).containsExactly(4L, 0L, 8L); + assertThat(date.getDates().getGaps()).hasSize(1); + assertThat(date.getChart().getSeries()).hasSize(2); + assertMetric(date.getDates().getMetrics(), "range", "2"); + assertDrills(date); + ColumnProfile instant = service.columnProfile(column("created")); + assertThat(instant.getDates().getHourOfDay()).extracting(v -> v.getValue().toString()).containsExactlyInAnyOrder("6", "12"); + assertDrills(instant); + ColumnProfile bool = service.columnProfile(column("paid")); + assertMetric(bool.getBooleans(), "true", "6"); + assertMetric(bool.getBooleans(), "false", "5"); + assertMetric(bool.getBooleans(), "nulls", "1"); + assertDrills(bool); + } + + /******************************************************************************* + ** Completeness, field and pair keys and duplicate tuples. + *******************************************************************************/ + @Test + void tableAndDuplicates() throws Exception + { + var table = service.tableProfile(new TableProfileRequest().withTableName("orders")); + assertThat(table.getFields()).noneMatch(f -> f.getField().equals("secret")); + assertThat(table.getFields()).anySatisfy(f -> + { + assertThat(f.getField()).isEqualTo("note"); + assertThat(f.getCompleteness().doubleValue()).isCloseTo(8.0 / 12, within(1e-9)); + }); + assertThat(table.getCandidateKeys()).anyMatch(k -> k.getFields().equals(List.of("id"))); + var pairs = service.tableProfile(new TableProfileRequest().withTableName("orders").withFields(List.of("status", "quantity"))); + assertThat(pairs.getCandidateKeys()).anyMatch(k -> Set.copyOf(k.getFields()).equals(Set.of("status", "quantity"))); + assertDrills(table); + assertDrills(pairs); + var duplicates = service.duplicates(new DuplicatesRequest().withTableName("orders").withFields(List.of("amount"))); + assertThat(duplicates.getGroups()).extracting(v -> v.getCount()).containsExactly(2L, 2L, 2L); + assertDrills(duplicates); + assertDrills(service.duplicates(new DuplicatesRequest().withTableName("orders").withFields(List.of("status", "customer")))); + } + + /******************************************************************************* + ** Paging and search retain exact memberships and enforce page size. + *******************************************************************************/ + @Test + void distinctPages() throws Exception + { + var first = service.distinctValues(new DistinctValuesRequest().withTableName("orders").withFieldName("note").withPageSize(2)); + var second = service.distinctValues(new DistinctValuesRequest().withTableName("orders").withFieldName("note").withPageSize(2).withPage(1)); + assertThat(first.getHasMore()).isTrue(); + assertThat(first.getValues()).hasSize(2); + assertThat(second.getValues()).hasSize(2); + assertThat(first.getValues().getFirst().getValue()).isNotEqualTo(second.getValues().getFirst().getValue()); + assertDrills(first); + assertDrills(second); + var search = service.distinctValues(new DistinctValuesRequest().withTableName("orders").withFieldName("note").withSearch("AA")); + assertThat(search.getValues()).hasSize(1); + assertDrills(search); + assertThatThrownBy(() -> service.distinctValues(new DistinctValuesRequest().withTableName("orders").withFieldName("note").withPageSize(501))).isInstanceOf(QException.class); + } + + /******************************************************************************* + ** Missing READ/analyze, locked fields and exclusions fail before computation. + *******************************************************************************/ + @Test + void permissionChecks() throws Exception + { + assertThat(service.capabilities("orders").getFields()).doesNotContain("secret"); + assertThatThrownBy(() -> service.columnProfile(column("secret"))).isInstanceOf(QPermissionDeniedException.class); + QContext.init(instance, TestSupport.user("alice").withPermission("orders.analyze")); + assertThatThrownBy(() -> service.columnProfile(column("amount"))).isInstanceOf(QPermissionDeniedException.class); + QContext.init(instance, TestSupport.user("alice").withPermission("orders.read")); + assertThatThrownBy(() -> service.columnProfile(column("amount"))).isInstanceOf(QPermissionDeniedException.class); + QContext.init(instance, session()); + config.withTableDefaults(Map.of("orders", new TableAnalysisDefaults().withExcludedFields(Set.of("amount")))); + service.getConfigResolver().invalidate(); + assertThat(service.capabilities("orders").getFields()).doesNotContain("amount"); + assertThatThrownBy(() -> service.columnProfile(column("amount"))).isInstanceOf(QPermissionDeniedException.class); + assertThatThrownBy(() -> service.capabilities("missing")).isInstanceOf(QPermissionDeniedException.class); + } + + /******************************************************************************* + ** Record locks constrain every backend primitive and every returned drill. + *******************************************************************************/ + @Test + void recordLocks() throws Exception + { + QSession alice = TestSupport.user("alice").withPermissions("orders.read", "orders.analyze", "customer.read", + "data_analysis.run.read", "data_analysis.run.insert", "data_analysis.run.edit").withSecurityKeyValue("customer_scope", 1); + QContext.init(instance, alice); + var result = service.columnProfile(column("amount")); + assertMetric(result.getCounts(), "rows", "8"); + assertMetric(result.getNumeric().getMetrics(), "sum", "120"); + assertDrills(result); + assertDrills(service.tableProfile(new TableProfileRequest().withTableName("orders"))); + assertDrills(service.distinctValues(new DistinctValuesRequest().withTableName("orders").withFieldName("note"))); + assertDrills(service.duplicates(new DuplicatesRequest().withTableName("orders").withFields(List.of("quantity")))); + QContext.getQSession().withSecurityKeyValue("customer_scope", 2); + assertMetric(service.columnProfile(column("amount")).getCounts(), "rows", "12"); + } + /******************************************************************************* + ** Guarded cache JSON contains no unmerged rare top values or extreme listings. + *******************************************************************************/ + @Test + void privacyGuard() throws Exception + { + config.withDefaultMinGroupSize(3); + service.getConfigResolver().invalidate(); + ColumnProfile note = service.columnProfile(column("note")); + assertThat(note.getTopValues()).singleElement().satisfies(v -> + { + assertThat(v.getLabel()).startsWith("Other ("); + assertThat(v.getCount()).isEqualTo(12); + assertThat(v.getDrill()).isNull(); + assertThat(v.getValue()).isNull(); + }); + ColumnProfile amount = service.columnProfile(column("amount")); + assertThat(amount.getNumeric().getOutliers()).isEmpty(); + assertThat(metric(amount.getNumeric().getMetrics(), "outliers").getDrill()).isNull(); + var tiny = service.columnProfile(column("amount").withFilter(DrillFilters.condition("id", QCriteriaOperator.EQUALS, 1))); + for(String key : List.of("min", "max", "mean", "stddev", "p1", "p50", "p99")) + { + assertThat(metric(tiny.getNumeric().getMetrics(), key).getSuppressed()).isTrue(); + assertThat(metric(tiny.getNumeric().getMetrics(), key).getValue()).isNull(); + assertThat(metric(tiny.getNumeric().getMetrics(), key).getSuppressedReason()).isEqualTo("Hidden: fewer than 3 records"); + } + var page = service.distinctValues(new DistinctValuesRequest().withTableName("orders").withFieldName("note").withPageSize(1)); + assertThat(page.getValues()).hasSize(1); + assertThat(page.getValues().getFirst().getCount()).isEqualTo(12); + assertThat(page.getHasMore()).isFalse(); + for(QRecord run : runs()) + { + JsonNode json = new ObjectMapper().readTree(run.getValueString("result_json")); + for(JsonNode group : json.path("topValues")) + { + if(group.path("count").asLong() < 3) + { + assertThat(group.path("drill").isMissingNode() || group.path("drill").isNull()).isTrue(); + assertThat(group.path("value").isMissingNode() || group.path("value").isNull()).isTrue(); + } + } + } + } + + /******************************************************************************* + ** Real asynchronous context propagation, completion, cache hit and invalidation. + *******************************************************************************/ + @Test + void estimatesRunsAndCache() throws Exception + { + config.withDefaultSampleThresholdRows(5).withDefaultSampleRows(4); + service = new DataAnalysisService(config); + ColumnProfile estimate = service.columnProfile(column("amount")); + assertThat(estimate.getEstimate()).isTrue(); + assertThat(estimate.getSampleRows()).isEqualTo(4); + assertMetric(estimate.getCounts(), "rows", "4"); + assertMetric(estimate.getNumeric().getMetrics(), "sum", "10"); + String id = service.startExactRun(column("amount")); + RunStatus run = awaitRun(id); + assertThat(run.getStatus()).isEqualTo("COMPLETE"); + assertMetric(run.getColumnProfile().getCounts(), "rows", "12"); + int before = runs().size(); + assertThat(service.startExactRun(column("amount"))).isEqualTo(id); + assertMetric(service.columnProfile(column("amount").withExact(true)).getCounts(), "rows", "12"); + assertThat(runs()).hasSize(before); + TestSupport.insert("orders", new QRecord().withValue("id", 13).withValue("customer", 1).withValue("amount", BigDecimal.ONE)); + String next = service.startExactRun(column("amount")); + assertThat(next).isNotEqualTo(id); + assertThat(awaitRun(next).getStatus()).isEqualTo("COMPLETE"); + assertMetric(service.getRun(next).getColumnProfile().getCounts(), "rows", "13"); + assertThat(service.tableProfile(new TableProfileRequest().withTableName("orders")).getEstimate()).isTrue(); + String table = service.startExactRun(new TableProfileRequest().withTableName("orders").withFields(List.of("id", "amount"))); + assertThat(awaitRun(table).getTableProfile().getRowCount().getValue().toString()).isEqualTo("13"); + } + + /******************************************************************************* + ** Partial results retain exact prefix drills; empty populations are valid. + *******************************************************************************/ + @Test + void truncatedAndEmpty() throws Exception + { + config.withDefaultMaxRows(3); + service.getConfigResolver().invalidate(); + ColumnProfile partial = service.columnProfile(column("amount").withExact(true)); + assertThat(partial.getTruncated()).isTrue(); + assertThat(partial.getRowsScanned()).isEqualTo(3); + assertMetric(partial.getCounts(), "rows", "3"); + assertDrills(partial); + var emptyFilter = DrillFilters.condition("id", QCriteriaOperator.EQUALS, -1); + var empty = service.columnProfile(column("amount").withFilter(emptyFilter)); + assertMetric(empty.getCounts(), "rows", "0"); + assertThat(empty.getNumeric().getHistogram()).isEmpty(); + assertDrills(empty); + assertDrills(service.columnProfile(column("created").withFilter(emptyFilter))); + assertDrills(service.columnProfile(column("note").withFilter(emptyFilter))); + } + + /******************************************************************************* + ** Drill composition never modifies the request's nested OR filter. + *******************************************************************************/ + @Test + void originalFilterUnchanged() throws Exception + { + QQueryFilter filter = DrillFilters.condition("id", QCriteriaOperator.EQUALS, 1).withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withCriteria("id", QCriteriaOperator.EQUALS, 12); + String original = JsonUtils.toJson(filter); + ColumnProfile result = service.columnProfile(column("amount").withFilter(filter)); + assertMetric(result.getCounts(), "rows", "2"); + assertDrills(result); + assertThat(JsonUtils.toJson(filter)).isEqualTo(original); + } + + /******************************************************************************* + ** Config rows override defaults and restore the session after privileged reads. + *******************************************************************************/ + @Test + void configResolution() throws Exception + { + config.withDefaultMaxRows(500).withTableDefaults(Map.of("orders", new TableAnalysisDefaults().withMaxRows(100).withSampleRows(8).withMinGroupSize(2))); + service.getConfigResolver().invalidate(); + assertThat(service.capabilities("orders").getConfig().getMaxRows()).isEqualTo(100); + TestSupport.insert(DataAnalysisTables.TABLE_CONFIG, new QRecord().withValue("table_name", "orders").withValue("max_rows", 50) + .withValue("min_group_size", 3).withValue("analyzable_fields", "amount, note").withValue("excluded_fields", "note").withValue("cache_ttl_minutes", 2)); + assertThat(service.capabilities("orders").getConfig().getMaxRows()).isEqualTo(100); + service.getConfigResolver().invalidate(); + QSession current = QContext.getQSession(); + var capabilities = service.capabilities("orders"); + assertThat(capabilities.getConfig().getMaxRows()).isEqualTo(50); + assertThat(capabilities.getConfig().getSampleRows()).isEqualTo(8); + assertThat(capabilities.getConfig().getMinGroupSize()).isEqualTo(3); + assertThat(capabilities.getConfig().getCacheTtlMinutes()).isEqualTo(2); + assertThat(capabilities.getFields()).containsExactly("amount"); + assertThat(QContext.getQSession()).isSameAs(current); + config.withExcludeTables(Set.of("orders")); + service.getConfigResolver().invalidate(); + assertThatThrownBy(() -> service.capabilities("orders")).isInstanceOf(QPermissionDeniedException.class); + } + + /******************************************************************************* + ** Changed privacy policy and another user cannot retrieve older cached data. + *******************************************************************************/ + @Test + void cachePolicyRechecked() throws Exception + { + String id = service.startExactRun(column("amount")); + config.withDefaultMinGroupSize(3); + service.getConfigResolver().invalidate(); + assertThatThrownBy(() -> service.getRun(id)).isInstanceOf(QPermissionDeniedException.class); + assertThat(service.startExactRun(column("amount"))).isNotEqualTo(id); + QContext.init(instance, TestSupport.user("bob").withPermissions("data_analysis.run.read", "orders.read", "orders.analyze")); + assertThatThrownBy(() -> service.getRun(id)).isInstanceOf(QPermissionDeniedException.class); + } + + /******************************************************************************* + ** Calendar boundaries, weekly/monthly/yearly drills and interpolation. + *******************************************************************************/ + @Test + void shapeAndDateRules() throws Exception + { + assertThat(StandInAnalysisBackend.shape("AAA-999 QQ_88!!")).isEqualTo("A-9 A_9!!"); + LocalDate first = LocalDate.of(2024, 1, 1); + assertThat(ColumnProfiler.autoBucket(first, first.plusDays(92))).isEqualTo(AnalysisBackend.DateUnit.DAY); + assertThat(ColumnProfiler.autoBucket(first, first.plusDays(93))).isEqualTo(AnalysisBackend.DateUnit.WEEK); + assertThat(ColumnProfiler.autoBucket(first, first.plusYears(2))).isEqualTo(AnalysisBackend.DateUnit.WEEK); + assertThat(ColumnProfiler.autoBucket(first, first.plusYears(2).plusDays(1))).isEqualTo(AnalysisBackend.DateUnit.MONTH); + assertThat(ColumnProfiler.autoBucket(first, first.plusYears(10))).isEqualTo(AnalysisBackend.DateUnit.MONTH); + assertThat(ColumnProfiler.autoBucket(first, first.plusYears(10).plusDays(1))).isEqualTo(AnalysisBackend.DateUnit.YEAR); + for(AnalysisBackend.DateUnit unit : List.of(AnalysisBackend.DateUnit.WEEK, AnalysisBackend.DateUnit.MONTH, AnalysisBackend.DateUnit.YEAR)) + { + var buckets = service.backend("orders").dateBuckets("orders", null, "order_date", unit, java.time.ZoneId.of("America/Chicago")); + assertThat(buckets.stream().mapToLong(AnalysisBackend.GroupCount::count).sum()).isEqualTo(12); + for(var bucket : buckets) + { + assertThat(count(new Drill().withTableName("orders").withFilter(bucket.condition()))).isEqualTo(bucket.count()); + } + } + assertThat(StandInAnalysisBackend.percentile(List.of(BigDecimal.ZERO, BigDecimal.TEN), new BigDecimal("0.25"))).isEqualByComparingTo("2.5"); + } + + /******************************************************************************* + ** Common request builder. + *******************************************************************************/ + protected static ColumnProfileRequest column(String field) + { + return (new ColumnProfileRequest().withTableName("orders").withFieldName(field)); + } + + /******************************************************************************* + ** Select metrics by their stable semantic keys. + *******************************************************************************/ + protected static Metric metric(List metrics, String key) + { + return (metrics.stream().filter(m -> m.getKey().equals(key)).findFirst().orElseThrow()); + } + + /******************************************************************************* + ** Scalar reference tolerance. + *******************************************************************************/ + protected static void assertMetric(List metrics, String key, String expected) + { + assertThat(new BigDecimal(metric(metrics, key).getValue().toString()).doubleValue()).as(key).isCloseTo(Double.parseDouble(expected), within(1e-9)); + } + + /******************************************************************************* + ** Walk actual serialized payloads so cached drills receive the same assertions. + *******************************************************************************/ + protected void assertDrills(Object result) throws Exception + { + walk(new ObjectMapper().readTree(JsonUtils.toJson(result))); + } + + /******************************************************************************* + ** Collect count metrics, group counts, chart points and candidate-key counts. + *******************************************************************************/ + private void walk(JsonNode node) throws Exception + { + if(node.isObject()) + { + if(node.hasNonNull("drill")) + { + JsonNode number = node.has("count") ? node.get("count") : node.has("rowCount") ? node.get("rowCount") : node.has("y") ? node.get("y") : node.get("value"); + assertThat(number).as(node.toString()).isNotNull(); + Drill drill = JsonUtils.toObject(node.get("drill").toString(), Drill.class); + assertThat(count(drill)).as(node.toString()).isEqualTo(number.asLong()); + } + var fields = node.fields(); + while(fields.hasNext()) + { + var field = fields.next(); + if(!field.getKey().equals("drill") && !field.getKey().equals("filter")) + { + walk(field.getValue()); + } + } + } + else if(node.isArray()) + { + for(JsonNode child : node) + { + walk(child); + } + } + } + + /******************************************************************************* + ** Use core count under the current user's locks. + *******************************************************************************/ + private static long count(Drill drill) throws Exception + { + CountInput input = new CountInput(drill.getTableName()).withFilter(drill.getFilter()); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + return (new CountAction().execute(input).getCount()); + } + + /******************************************************************************* + ** Inspect persistent cache rows through enforced actions. + *******************************************************************************/ + protected List runs() throws Exception + { + QueryInput input = new QueryInput(DataAnalysisTables.RUN).withShouldFetchHeavyFields(true); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + return (new QueryAction().execute(input).getRecords()); + } + + /******************************************************************************* + ** Wait for actual asynchronous completion with a fixed deadline. + *******************************************************************************/ + protected RunStatus awaitRun(String id) throws Exception + { + long deadline = System.nanoTime() + 20_000_000_000L; + RunStatus run; + do + { + run = service.getRun(id); + if(!"RUNNING".equals(run.getStatus())) + { + return (run); + } + Thread.sleep(10); + } + while(System.nanoTime() < deadline); + throw new AssertionError("Run did not finish: " + id); + } + + + /******************************************************************************* + ** Related labels obey READ, field locks and record locks without bypasses. + *******************************************************************************/ + @Test + void relatedLabelSecurity() throws Exception + { + String original = service.startExactRun(column("customer")); + QContext.getQSession().getPermissions().remove("customer.read"); + assertThat(service.columnProfile(column("customer")).getTopValues()).extracting(v -> v.getLabel()).containsExactly("1", "2"); + QContext.getQSession().withPermission("customer.read"); + instance.getTable("customer").getField("name").withFieldSecurityLock(new com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock() + .withSecurityKeyType("secret_access").withDefaultBehavior(com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock.Behavior.DENY)); + assertThatThrownBy(() -> service.getRun(original)).isInstanceOf(QPermissionDeniedException.class); + assertThat(service.columnProfile(column("customer")).getTopValues()).extracting(v -> v.getLabel()).containsExactly("1", "2"); + instance.getTable("customer").getField("name").withFieldSecurityLock(null); + instance.getTable("customer").withRecordSecurityLock(new com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock() + .withSecurityKeyType("customer_scope").withFieldName("id")); + QContext.getQSession().getSecurityKeyValues().remove("orders.all"); + QContext.getQSession().withSecurityKeyValue("customer_scope", 1); + assertThat(service.columnProfile(column("customer")).getTopValues()).extracting(v -> v.getLabel()).containsExactly("Alice Co"); + } + + /******************************************************************************* + ** Zero/constant populations, zero limits, rejected executors and disabled cache. + *******************************************************************************/ + @Test + void boundaryAndFailurePaths() throws Exception + { + assertThat(DataAnalysisRuntime.service()).isNotNull(); + var constant = service.columnProfile(column("amount").withFilter(DrillFilters.condition("id", QCriteriaOperator.EQUALS, 2))); + assertThat(constant.getNumeric().getHistogram().stream().mapToLong(v -> v.getCount()).sum()).isEqualTo(1); + assertDrills(constant); + config.withDefaultMaxRows(0).withDefaultCacheTtlMinutes(0); + service.getConfigResolver().invalidate(); + assertMetric(service.columnProfile(column("amount")).getCounts(), "rows", "0"); + int previous = runs().size(); + service.columnProfile(column("amount")); + assertThat(runs()).hasSize(previous + 1); + service = new DataAnalysisService(config, task -> + { + throw new java.util.concurrent.RejectedExecutionException("test"); + }); + assertThatThrownBy(() -> service.startExactRun(column("amount"))).isInstanceOf(QException.class); + assertThat(runs()).anyMatch(r -> "FAILED".equals(r.getValueString("status"))); + assertThatThrownBy(() -> service.getRun("bad")).isInstanceOf(QException.class); + assertThatThrownBy(() -> service.duplicates(new DuplicatesRequest().withTableName("orders"))).isInstanceOf(QException.class); + assertThatThrownBy(() -> service.columnProfile(null)).isInstanceOf(QException.class); + assertThatThrownBy(() -> service.startExactRun(new DistinctValuesRequest().withTableName("orders").withFieldName("note"))).isInstanceOf(QException.class); + } + + /******************************************************************************* + ** Decimal scales, integral boundaries and final-bin closure must be portable. + *******************************************************************************/ + @Test + void scaledDecimalsAndIntegerBins() throws Exception + { + TestSupport.insert("orders", new QRecord().withValue("id", 13).withValue("customer", 1).withValue("amount", new BigDecimal("0.00"))); + TestSupport.insert("orders", new QRecord().withValue("id", 14).withValue("customer", 1).withValue("amount", new BigDecimal("10.000"))); + var result = service.columnProfile(column("amount")); + assertMetric(result.getCounts(), "distinct", "8"); + assertMetric(result.getNumeric().getMetrics(), "zeros", "3"); + assertThat(result.getNumeric().getHistogram().stream().mapToLong(v -> v.getCount()).sum()).isEqualTo(13); + assertDrills(result); + var quantity = service.columnProfile(column("quantity")); + assertMetric(quantity.getNumeric().getMetrics(), "mean", "3.5"); + assertThat(quantity.getNumeric().getHistogram().stream().mapToLong(v -> v.getCount()).sum()).isEqualTo(12); + assertDrills(quantity); + var filtered = column("amount").withFilter(DrillFilters.condition("amount", QCriteriaOperator.EQUALS, new BigDecimal("0.00"))); + var selected = service.columnProfile(filtered); + assertThat(((Number) metric(selected.getCounts(), "rows").getValue()).longValue()).isEqualTo(count(new Drill().withTableName("orders").withFilter(filtered.getFilter()))); + assertDrills(selected); + assertDrills(service.columnProfile(filtered)); + } + + + /******************************************************************************* + ** Expired cache rows miss; reordered AND criteria share normalized identity. + *******************************************************************************/ + @Test + void cacheExpiryAndNormalizedFilters() throws Exception + { + QQueryFilter a = DrillFilters.condition("id", QCriteriaOperator.GREATER_THAN, 1).withCriteria("id", QCriteriaOperator.LESS_THAN, 8); + QQueryFilter b = DrillFilters.condition("id", QCriteriaOperator.LESS_THAN, 8).withCriteria("id", QCriteriaOperator.GREATER_THAN, 1); + String id = service.startExactRun(column("amount").withFilter(a)); + assertThat(service.startExactRun(column("amount").withFilter(b))).isEqualTo(id); + var update = new com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput(DataAnalysisTables.RUN) + .withRecords(List.of(new QRecord().withValue("id", Integer.valueOf(id)).withValue("finished_at", java.time.Instant.now().minusSeconds(7200)))); + update.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + new com.kingsrook.qqq.backend.core.actions.tables.UpdateAction().execute(update); + assertThat(service.startExactRun(column("amount").withFilter(a))).isNotEqualTo(id); + var backend = service.backend("customer"); + assertThat(com.kingsrook.qbits.dataanalysis.run.DataVersion.of(backend, "customer", null)).isEqualTo("2:id:2"); + } + + /******************************************************************************* + ** Local midnight date drills span 23 hours across daylight-saving transitions. + *******************************************************************************/ + @Test + void daylightSavingAndSearch() throws Exception + { + TestSupport.insert("orders", new QRecord().withValue("id", 13).withValue("customer", 1).withValue("created", java.time.Instant.parse("2026-03-08T06:00:00Z"))); + TestSupport.insert("orders", new QRecord().withValue("id", 14).withValue("customer", 1).withValue("created", java.time.Instant.parse("2026-03-09T04:59:59Z"))); + TestSupport.insert("orders", new QRecord().withValue("id", 15).withValue("customer", 1).withValue("created", java.time.Instant.parse("2026-03-09T05:00:00Z"))); + var drill = new Drill().withTableName("orders").withFilter(DrillFilters.dateRange("created", LocalDate.of(2026, 3, 8), LocalDate.of(2026, 3, 9), true, java.time.ZoneId.of("America/Chicago"))); + assertThat(count(drill)).isEqualTo(2); + assertDrills(service.columnProfile(column("created").withFilter(DrillFilters.condition("id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, 13)))); + var values = service.distinctValues(new DistinctValuesRequest().withTableName("orders").withFieldName("amount").withSearch("1000")); + assertThat(values.getValues()).hasSize(1); + assertThat(values.getTotalDistinct()).isEqualTo(1); + assertDrills(values); + } + + /******************************************************************************* + ** Computation failures persist a safe FAILED row, with no partial result JSON. + *******************************************************************************/ + @Test + void executionFailureAndGuardedBooleans() throws Exception + { + List jobs = new java.util.ArrayList<>(); + service = new DataAnalysisService(config, jobs::add); + String id = service.startExactRun(column("amount")); + var table = instance.getTables().remove("orders"); + jobs.getFirst().run(); + instance.addTable(table); + assertThat(service.getRun(id).getStatus()).isEqualTo("FAILED"); + assertThat(service.getRun(id).getError()).contains("could not be completed"); + assertThat(runs().getFirst().getValue("result_json")).isNull(); + config.withDefaultMinGroupSize(3); + service = new DataAnalysisService(config, Runnable::run); + var paid = service.columnProfile(column("paid")); + assertThat(paid.getBooleans()).anySatisfy(m -> + { + assertThat(m.getKey()).isEqualTo("other"); + assertThat(m.getDrill()).isNull(); + }); + assertDrills(paid); + assertThat(service.columnProfile(column("created")).getDates().getGaps()).isEmpty(); + assertThat(service.duplicates(new DuplicatesRequest().withTableName("orders").withFields(List.of("quantity"))).getGroups()) + .allMatch(v -> v.getDrill() == null); + } +} diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/EngineFixture.java b/src/test/java/com/kingsrook/qbits/dataanalysis/EngineFixture.java new file mode 100644 index 0000000..cb3a382 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/EngineFixture.java @@ -0,0 +1,145 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis; + +import java.math.BigDecimal; +import java.sql.Connection; +import java.sql.DriverManager; +import java.time.Instant; +import java.time.LocalDate; +import java.util.List; +import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; +import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock; +import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; +import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; + +/******************************************************************************* + ** Identical fixture built through core actions on all three supported test backends. + *******************************************************************************/ +abstract class EngineFixture +{ + protected DataAnalysisService service; + protected DataAnalysisQBitConfig config; + protected QInstance instance; + protected Connection connection; + + /******************************************************************************* + ** Null selects memory; subclasses supply real database metadata. + *******************************************************************************/ + protected RDBMSBackendMetaData database() + { + return (null); + } + + /******************************************************************************* + ** Build metadata and insert rows without direct memory-store injection. + *******************************************************************************/ + @BeforeEach + void setup() throws Exception + { + instance = TestSupport.instance(); + RDBMSBackendMetaData database = database(); + String backend = database == null ? "memory" : "database_" + java.util.UUID.randomUUID(); + if(database != null) + { + instance.addBackend(database.withName(backend)); + connection = DriverManager.getConnection(database.getJdbcUrl(), database.getUsername(), database.getPassword()); + try(var statement = connection.createStatement()) + { + statement.execute("DROP TABLE IF EXISTS orders"); + statement.execute("DROP TABLE IF EXISTS customer"); + statement.execute("CREATE TABLE customer (id INTEGER PRIMARY KEY, name VARCHAR(100))"); + statement.execute("CREATE TABLE orders (id INTEGER PRIMARY KEY, customer INTEGER, status VARCHAR(20), amount DECIMAL(20,4), quantity INTEGER, order_date DATE, created TIMESTAMP, note VARCHAR(100), paid BOOLEAN, modify_date TIMESTAMP, secret VARCHAR(100))"); + } + } + instance.addSecurityKeyType(new QSecurityKeyType().withName("customer_scope").withAllAccessKeyName("orders.all")); + instance.addSecurityKeyType(new QSecurityKeyType().withName("secret_access")); + instance.addTable(new QTableMetaData().withName("customer").withBackendName(backend).withPrimaryKeyField("id") + .withPermissionRules(DataAnalysisTables.permissionRules("customer")).withRecordLabelFormatAndFields("%s", "name") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)).withField(new QFieldMetaData("name", QFieldType.STRING))); + instance.addPossibleValueSource(QPossibleValueSource.newForTable("customer")); + instance.addPossibleValueSource(new QPossibleValueSource().withName("statusValues").withType(QPossibleValueSourceType.ENUM) + .withEnumValues(List.of(new QPossibleValue<>("NEW", "New order"), new QPossibleValue<>("PAID", "Paid order")))); + instance.addTable(new QTableMetaData().withName("orders").withBackendName(backend).withPrimaryKeyField("id") + .withPermissionRules(DataAnalysisTables.permissionRules("orders")) + .withField(new QFieldMetaData("id", QFieldType.INTEGER)).withField(new QFieldMetaData("customer", QFieldType.INTEGER).withPossibleValueSourceName("customer")) + .withField(new QFieldMetaData("status", QFieldType.STRING).withPossibleValueSourceName("statusValues")) + .withField(new QFieldMetaData("amount", QFieldType.DECIMAL)).withField(new QFieldMetaData("quantity", QFieldType.INTEGER)) + .withField(new QFieldMetaData("order_date", QFieldType.DATE)).withField(new QFieldMetaData("created", QFieldType.DATE_TIME)) + .withField(new QFieldMetaData("note", QFieldType.STRING)).withField(new QFieldMetaData("paid", QFieldType.BOOLEAN)) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date")) + .withField(new QFieldMetaData("secret", QFieldType.STRING).withFieldSecurityLock(new FieldSecurityLock().withSecurityKeyType("secret_access") + .withDefaultBehavior(FieldSecurityLock.Behavior.DENY).withOverrideValues(List.of(true)))) + .withRecordSecurityLock(new RecordSecurityLock().withFieldName("customer").withSecurityKeyType("customer_scope").withNullValueBehavior(RecordSecurityLock.NullValueBehavior.DENY))); + new QInstanceEnricher(instance).enrich(); + QContext.init(instance, session()); + TestSupport.insert("customer", new QRecord().withValue("id", 1).withValue("name", "Alice Co")); + TestSupport.insert("customer", new QRecord().withValue("id", 2).withValue("name", "Bob Co")); + Integer[] amounts = {-10, 0, 10, 10, 20, 20, 30, 40, 50, 1000, null, 0}; + String[] notes = {null, "", " ", "AA-11", "BB-22", "CC-33", "x", "yy", "z", "", "AB12", "CD34"}; + for(int i = 0; i < amounts.length; i++) + { + TestSupport.insert("orders", new QRecord().withValue("id", i + 1).withValue("customer", i < 8 ? 1 : 2) + .withValue("status", i % 2 == 0 ? "NEW" : "PAID").withValue("amount", amounts[i] == null ? null : BigDecimal.valueOf(amounts[i])) + .withValue("quantity", i / 2 + 1).withValue("order_date", LocalDate.of(2026, 1, i < 4 ? 1 : 3)) + .withValue("created", Instant.parse(i < 4 ? "2026-01-01T12:00:00Z" : "2026-01-03T18:00:00Z")) + .withValue("note", notes[i]).withValue("paid", i == 11 ? null : i % 2 == 0).withValue("modifyDate", Instant.parse("2026-10-13T12:00:00Z"))); + } + config = new DataAnalysisQBitConfig().withBackendName("memory"); + service = new DataAnalysisService(config, Runnable::run); + } + + /******************************************************************************* + ** Fixture privileges include an explicit record-lock key. + *******************************************************************************/ + protected QSession session() + { + return (TestSupport.admin().withPermissions("orders.read", "orders.insert", "orders.edit", "orders.analyze", "customer.read", "customer.insert") + .withSecurityKeyValue("orders.all", true).withValue(QSession.VALUE_KEY_USER_TIMEZONE, "America/Chicago")); + } + + /******************************************************************************* + ** Clear context and close SQL connections. + *******************************************************************************/ + @AfterEach + void cleanup() throws Exception + { + QContext.clear(); + if(connection != null) + { + connection.close(); + } + } +} diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/H2EngineTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/H2EngineTest.java new file mode 100644 index 0000000..d989c4d --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/H2EngineTest.java @@ -0,0 +1,40 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis; + +import java.util.UUID; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; + +/******************************************************************************* + ** H2 executes the identical fixture through the RDBMS module. + *******************************************************************************/ +class H2EngineTest extends EngineContractTest +{ + /******************************************************************************* + ** An isolated database for each test. + *******************************************************************************/ + @Override + protected RDBMSBackendMetaData database() + { + return (new RDBMSBackendMetaData().withVendor("h2").withUsername("sa").withPassword("") + .withJdbcUrl("jdbc:h2:mem:analysis_" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1;DATABASE_TO_UPPER=FALSE")); + } +} diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/MemoryEngineTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/MemoryEngineTest.java new file mode 100644 index 0000000..48aa7c6 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/MemoryEngineTest.java @@ -0,0 +1,28 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis; + +/******************************************************************************* + ** Memory reference implementation of the shared engine contract. + *******************************************************************************/ +class MemoryEngineTest extends EngineContractTest +{ +} diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/PostgresEngineTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/PostgresEngineTest.java new file mode 100644 index 0000000..9a23267 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/PostgresEngineTest.java @@ -0,0 +1,73 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis; + +import com.kingsrook.qqq.backend.module.postgres.model.metadata.PostgreSQLBackendMetaData; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Assumptions; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.postgresql.PostgreSQLContainer; + +/******************************************************************************* + ** Actual PostgreSQL tests; skip cleanly if Docker is unavailable. + *******************************************************************************/ +class PostgresEngineTest extends EngineContractTest +{ + private static PostgreSQLContainer postgres; + + /******************************************************************************* + ** Never substitute another database for the PostgreSQL contract. + *******************************************************************************/ + @BeforeAll + static void start() + { + if(DockerClientFactory.instance().isDockerAvailable()) + { + postgres = new PostgreSQLContainer("postgres:17-alpine"); + postgres.start(); + } + } + + /******************************************************************************* + ** Release the test container. + *******************************************************************************/ + @AfterAll + static void stop() + { + if(postgres != null) + { + postgres.stop(); + } + } + + /******************************************************************************* + ** Credentials belong only to the ephemeral test container. + *******************************************************************************/ + @Override + protected RDBMSBackendMetaData database() + { + Assumptions.assumeTrue(postgres != null, "Docker unavailable; PostgreSQL contract skipped"); + return (new PostgreSQLBackendMetaData().withJdbcUrl(postgres.getJdbcUrl()) + .withUsername(postgres.getUsername()).withPassword(postgres.getPassword())); + } +} -- 2.54.0