From 470b0bd21412a6ccbdc524c7a1559d6ac508abc6 Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 14 Oct 2026 07:35:44 -0500 Subject: [PATCH] feat: retention and wipe with crypto-shredding; move to QQQ RC.6 (phase f) Moves the pin to QQQ 4.1.0-RC.6 and removes the stand-ins RC.6 provides (classification, client channels, table features, the UI API extension point). Adds retention rules, wipe markers, legal holds, wipe runs and the per-record-per-field audit key table; the wipe process (null with marker, overwrite, keyed hash, delete, archive then wipe) runs in one transaction per batch, destroying the audit keys with the field updates; the audit protector and key source (AES-GCM wrapped DEKs, AAD-bound); the backups warning; canary and wipe-proof tests. Fixes a key-source bug that zeroed each DEK after returning it. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 16 + README.md | 70 ++- docs/ddl/secured-data-postgres.sql | 43 +- pom.xml | 32 +- .../qbits/secureddata/CoreAuditBridge.java | 220 +++++++++ .../secureddata/CoreFieldAccessBridge.java | 1 + .../secureddata/SecuredDataAccessLogger.java | 4 +- .../SecuredDataAuditProtector.java | 62 +++ .../SecuredDataClassifications.java | 18 +- .../secureddata/SecuredDataKeySource.java | 290 ++++++++++++ .../secureddata/SecuredDataQBitConfig.java | 65 ++- .../secureddata/SecuredDataQBitProducer.java | 21 +- .../secureddata/SecuredDataResolver.java | 14 +- .../qbits/secureddata/SecuredDataRuntime.java | 10 + .../secureddata/app/SecuredDataHomeData.java | 18 + .../secureddata/app/SecuredDataHomeView.java | 4 + .../breakglass/BreakGlassService.java | 6 +- .../secureddata/metadata/RetentionTables.java | 134 ++++++ .../metadata/SecuredDataEnums.java | 2 +- .../metadata/SecuredDataTableCustomizers.java | 36 +- .../metadata/SecuredDataTables.java | 28 +- .../SecuredDataPermissionChecker.java | 4 +- .../processes/BreakGlassProcesses.java | 2 +- .../retention/ArchiveSinkInterface.java | 36 ++ .../secureddata/retention/RetentionRules.java | 150 ++++++ .../retention/RetentionWidgets.java | 163 +++++++ .../secureddata/retention/WipeProcesses.java | 258 +++++++++++ .../secureddata/retention/WipeService.java | 426 ++++++++++++++++++ .../secureddata/reveal/RevealService.java | 6 +- .../secureddata/standin/AiFieldMetaData.java | 67 --- .../secureddata/standin/AiTableMetaData.java | 67 --- .../AuditValueProtectorInterface.java} | 42 +- .../standin/ClassifiedValueKeySource.java | 70 +++ .../standin/ClassifiedValueTokens.java | 218 +++++++++ .../standin/ClientChannelSession.java | 66 --- .../standin/ClientChannelsMetaData.java | 79 ---- .../qbits/secureddata/standin/DMLType.java} | 18 +- .../standin/DataClassification.java | 72 --- .../standin/FieldAccessDecision.java | 2 +- .../secureddata/standin/FieldAccessEvent.java | 2 +- .../standin/FieldAccessEventType.java | 2 +- .../standin/FieldAccessOutcome.java | 2 +- .../standin/FieldAccessResolverInterface.java | 2 +- .../secureddata/standin/FieldAccessTier.java | 2 +- .../secureddata/standin/FieldMaskStyle.java | 2 +- .../standin/ProtectedAuditValues.java | 30 ++ .../TableFeaturePermissionsMetaData.java | 79 ---- .../secureddata/support/ActivityLog.java | 2 +- .../support/CoreMetadataBridge.java | 286 ------------ .../support/SecuredDataMetadata.java | 124 +++++ .../secureddata/support/SystemWrites.java | 2 +- .../uiapi/SecuredDataExtension.java | 4 +- .../uiapi/SecuredDataRevealOptionsSpecV1.java | 0 .../uiapi/SecuredDataRevealSpecV1.java | 0 .../secureddata/uiapi/SecuredDataSchemas.java | 0 .../uiapi/SecuredDataStatusSpecV1.java | 0 .../secureddata/CoreFieldAccessBridgeIT.java | 2 +- .../qbits/secureddata/LogCapture.java | 119 +++++ .../qbits/secureddata/RetentionProofTest.java | 371 +++++++++++++++ .../secureddata/RetentionTestSupport.java | 147 ++++++ .../qbits/secureddata/RetentionWipeTest.java | 265 +++++++++++ .../secureddata/RevealAndBreakGlassTest.java | 2 +- .../SecuredDataAccessLoggerTest.java | 2 +- .../secureddata/SecuredDataHomeTest.java | 2 +- .../secureddata/SecuredDataHttpTest.java | 0 .../SecuredDataQBitProducerTest.java | 18 +- .../secureddata/SecuredDataResolverTest.java | 8 +- .../secureddata/SecuredDataTestSupport.java | 20 +- .../SecuredDataUiApiContractTest.java} | 17 +- 69 files changed, 3459 insertions(+), 893 deletions(-) create mode 100644 src/main/java/com/kingsrook/qbits/secureddata/CoreAuditBridge.java create mode 100644 src/main/java/com/kingsrook/qbits/secureddata/SecuredDataAuditProtector.java create mode 100644 src/main/java/com/kingsrook/qbits/secureddata/SecuredDataKeySource.java create mode 100644 src/main/java/com/kingsrook/qbits/secureddata/metadata/RetentionTables.java create mode 100644 src/main/java/com/kingsrook/qbits/secureddata/retention/ArchiveSinkInterface.java create mode 100644 src/main/java/com/kingsrook/qbits/secureddata/retention/RetentionRules.java create mode 100644 src/main/java/com/kingsrook/qbits/secureddata/retention/RetentionWidgets.java create mode 100644 src/main/java/com/kingsrook/qbits/secureddata/retention/WipeProcesses.java create mode 100644 src/main/java/com/kingsrook/qbits/secureddata/retention/WipeService.java delete mode 100644 src/main/java/com/kingsrook/qbits/secureddata/standin/AiFieldMetaData.java delete mode 100644 src/main/java/com/kingsrook/qbits/secureddata/standin/AiTableMetaData.java rename src/main/java/com/kingsrook/qbits/secureddata/{support/UiApiBridge.java => standin/AuditValueProtectorInterface.java} (52%) create mode 100644 src/main/java/com/kingsrook/qbits/secureddata/standin/ClassifiedValueKeySource.java create mode 100644 src/main/java/com/kingsrook/qbits/secureddata/standin/ClassifiedValueTokens.java delete mode 100644 src/main/java/com/kingsrook/qbits/secureddata/standin/ClientChannelSession.java delete mode 100644 src/main/java/com/kingsrook/qbits/secureddata/standin/ClientChannelsMetaData.java rename src/{test/java-uiapi/com/kingsrook/qbits/secureddata/SecuredDataUiApiContractTest.java => main/java/com/kingsrook/qbits/secureddata/standin/DMLType.java} (64%) delete mode 100644 src/main/java/com/kingsrook/qbits/secureddata/standin/DataClassification.java create mode 100644 src/main/java/com/kingsrook/qbits/secureddata/standin/ProtectedAuditValues.java delete mode 100644 src/main/java/com/kingsrook/qbits/secureddata/standin/TableFeaturePermissionsMetaData.java delete mode 100644 src/main/java/com/kingsrook/qbits/secureddata/support/CoreMetadataBridge.java create mode 100644 src/main/java/com/kingsrook/qbits/secureddata/support/SecuredDataMetadata.java rename src/main/{java-uiapi => java}/com/kingsrook/qbits/secureddata/uiapi/SecuredDataExtension.java (95%) rename src/main/{java-uiapi => java}/com/kingsrook/qbits/secureddata/uiapi/SecuredDataRevealOptionsSpecV1.java (100%) rename src/main/{java-uiapi => java}/com/kingsrook/qbits/secureddata/uiapi/SecuredDataRevealSpecV1.java (100%) rename src/main/{java-uiapi => java}/com/kingsrook/qbits/secureddata/uiapi/SecuredDataSchemas.java (100%) rename src/main/{java-uiapi => java}/com/kingsrook/qbits/secureddata/uiapi/SecuredDataStatusSpecV1.java (100%) create mode 100644 src/test/java/com/kingsrook/qbits/secureddata/LogCapture.java create mode 100644 src/test/java/com/kingsrook/qbits/secureddata/RetentionProofTest.java create mode 100644 src/test/java/com/kingsrook/qbits/secureddata/RetentionTestSupport.java create mode 100644 src/test/java/com/kingsrook/qbits/secureddata/RetentionWipeTest.java rename src/test/{java-uiapi => java}/com/kingsrook/qbits/secureddata/SecuredDataHttpTest.java (100%) rename src/test/{java-uiapi/com/kingsrook/qbits/secureddata/UiApiContractAssertions.java => java/com/kingsrook/qbits/secureddata/SecuredDataUiApiContractTest.java} (71%) diff --git a/CHANGELOG.md b/CHANGELOG.md index da4bb7b..9c85461 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,22 @@ All notable changes are documented here, following [Keep a Changelog](https://ke ## [Unreleased] +### Changed — RC.6 migration +- Pin QQQ to 4.1.0-RC.6; use released classification, client-channel and table-feature APIs directly. +- Fold UI API code and tests into the main build and remove the profile and reflective registration. +- Remove released-API stand-ins and the local contract assertion facade; retain #1172 field-access stand-ins with RC.7 removal markers. + +### Added: phase (f) retention and wipe +- Retention rules, wipe markers, legal holds, wipe runs and a hidden, locked audit-key table, with the `secured_data.retention` permission and a "Retention rules" navigation entry. +- Actions NULL_WITH_MARKER, OVERWRITE_TEXT, KEYED_HASH, DELETE_RECORD and ARCHIVE_THEN_WIPE (host `ArchiveSinkInterface`), with validation and low-entropy refusal. +- Atomic wipe batches (fields, markers, key deletions, audits, access log and run row in one rollback-capable transaction), per-record and per-table legal holds, a Review/Result "Run wipe now" process and a daily scheduled run. +- Per-record-field AES-GCM wrapped DEKs (AAD-bound) for crypto-shredding of audit tokens, `CHANGED_ONLY` audit treatment, and the audit-value protector. +- App-home warning when known backup retention exceeds a rule's `age_days`; `withBackupRetentionDays` and `withArchiveSink` options; README retention section and the backups residual risk. +- Tests: all actions, holds, rollback, Review/Result, schedule, permissions, wrap/unwrap, and (on the SECUREDATA snapshot) the section 15 canary and wipe-proof tests. + +### Fixed +- The key source returned a DEK array it then zeroed; it now returns copies, with a regression test. + ### Added - Phase (e) configurable reveal-median, distinct-record and exact-match anomaly detection, with system access-log counters, bounded daily median caching, per-user/day alerts, and flags retained without an alert sink. diff --git a/README.md b/README.md index 42619ea..fd0c0c3 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,14 @@ # QBit Secured Data -Phases (b), (c), and (e) of [SECURED-DATA](https://github.com/QRun-IO/qqq/blob/develop/docs/SECURED-DATA.md): classification overrides, policy resolution, enumeration guardrails, and value-free access logging. QQQ core owns masking and query enforcement. Reveal, break-glass, the app home and anomaly alerts are implemented. Retention and audit-value protection remain later phases. +Phases (b), (c), (e) and (f) of [SECURED-DATA](https://github.com/QRun-IO/qqq/blob/develop/docs/SECURED-DATA.md): classification overrides, policy resolution, enumeration guardrails, and value-free access logging. QQQ core owns masking and query enforcement. Reveal, break-glass, the app home and anomaly alerts are implemented. Retention, wipe and audit-value protection are implemented (phase f); see Retention and wipe. ## Compatibility -Java 21, Maven wrapper 3.9.11, **QQQ 4.1.0-RC.5**. The released dependency pin is unchanged. +Java 21, Maven wrapper 3.9.11, **QQQ 4.1.0-RC.6**. -RC.5 contains AI exposure metadata, the alerts producer SPI and the tables picker. It does **not** contain field-access enforcement, `DataClassification`, classification properties on AI metadata, client channels, or table-feature permissions. Jar contents and embedded sources were inspected locally (`jar tf`; Git's `unzip` could not run in the Windows sandbox). +RC.6 supplies core classifications, AI metadata, client channels, table-feature permissions and the UI API extension points. These are used directly; the UI API and its contract test are part of the main build. The installed middleware jar provides `UiApiContract`, but no `UiApiContractAssertions` facade, so the contract test calls that released API directly. -Missing APIs live under `com.kingsrook.qbits.secureddata.standin`, with `STAND-IN` removal markers. `CoreFieldAccessBridge` registers a dynamic proxy only when core's `FieldAccessResolverMetaData` exists. It converts decision record components and event enums by name. On RC.5 it logs one INFO that enforcement requires RC.6+; installing this QBit on RC.5 **does not enforce field tiers**. - -`CoreMetadataBridge` promotes temporary classifications into core AI metadata when supported, registers the channel wall and table features reflectively, and uses core feature-permission checks when present. A missing API is supported; a present but incompatible API fails startup. The public classification API temporarily returns the stand-in `DataClassification` because the core enum is absent from RC.5. At RC.6, move these imports to core and delete the adapters/stand-ins. +Field access (#1172) and audit-value protection (#1182) are not in RC.6. Field-access stand-ins and `CoreFieldAccessBridge` remain marked for RC.7. RC.6 **does not enforce field tiers**. Integration tests can use `4.1.0-SECUREDATA-SNAPSHOT`. ## Install and register @@ -50,23 +48,23 @@ Every table has audit rules, explicit record sections and CONFIDENTIAL code clas Both table features are registered for **all tables**: `secured_data_full` (Secured data: full) and `secured_data_reveal` (Secured data: reveal). This deliberate catalog choice supports new admin classification rows without a restart. Core's permission-base, READ prerequisite and NOT_PROTECTED semantics are preserved: NOT_PROTECTED application tables grant their features without an explicit permission. Use protected table permission rules when field policies must restrict users. Feature grants have no standard WRITE/admin implication. -The assistant is denied `secured_data_*` through core client-channel metadata. The local custom checker enforces the same wall on RC.5 when the trusted host uses `standin.ClientChannelSession`. On newer core use `QSession.withClientChannel("assistant/t1")`. The normalized channel `assistant` is stored in logs and used for policy caps. A host must set the channel from trusted request context. +The assistant is denied `secured_data_*` through core client-channel metadata. Use `QSession.withClientChannel("assistant/t1")` from trusted request context. The normalized channel `assistant` is stored in logs and used for policy caps. ESB integration follows webhooks' host-wiring pattern: `SecuredDataTables.esbTableNames()` lists Classification, Policy, Access log and Break-glass. Connect those names to the host ESB table-event producer. No ESB or qbit-alerts dependency is added. ## Classifying fields -Code metadata establishes defaults. While pinned to RC.5: +Code metadata establishes defaults using the core APIs: ```java -import com.kingsrook.qbits.secureddata.standin.AiFieldMetaData; -import com.kingsrook.qbits.secureddata.standin.DataClassification; +import com.kingsrook.qqq.backend.core.actions.ai.AiFieldMetaData; +import com.kingsrook.qqq.backend.core.actions.ai.DataClassification; email.withSupplementalMetaData( new AiFieldMetaData().withClassification(DataClassification.PII)); ``` -The matching stand-in `AiTableMetaData` sets a table default. These supplemental objects coexist with RC.5's core AI exposure metadata. The producer promotes them into core AI metadata when the newer API is available. After moving to RC.6, use `com.kingsrook.qqq.backend.core.actions.ai.AiFieldMetaData` and core `DataClassification` with the same classification pattern. +Core `AiTableMetaData` sets a table default. Runtime classification rows take precedence over code metadata. Admin rows in Classification can raise **or lower** the default. Effective order: @@ -129,7 +127,7 @@ The `secured-data` extension is version **1.0.0**: - `GET /qqq/v1/qbit/secured-data/reveal/options?tableName=&fieldName=` - `GET /qqq/v1/qbit/secured-data/status` -Build with `-Duiapi=true` and a compatible core to compile `src/main/java-uiapi` and `src/test/java-uiapi`. The producer registers the extension only when both its classes and core's extension metadata exist. The committed contract is [secured-data.yaml](src/main/resources/openapi/qqq-v1/secured-data.yaml); regenerate intentionally with `-Dqqq.uiApi.regenerate=true`. The supplied core exposes `UiApiContract.check`, so a local test-only `UiApiContractAssertions` adapter supplies the documented assertion facade. All UI API roots and adapters are `STAND-IN(core=QRun-IO/qqq#1128, removeBy=RC.6)` and fold into the normal roots at RC.6. The core OpenAPI model cannot mark object properties required; the request schema documents required properties and the handler validates them. +The producer registers the UI API extension directly. The committed contract is [secured-data.yaml](src/main/resources/openapi/qqq-v1/secured-data.yaml); regenerate intentionally with `-Dqqq.uiApi.regenerate=true`. Contract and HTTP tests run in the normal build. The classification field picker uses CUSTOM possible values, reading `table_name` or `tableName` from the form's other values. It returns only the selected readable table's labeled fields, returns none without a selection, and rejects an unknown table. TODO: add `dependsOn("table_name")` when the core metadata API supports it. @@ -151,6 +149,8 @@ The classification field picker uses CUSTOM possible values, reading `table_name | `withDistinctValueAlert(int)` | 15; zero disables | | `withMinEntropyEnabled(boolean)` | true | | `withSchedulerName(String)` | Host scheduler for the 60-second break-glass expiry job; null means host-managed invocation | +| `withBackupRetentionDays(Integer)` | Unknown; physical backup window in days, used for the retention warning (must not be negative) | +| `withArchiveSink(ArchiveSinkInterface)` | None; required before any `ARCHIVE_THEN_WIPE` rule can run | | `withClock(Clock)` | UTC system clock; injectable for deterministic expiry tests | Configure before producing the QBit; runtime infrastructure and HMAC keys are captured at startup. Classification/policy/break-glass lookups always run as system, with a thread-local recursion guard. No reveal values are cached. @@ -159,7 +159,7 @@ Configure before producing the QBit; runtime infrastructure and HMAC keys are ca ```powershell ./mvnw.cmd -q -o clean verify -./mvnw.cmd -q -o clean verify -Dqqq.snapshot.version=4.1.0-SECUREDATA-SNAPSHOT -Duiapi=true +./mvnw.cmd -q -o clean verify -Dqqq.snapshot.version=4.1.0-SECUREDATA-SNAPSHOT ``` The direct suite uses memory and H2. `CoreFieldAccessBridgeIT` is included by Surefire and uses JUnit assumptions to skip unless the actual core resolver metadata is present. With the installed core snapshot it exercises Query/Get, policy tiers, masking, exact matches and outcomes, sorting denial, rate/entropy refusal and the assistant wall. Checkstyle and JaCoCo remain 70% instructions / 90% classes; stand-ins are not excluded. @@ -191,7 +191,7 @@ The single Secured Data app owns one full-width HTML home widget, following qbit Lists order newest first with id tie-breaking; rankings order by count descending then group alphabetically. Narrow screens use wrapping labelled card rows. Dates stay compact on one line, text is ellipsized with full tooltips, and colors inherit the host with `--qqq-grid-border` for borders. There are no inner scrollers. Empty lists explicitly say there is no activity. -Navigation contains Classification, Policy, Access log and Break-glass sessions. Reveal grants remain a child list; retention is not implemented. Successful policy/classification inserts, updates and deletes produce value-free change rows with actor, record id and action in the source transaction. No configuration values are copied into these rows. +Navigation contains Classification, Policy, Access log, Break-glass sessions and Retention rules. Reveal grants remain a child list. A warning line appears under the KPIs for each active retention rule whose known backup retention exceeds its `age_days`. Successful policy/classification inserts, updates and deletes produce value-free change rows with actor, record id and action in the source transaction. No configuration values are copied into these rows. ### Local screenshot harness @@ -205,9 +205,49 @@ java -cp $demoClasspath com.kingsrook.qbits.secureddata.SecuredDataDemoServer Open `http://localhost:8098/app/secured_data`. Stop the demo before `clean` on Windows. Capture light and dark at 390, 768, 1024, 1440, 1920 and 3200 px under `target/screenshots/`. This session started the server, but screenshot capture was blocked by Chromium `spawn EPERM` in the sandbox and two connected-browser timeouts. The coordinator still needs to capture and inspect the twelve images through Next. +## Retention and wipe + +Spec: SECURED-DATA sections 13 to 15. Retention rules live in the **Retention rules** table (permission `secured_data.retention`). Each rule names a host table, the fields to wipe, an age field, `age_days`, a batch size, an action and an audit treatment. + +Actions: + +- `NULL_WITH_MARKER`: set the fields to null (fields must be nullable) and record a wipe marker. +- `OVERWRITE_TEXT`: replace with the rule's text (STRING/TEXT fields only; the text must fit the field). +- `KEYED_HASH`: replace with an instance-key HMAC token (`hmac:v1:...`) that the host can still verify. Refused for low-entropy fields and for fields shorter than 200 characters. +- `DELETE_RECORD`: delete the record, with markers for every field. +- `ARCHIVE_THEN_WIPE`: write the field values as AES-GCM encrypted JSON through the host `ArchiveSinkInterface` (an adapter to object storage, ideally a WORM bucket), then null them. The wipe is refused if no sink is configured. + +Audit treatment: `SHREDDABLE` (default) gives each audited record-field its own data key (DEK), so deleting the key makes the audit tokens unverifiable (crypto-shredding). `CHANGED_ONLY` stores only `changed:v1` in audits and never creates a DEK. + +Mechanics: + +- Eligibility is evaluated in the database and returns record ids only; values are read only for `KEYED_HASH` and `ARCHIVE_THEN_WIPE`. +- Each batch is one transaction in a rollback-capable backend: field updates, wipe markers, audit-key deletions, audit details, the access-log `WIPE` rows and the run row commit or roll back together. Cross-backend or non-transactional deployments are refused before any write. +- Legal holds (per record, or per table with a blank record id) skip records; age and holds are re-checked inside the batch. +- **Run wipe now** is a standard Review/Result process with counts only; the hidden `secured_data_wipe_daily` process runs daily from the host scheduler (`withSchedulerName`). Neither places a host value in a message, log, run row or marker. +- DEKs are wrapped with AES-GCM under the KEK, with the table, field and record id as additional authenticated data. A wrong KEK or a wrong binding fails the unwrap. The wrapped-key table is hidden, locked, not exportable and audited at level NONE. + +### Backups are a residual risk (section 14.3) + +A wipe removes the value and its audit key from the live database only. Database backups taken before the wipe still contain the value and the wrapped DEK, and remain recoverable until they expire. Set `withBackupRetentionDays(...)`; when it exceeds a rule's `age_days` the rule and the app home warn that wiped values stay recoverable from backups. Unknown retention is reported as unknown, not as safe. Backups that must be purged earlier have to be handled outside this qbit. + +### Keys and environment + +| Variable | Purpose | +|---|---| +| `QQQ_AUDIT_HMAC_KEY` | Base64 32-byte instance key for `hmac:v1:` audit tokens and `KEYED_HASH`. Without it audits fall back to `changed:v1`. | +| `QQQ_AUDIT_HMAC_KEY_ID` | Optional id embedded in tokens, default `env1`; change it when rotating the key. | +| `QQQ_SECURED_DATA_KEK` | Base64 32-byte key-encryption key that wraps the per-record DEKs and archive objects. Required for `SHREDDABLE` and archive; a missing or invalid KEK fails closed. | + +Each is read from a JVM property or the environment and is never logged. Losing the KEK makes every existing DEK unrecoverable, so protect and back it up separately from the database. + +### Core hooks + +Audit-value protection (QRun-IO/qqq#1182) and the wipe audit flag need core classes that are not in RC.6. On RC.6 the qbit uses marked stand-ins (`removeBy=RC.7`), writes value-free wipe audit details itself, and core does not tokenise ordinary audits. Run the build with `-Dqqq.snapshot.version=4.1.0-SECUREDATA-SNAPSHOT` to exercise the real hooks, including the section 15 canary and wipe-proof tests (`RetentionProofTest`), which skip themselves when the hooks are absent. + ## Remaining phases -RC.6 migration removes temporary types and reflection. Retention, wipe and audit-value protection remain phases (f)/(g). +RC.6 migration is complete; field-access compatibility types remain until RC.7. Phase (g) remains. ## License diff --git a/docs/ddl/secured-data-postgres.sql b/docs/ddl/secured-data-postgres.sql index 2bcf9c8..1b220ef 100644 --- a/docs/ddl/secured-data-postgres.sql +++ b/docs/ddl/secured-data-postgres.sql @@ -86,7 +86,8 @@ CREATE TABLE secured_data_access_log ( anomaly_flag BOOLEAN DEFAULT false, break_glass_id INTEGER, created_at TIMESTAMP WITH TIME ZONE NOT NULL, - search_value_fingerprints TEXT + search_value_fingerprints TEXT, + rule_id INTEGER ); CREATE INDEX secured_data_policy_lookup ON secured_data_policy (is_active, permission_name, classification, table_name); @@ -97,3 +98,43 @@ CREATE INDEX secured_data_access_lookup ON secured_data_access_log (user_id, tab -- Phase (e): bounded user counters and home time windows. CREATE INDEX secured_data_anomaly_lookup ON secured_data_access_log (user_id, event_type, created_at); CREATE INDEX secured_data_home_lookup ON secured_data_access_log (event_type, created_at); + + +-- Phase (f): all policy targets and audit tables must share this transactional backend. +CREATE TABLE secured_data_retention_rule ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + name VARCHAR(255) NOT NULL, table_name VARCHAR(255) NOT NULL, field_names TEXT NOT NULL, + age_field_name VARCHAR(255) NOT NULL, age_days INTEGER NOT NULL CHECK (age_days >= 1), + action VARCHAR(40) DEFAULT 'NULL_WITH_MARKER' NOT NULL CHECK (action IN ('NULL_WITH_MARKER','OVERWRITE_TEXT','KEYED_HASH','DELETE_RECORD','ARCHIVE_THEN_WIPE')), + overwrite_text TEXT DEFAULT '[wiped per policy]', + audit_treatment VARCHAR(30) DEFAULT 'SHREDDABLE' NOT NULL CHECK (audit_treatment IN ('SHREDDABLE','CHANGED_ONLY')), + archive_storage_name VARCHAR(255), batch_size INTEGER DEFAULT 500 NOT NULL CHECK (batch_size >= 1), + is_active BOOLEAN DEFAULT true, last_run_at TIMESTAMP WITH TIME ZONE, last_run_count INTEGER, + create_date TIMESTAMP WITH TIME ZONE, modify_date TIMESTAMP WITH TIME ZONE, modified_by VARCHAR(255) +); +CREATE TABLE secured_data_wipe_run ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, rule_id INTEGER NOT NULL REFERENCES secured_data_retention_rule(id), + started_at TIMESTAMP WITH TIME ZONE NOT NULL, finished_at TIMESTAMP WITH TIME ZONE, status VARCHAR(30), + eligible_count INTEGER, held_count INTEGER, wiped_count INTEGER, error_count INTEGER, + record_ids VARCHAR(4000), message VARCHAR(500) +); +CREATE TABLE secured_data_wipe_marker ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, table_name VARCHAR(255) NOT NULL, record_id VARCHAR(255) NOT NULL, + field_name VARCHAR(255) NOT NULL, rule_id INTEGER NOT NULL REFERENCES secured_data_retention_rule(id), action VARCHAR(40) NOT NULL, + wiped_at TIMESTAMP WITH TIME ZONE NOT NULL, run_id INTEGER NOT NULL REFERENCES secured_data_wipe_run(id), + UNIQUE (table_name, record_id, field_name) +); +CREATE TABLE secured_data_legal_hold ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, table_name VARCHAR(255) NOT NULL, record_id VARCHAR(255), reason TEXT NOT NULL, + placed_by VARCHAR(255), placed_at TIMESTAMP WITH TIME ZONE, released_by VARCHAR(255), released_at TIMESTAMP WITH TIME ZONE, + status VARCHAR(20) DEFAULT 'ACTIVE' NOT NULL CHECK (status IN ('ACTIVE','RELEASED')) +); +-- Mutable key material: never register this table in ESB, WORM or long-term logical backups. +CREATE TABLE secured_data_audit_key ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, table_name VARCHAR(255) NOT NULL, + record_id VARCHAR(255) NOT NULL, field_name VARCHAR(255) NOT NULL, wrapped_key TEXT NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL, UNIQUE (table_name, record_id, field_name) +); +CREATE INDEX secured_data_hold_lookup ON secured_data_legal_hold (table_name, status, record_id); +CREATE INDEX secured_data_retention_lookup ON secured_data_retention_rule (table_name, is_active); +CREATE INDEX secured_data_wipe_rule_lookup ON secured_data_wipe_marker (rule_id); diff --git a/pom.xml b/pom.xml index 1e89ae3..a5bba5d 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,8 @@ - 4.1.0-RC.5 + + 4.1.0-RC.6 1.0.0-SNAPSHOT 21 21 @@ -215,33 +216,4 @@ - - - - uiapi - uiapitrue - - org.apache.maven.plugins - maven-checkstyle-plugin - - - ${project.basedir}/src/main/java - ${project.basedir}/src/main/java-uiapi - - - - org.codehaus.mojo - build-helper-maven-plugin - 3.6.1 - - uiapi-sourcegenerate-sourcesadd-source - src/main/java-uiapi - - uiapi-testsgenerate-test-sourcesadd-test-source - src/test/java-uiapi - - - - - diff --git a/src/main/java/com/kingsrook/qbits/secureddata/CoreAuditBridge.java b/src/main/java/com/kingsrook/qbits/secureddata/CoreAuditBridge.java new file mode 100644 index 0000000..f37aea1 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/secureddata/CoreAuditBridge.java @@ -0,0 +1,220 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.secureddata; + +import java.io.Serializable; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Proxy; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import com.kingsrook.qbits.secureddata.standin.ClassifiedValueKeySource; +import com.kingsrook.qbits.secureddata.standin.ClassifiedValueTokens; +import com.kingsrook.qbits.secureddata.standin.DMLType; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.audits.AuditAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.ActionFlag; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + +/******************************************************************************* + ** A single reflection boundary for #1182, failing closed for incompatible hooks. + *******************************************************************************/ +// STAND-IN(core=QRun-IO/qqq#1182, removeBy=RC.7) +public final class CoreAuditBridge +{ + private static final String TOKENS = "com.kingsrook.qqq.backend.core.actions.permissions.fieldaccess."; + private static final String AUDITS = "com.kingsrook.qqq.backend.core.actions.audits."; + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private CoreAuditBridge() + { + } + + /******************************************************************************* + ** Core hook availability (RC.6 has none). + *******************************************************************************/ + public static boolean available() + { + try + { + Class.forName(TOKENS + "ClassifiedValueTokens"); + return (true); + } + catch(ClassNotFoundException expected) + { + return (false); + } + } + + /******************************************************************************* + ** Register exact #1182 contracts when present; no silent partial registration. + *******************************************************************************/ + public static void register(QInstance instance, SecuredDataRuntime runtime) + { + if(!available()) + { + return; + } + try + { + Class contract = Class.forName(TOKENS + "ClassifiedValueKeySource"); + Object keyProxy = Proxy.newProxyInstance(contract.getClassLoader(), new Class[] {contract}, (self, method, args) -> + { + if(method.getDeclaringClass() == Object.class) + { + return (objectMethod(self, method.getName(), args)); + } + Optional key = switch(method.getName()) + { + case "instanceKey" -> runtime.keys().instanceKey(); + case "keyForRecordField" -> runtime.keys().keyForRecordField((String) args[0], (String) args[1], (Serializable) args[2], (Boolean) args[3]); + default -> throw new UnsupportedOperationException(method.getName()); + }; + return (key.isEmpty() ? Optional.empty() : Optional.of(Class.forName(TOKENS + "ClassifiedValueKeySource$KeyAndId") + .getConstructor(byte[].class, String.class).newInstance(key.get().key(), key.get().id()))); + }); + Class metadata = Class.forName(TOKENS + "ClassifiedValueKeySourceMetaData"); + Object registry = metadata.getMethod("ofOrWithNew", QInstance.class).invoke(null, instance); + metadata.getMethod("withKeySourceInstance", contract).invoke(registry, keyProxy); + SecuredDataAuditProtector protector = new SecuredDataAuditProtector(); + Class auditContract = Class.forName(AUDITS + "AuditValueProtectorInterface"); + Object auditProxy = Proxy.newProxyInstance(auditContract.getClassLoader(), new Class[] {auditContract}, (self, method, args) -> + { + if(method.getDeclaringClass() == Object.class) + { + return (objectMethod(self, method.getName(), args)); + } + if("classificationOf".equals(method.getName())) + { + return (protector.classificationOf((QTableMetaData) args[0], (QFieldMetaData) args[1])); + } + if(!"protect".equals(method.getName())) + { + throw (new UnsupportedOperationException(method.getName())); + } + var values = protector.protect((QTableMetaData) args[0], (QFieldMetaData) args[1], (Serializable) args[2], (Serializable) args[3], (Serializable) args[4], DMLType.valueOf(args[5].toString())); + return (Class.forName(AUDITS + "ProtectedAuditValues").getConstructor(String.class, String.class, String.class) + .newInstance(values.oldToken(), values.newToken(), values.message())); + }); + metadata = Class.forName(AUDITS + "AuditValueProtectionMetaData"); + registry = metadata.getMethod("ofOrWithNew", QInstance.class).invoke(null, instance); + metadata.getMethod("withProtectorInstance", auditContract).invoke(registry, auditProxy); + } + catch(ReflectiveOperationException e) + { + throw (new IllegalStateException("Unable to register core audit protection", e)); + } + } + + /******************************************************************************* + ** Stable object behavior for registered service proxies. + *******************************************************************************/ + private static Object objectMethod(Object self, String name, Object[] args) + { + return (switch(name) + { + case "toString" -> "Secured-data audit bridge"; + case "hashCode" -> System.identityHashCode(self); + case "equals" -> self == args[0]; + default -> throw new UnsupportedOperationException(name); + }); + } + + /******************************************************************************* + ** Delegate to the installed core helper, with contract-identical RC.6 fallback. + *******************************************************************************/ + public static String token(QTableMetaData table, QFieldMetaData field, Serializable pk, Serializable value) + { + if(!available()) + { + return (ClassifiedValueTokens.forRecordField(table, field, pk, value)); + } + try + { + return ((String) Class.forName(TOKENS + "ClassifiedValueTokens").getMethod("forRecordField", QTableMetaData.class, QFieldMetaData.class, Serializable.class, Serializable.class) + .invoke(null, table, field, pk, value)); + } + catch(InvocationTargetException e) + { + throw (new IllegalStateException("Classified token creation failed")); + } + catch(ReflectiveOperationException e) + { + throw (new IllegalStateException("Incompatible core token API", e)); + } + } + + /******************************************************************************* + ** Real WipeAuditFlag on snapshot; RC.6 uses a value-free explicit audit below. + *******************************************************************************/ + public static void flag(UpdateInput input, String ruleId) + { + if(!available()) + { + input.setOmitDmlAudit(true); + return; + } + try + { + ActionFlag flag = (ActionFlag) Class.forName("com.kingsrook.qqq.backend.core.model.actions.tables.WipeAuditFlag").getConstructor(String.class).newInstance(ruleId); + input.setFlags(Set.of(flag)); + } + catch(ReflectiveOperationException e) + { + throw (new IllegalStateException("Incompatible core wipe API", e)); + } + } + + /******************************************************************************* + ** Append immutable value-free audit evidence in the batch transaction. + *******************************************************************************/ + public static void detail(QTableMetaData table, Serializable pk, String field, String ruleId, boolean shredded, QBackendTransaction transaction) throws QException + { + if(shredded && available()) + { + try + { + Class.forName(AUDITS + "AuditShreddedEvent").getMethod("shreddedDetail", QTableMetaData.class, Serializable.class, QFieldMetaData.class, String.class, QBackendTransaction.class) + .invoke(null, table, pk, table.getField(field), ruleId, transaction); + return; + } + catch(ReflectiveOperationException e) + { + throw (new QException("Unable to append shredded audit detail")); + } + } + String message = (shredded ? "Audit values shredded" : "Wiped") + " per policy"; + QRecord detail = new QRecord().withValue("fieldName", field).withValue("newValue", (shredded ? "shredded:v1:" : "wiped:v1:") + ruleId).withValue("message", message); + AuditInput input = new AuditInput(); + input.setTransaction(transaction); + AuditAction.appendToInput(input, table.getName(), pk, AuditAction.getRecordSecurityKeyValues(table, + new QRecord().withValue(table.getPrimaryKeyField(), pk), Optional.empty()), message, List.of(detail)); + new AuditAction().execute(input); + } +} diff --git a/src/main/java/com/kingsrook/qbits/secureddata/CoreFieldAccessBridge.java b/src/main/java/com/kingsrook/qbits/secureddata/CoreFieldAccessBridge.java index 0ca011c..b299aed 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/CoreFieldAccessBridge.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/CoreFieldAccessBridge.java @@ -40,6 +40,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; /******************************************************************************* ** The only boundary between the temporary field-access API and unreleased core. *******************************************************************************/ +// STAND-IN(core=QRun-IO/qqq#1172, removeBy=RC.7) public final class CoreFieldAccessBridge { private static final String PACKAGE = "com.kingsrook.qqq.backend.core.actions.permissions.fieldaccess."; diff --git a/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataAccessLogger.java b/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataAccessLogger.java index 29abf82..8cf0822 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataAccessLogger.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataAccessLogger.java @@ -41,7 +41,7 @@ import com.kingsrook.qbits.secureddata.metadata.SecuredDataTables; import com.kingsrook.qbits.secureddata.standin.FieldAccessEvent; import com.kingsrook.qbits.secureddata.standin.FieldAccessEventType; import com.kingsrook.qbits.secureddata.standin.FieldAccessOutcome; -import com.kingsrook.qbits.secureddata.support.CoreMetadataBridge; +import com.kingsrook.qbits.secureddata.support.SecuredDataMetadata; import com.kingsrook.qbits.secureddata.support.SystemWrites; import com.kingsrook.qqq.backend.core.alerts.AlertSignal; import com.kingsrook.qqq.backend.core.alerts.QAlerts; @@ -179,7 +179,7 @@ public class SecuredDataAccessLogger implements AutoCloseable .withValue("user_name", session == null || session.getUser() == null ? null : session.getUser().getFullName()) .withValue("table_name", event.getTableName()).withValue("field_name", String.join(",", fields)) .withValue("record_count", event.getRecordCount()).withValue("action_name", event.getActionName()) - .withValue("client_channel", CoreMetadataBridge.channel(session)).withValue("outcome", outcome) + .withValue("client_channel", SecuredDataMetadata.channel(session)).withValue("outcome", outcome) .withValue("anomaly_flag", false).withValue("created_at", runtime.config().getClock().instant()); if(event.getPrimaryKeys() != null) { diff --git a/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataAuditProtector.java b/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataAuditProtector.java new file mode 100644 index 0000000..a3630e1 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataAuditProtector.java @@ -0,0 +1,62 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.secureddata; + +import java.io.Serializable; +import com.kingsrook.qbits.secureddata.retention.RetentionRules; +import com.kingsrook.qbits.secureddata.standin.AuditValueProtectorInterface; +import com.kingsrook.qbits.secureddata.standin.DMLType; +import com.kingsrook.qbits.secureddata.standin.ProtectedAuditValues; +import com.kingsrook.qqq.backend.core.actions.ai.DataClassification; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + +/******************************************************************************* + ** Classifications and policy determine whether audits retain a keyed derivative. + *******************************************************************************/ +public final class SecuredDataAuditProtector implements AuditValueProtectorInterface +{ + /******************************************************************************* + ** Core also uses this source for runtime classification overrides. + *******************************************************************************/ + @Override + public DataClassification classificationOf(QTableMetaData table, QFieldMetaData field) + { + DataClassification classification = SecuredDataClassifications.effective(table, field.getName()); + if(com.kingsrook.qbits.secureddata.metadata.SecuredDataTables.ALL.contains(table.getName())) + { + return (classification); + } + return (RetentionRules.active(table.getName(), field.getName()).isEmpty() ? classification : DataClassification.max(classification, DataClassification.PII)); + } + + /******************************************************************************* + ** Null stays absent; CHANGED_ONLY never creates a DEK. + *******************************************************************************/ + @Override + public ProtectedAuditValues protect(QTableMetaData table, QFieldMetaData field, Serializable pk, Serializable oldValue, Serializable newValue, DMLType dmlType) + { + boolean changed = RetentionRules.changedOnly(table.getName(), field.getName()); + String message = (oldValue == null ? "Set " : newValue == null ? "Removed " : "Changed ") + field.getLabel(); + return (new ProtectedAuditValues(oldValue == null ? null : changed ? "changed:v1" : CoreAuditBridge.token(table, field, pk, oldValue), + newValue == null ? null : changed ? "changed:v1" : CoreAuditBridge.token(table, field, pk, newValue), message)); + } +} diff --git a/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataClassifications.java b/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataClassifications.java index 19cf833..6347380 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataClassifications.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataClassifications.java @@ -23,8 +23,8 @@ package com.kingsrook.qbits.secureddata; import java.util.Collection; import java.util.Objects; -import com.kingsrook.qbits.secureddata.standin.DataClassification; -import com.kingsrook.qbits.secureddata.support.CoreMetadataBridge; +import com.kingsrook.qbits.secureddata.support.SecuredDataMetadata; +import com.kingsrook.qqq.backend.core.actions.ai.DataClassification; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -55,8 +55,8 @@ public final class SecuredDataClassifications return (DataClassification.valueOf(row.getValueString("classification"))); } QFieldMetaData field = table.getField(fieldName); - DataClassification classification = field == null ? null : CoreMetadataBridge.classification(table, field); - return (classification == null ? CoreMetadataBridge.classification(table, null) : classification); + DataClassification classification = field == null ? null : SecuredDataMetadata.classification(table, field); + return (classification == null ? SecuredDataMetadata.classification(table, null) : classification); } @@ -67,7 +67,7 @@ public final class SecuredDataClassifications public static DataClassification effective(QTableMetaData table) { QRecord row = row(table, null); - return (row == null ? CoreMetadataBridge.classification(table, null) : DataClassification.valueOf(row.getValueString("classification"))); + return (row == null ? SecuredDataMetadata.classification(table, null) : DataClassification.valueOf(row.getValueString("classification"))); } @@ -126,4 +126,12 @@ public final class SecuredDataClassifications } return (tableRow); } + + /******************************************************************************* + ** Shared phase-b entropy guardrail, including runtime overrides. + *******************************************************************************/ + public static boolean lowEntropy(QTableMetaData table, String field) + { + return (SecuredDataResolver.lowCardinality(table.getField(field), row(table, field))); + } } diff --git a/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataKeySource.java b/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataKeySource.java new file mode 100644 index 0000000..cdf6c15 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataKeySource.java @@ -0,0 +1,290 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.secureddata; + +import javax.crypto.Cipher; +import javax.crypto.spec.GCMParameterSpec; +import javax.crypto.spec.SecretKeySpec; +import java.io.Serializable; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.GeneralSecurityException; +import java.security.SecureRandom; +import java.time.Instant; +import java.util.Arrays; +import java.util.Base64; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import com.kingsrook.qbits.secureddata.metadata.SecuredDataTables; +import com.kingsrook.qbits.secureddata.retention.RetentionRules; +import com.kingsrook.qbits.secureddata.standin.ClassifiedValueKeySource; +import com.kingsrook.qbits.secureddata.support.SystemWrites; +import com.kingsrook.qqq.backend.core.actions.ai.DataClassification; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; + +/******************************************************************************* + ** Per-instance audit DEKs. Cache hits still require the key row to exist, so + ** another node's committed deletion takes effect immediately on the next lookup. + *******************************************************************************/ +public final class SecuredDataKeySource implements ClassifiedValueKeySource, AutoCloseable +{ + private final ThreadLocal instanceOnly = ThreadLocal.withInitial(() -> false); + private static final SecureRandom RANDOM = new SecureRandom(); + private final Map cache = new LinkedHashMap<>(); + + /******************************************************************************* + ** Short-lived cache; never populated by uncommitted work. + *******************************************************************************/ + private record CachedKey(byte[] key, Instant expires) + { + } + + /******************************************************************************* + ** Use core's interpreter directly: convenience environment getters log values. + *******************************************************************************/ + public static String environment(String name) + { + return (new QMetaDataVariableInterpreter().interpret("${prop." + name + "}??${env." + name + "}")); + } + + /******************************************************************************* + ** Instance HMAC version, without retaining or logging configuration text. + *******************************************************************************/ + @Override + public Optional instanceKey() + { + String encoded = environment("QQQ_AUDIT_HMAC_KEY"); + if(encoded == null || encoded.isBlank()) + { + return (Optional.empty()); + } + try + { + String id = environment("QQQ_AUDIT_HMAC_KEY_ID"); + return (Optional.of(new KeyAndId(Base64.getDecoder().decode(encoded), id == null || id.isBlank() ? "env1" : id))); + } + catch(IllegalArgumentException e) + { + throw (new IllegalStateException("Invalid audit HMAC key configuration")); + } + } + + /******************************************************************************* + ** Policies and entropy share the phase-b classification rules. + *******************************************************************************/ + public boolean shreds(String table, String field) + { + var metadata = QContext.getQInstance().getTable(table); + DataClassification classification = SecuredDataClassifications.effective(metadata, field); + return (RetentionRules.active(table, field).stream().anyMatch(rule -> "SHREDDABLE".equals(rule.getValueString("audit_treatment"))) + || ((classification == DataClassification.PII || classification == DataClassification.PHI) && SecuredDataClassifications.lowEntropy(metadata, field))); + } + + /******************************************************************************* + ** Verification never creates. Missing KEK fails closed rather than falling back + ** to an instance HMAC for a field that must be shredded. + *******************************************************************************/ + @Override + public synchronized Optional keyForRecordField(String table, String field, Serializable pk, boolean create) + { + if(instanceOnly.get()) + { + return (Optional.empty()); + } + String binding = table + "\u001f" + field + "\u001f" + pk; + try + { + List rows = SystemWrites.call(() -> new QueryAction().execute(new QueryInput(SecuredDataTables.AUDIT_KEY) + .withFilter(recordField(table, field, pk)).withShouldOmitHiddenFields(false) + .withTransaction(QContext.getQBackendTransaction())).getRecords()); + if(rows.isEmpty()) + { + evict(binding); + if(!create || RetentionRules.changedOnly(table, field) || !shreds(table, field)) + { + return (Optional.empty()); + } + byte[] dek = new byte[32]; + RANDOM.nextBytes(dek); + try + { + String wrapped = Base64.getEncoder().encodeToString(encrypt(dek, binding)); + QRecord row = SystemWrites.insertOne(SecuredDataTables.AUDIT_KEY, new QRecord() + .withValue("table_name", table).withValue("field_name", field).withValue("record_id", String.valueOf(pk)) + .withValue("wrapped_key", wrapped).withValue("created_at", SecuredDataRuntime.current().config().getClock().instant())); + return (Optional.of(new KeyAndId(dek.clone(), "dek" + row.getValueString("id")))); + } + finally + { + Arrays.fill(dek, (byte) 0); + } + } + QRecord row = rows.getFirst(); + Instant now = SecuredDataRuntime.current().config().getClock().instant(); + CachedKey cached = cache.get(binding); + if(cached != null && cached.expires().isAfter(now) && QContext.getQBackendTransaction() == null) + { + return (Optional.of(new KeyAndId(cached.key().clone(), "dek" + row.getValueString("id")))); + } + evict(binding); + byte[] dek = decrypt(Base64.getDecoder().decode(row.getValueString("wrapped_key")), binding); + try + { + if(QContext.getQBackendTransaction() == null) + { + if(cache.size() >= 128) + { + close(); + } + cache.put(binding, new CachedKey(dek.clone(), now.plusSeconds(5))); + } + return (Optional.of(new KeyAndId(dek.clone(), "dek" + row.getValueString("id")))); + } + finally + { + Arrays.fill(dek, (byte) 0); + } + } + catch(QException e) + { + throw (new IllegalStateException("Unable to access audit key storage")); + } + } + + /******************************************************************************* + ** Exact key identity, shared by deletion and lookups. + *******************************************************************************/ + public static QQueryFilter recordField(String table, String field, Serializable pk) + { + return (new QQueryFilter().withCriteria(new QFilterCriteria("table_name", QCriteriaOperator.EQUALS, table)) + .withCriteria(new QFilterCriteria("field_name", QCriteriaOperator.EQUALS, field)) + .withCriteria(new QFilterCriteria("record_id", QCriteriaOperator.EQUALS, String.valueOf(pk)))); + } + + /******************************************************************************* + ** Erase cached key bytes, including on rollback and runtime shutdown. + *******************************************************************************/ + private void evict(String binding) + { + CachedKey old = cache.remove(binding); + if(old != null) + { + Arrays.fill(old.key(), (byte) 0); + } + } + + /******************************************************************************* + ** AES-GCM envelope: version byte, 12-byte random nonce, ciphertext and tag. + ** The binding is authenticated additional data and prevents key-row swapping. + *******************************************************************************/ + public static byte[] encrypt(byte[] plaintext, String binding) + { + byte[] nonce = new byte[12]; + RANDOM.nextBytes(nonce); + byte[] ciphertext = crypt(Cipher.ENCRYPT_MODE, plaintext, nonce, binding); + return (ByteBuffer.allocate(13 + ciphertext.length).put((byte) 1).put(nonce).put(ciphertext).array()); + } + + /******************************************************************************* + ** Authenticate before exposing any plaintext. + *******************************************************************************/ + public static byte[] decrypt(byte[] envelope, String binding) + { + if(envelope.length < 29 || envelope[0] != 1) + { + throw (new IllegalArgumentException("Invalid encrypted envelope")); + } + return (crypt(Cipher.DECRYPT_MODE, Arrays.copyOfRange(envelope, 13, envelope.length), Arrays.copyOfRange(envelope, 1, 13), binding)); + } + + /******************************************************************************* + ** Require exactly a 256-bit KEK; never include secret text in failures. + *******************************************************************************/ + private static byte[] crypt(int mode, byte[] input, byte[] nonce, String binding) + { + byte[] kek = null; + try + { + String encoded = environment("QQQ_SECURED_DATA_KEK"); + kek = encoded == null ? new byte[0] : Base64.getDecoder().decode(encoded); + if(kek.length != 32) + { + throw (new IllegalArgumentException("QQQ_SECURED_DATA_KEK must be base64 for 32 bytes")); + } + Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding"); + cipher.init(mode, new SecretKeySpec(kek, "AES"), new GCMParameterSpec(128, nonce)); + cipher.updateAAD(binding.getBytes(StandardCharsets.UTF_8)); + return (cipher.doFinal(input)); + } + catch(GeneralSecurityException | IllegalArgumentException e) + { + throw (new IllegalStateException("Unable to protect audit data; check QQQ_SECURED_DATA_KEK and envelope integrity")); + } + finally + { + if(kek != null) + { + Arrays.fill(kek, (byte) 0); + } + } + } + + /******************************************************************************* + ** Clear every decrypted cached key. + *******************************************************************************/ + @Override + public synchronized void close() + { + cache.values().forEach(key -> Arrays.fill(key.key(), (byte) 0)); + cache.clear(); + } + + /******************************************************************************* + ** KEYED_HASH deliberately uses only the instance key and record binding. + *******************************************************************************/ + public String instanceToken(com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData table, + com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData field, Serializable pk, Serializable value) + { + if(instanceKey().isEmpty()) + { + throw (new IllegalStateException("KEYED_HASH requires QQQ_AUDIT_HMAC_KEY")); + } + instanceOnly.set(true); + try + { + return (CoreAuditBridge.token(table, field, pk, value)); + } + finally + { + instanceOnly.remove(); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataQBitConfig.java b/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataQBitConfig.java index 949b94c..10e5f8a 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataQBitConfig.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataQBitConfig.java @@ -29,8 +29,8 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.function.Supplier; -import com.kingsrook.qbits.secureddata.standin.DataClassification; import com.kingsrook.qbits.secureddata.standin.FieldAccessTier; +import com.kingsrook.qqq.backend.core.actions.ai.DataClassification; import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface; import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -49,6 +49,69 @@ public class SecuredDataQBitConfig implements QBitConfig private int anomalyExactMatchesPerHour = 50; private String tablePathPrefix = "/app"; private String schedulerName; + private Integer backupRetentionDays; + private int wipeIntervalSeconds = 86400; + private com.kingsrook.qbits.secureddata.retention.ArchiveSinkInterface archiveSink; + + /******************************************************************************* + ** Known CNPG/WAL backup retention; null means unknown. + *******************************************************************************/ + public Integer getBackupRetentionDays() + { + return (backupRetentionDays); + } + + /******************************************************************************* + ** Configure the physical backup recovery window. + *******************************************************************************/ + public SecuredDataQBitConfig withBackupRetentionDays(Integer days) + { + if(days != null && days < 0) + { + throw (new IllegalArgumentException("Backup retention cannot be negative")); + } + backupRetentionDays = days; + return (this); + } + + /******************************************************************************* + ** Optional archive adapter. + *******************************************************************************/ + public com.kingsrook.qbits.secureddata.retention.ArchiveSinkInterface getArchiveSink() + { + return (archiveSink); + } + + /******************************************************************************* + ** Supply a durable encrypted-object sink. + *******************************************************************************/ + public SecuredDataQBitConfig withArchiveSink(com.kingsrook.qbits.secureddata.retention.ArchiveSinkInterface sink) + { + archiveSink = sink; + return (this); + } + + /******************************************************************************* + ** Daily by default. + *******************************************************************************/ + public int getWipeIntervalSeconds() + { + return (wipeIntervalSeconds); + } + + /******************************************************************************* + ** Configure the host scheduler interval. + *******************************************************************************/ + public SecuredDataQBitConfig withWipeIntervalSeconds(int seconds) + { + if(seconds < 1) + { + throw (new IllegalArgumentException("Wipe interval must be positive")); + } + wipeIntervalSeconds = seconds; + return (this); + } + private String backendName = "memory"; private Clock clock = Clock.systemUTC(); private int accessLogQueueCapacity = 4096; diff --git a/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataQBitProducer.java b/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataQBitProducer.java index 1a450ec..f9041f0 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataQBitProducer.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataQBitProducer.java @@ -27,8 +27,8 @@ import java.util.Locale; import java.util.Map; import com.kingsrook.qbits.secureddata.metadata.SecuredDataTableCustomizers; import com.kingsrook.qbits.secureddata.metadata.SecuredDataTables; -import com.kingsrook.qbits.secureddata.standin.DataClassification; -import com.kingsrook.qbits.secureddata.support.CoreMetadataBridge; +import com.kingsrook.qbits.secureddata.support.SecuredDataMetadata; +import com.kingsrook.qqq.backend.core.actions.ai.DataClassification; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput; @@ -153,21 +153,26 @@ public class SecuredDataQBitProducer implements QBitMetaDataProducer matching(QTableMetaData table, DataClassification classification) { - String channel = CoreMetadataBridge.channel(QContext.getQSession()); + String channel = SecuredDataMetadata.channel(QContext.getQSession()); return (state().policies().stream().filter(row -> classification != null && classification.name().equals(row.getValueString("classification"))) .filter(row -> blank(row.getValueString("table_name")) || table.getName().equals(row.getValueString("table_name"))) .filter(row -> blank(row.getValueString("client_channel")) || Objects.equals(channel, row.getValueString("client_channel"))).toList()); @@ -372,7 +372,7 @@ public class SecuredDataResolver implements FieldAccessResolverInterface } for(QTableMetaData table : QContext.getQInstance().getTables().values()) { - String base = CoreMetadataBridge.permissionBase(table); + String base = SecuredDataMetadata.permissionBase(table); for(String suffix : List.of(SecuredDataPermissions.FULL, SecuredDataPermissions.REVEAL)) { if(session != null && session.hasPermission(base + "." + suffix)) @@ -383,7 +383,7 @@ public class SecuredDataResolver implements FieldAccessResolverInterface } List breaks = activeBreakGlass().stream().map(row -> row.getValueString("id")).sorted().toList(); String payload = JsonUtils.toJson(List.of(userId(), relevant, runtime.policyVersion(), runtime.classificationVersion(), breaks, - Objects.requireNonNullElse(CoreMetadataBridge.channel(session), ""))); + Objects.requireNonNullElse(SecuredDataMetadata.channel(session), ""))); try { return (Base64.getUrlEncoder().withoutPadding().encodeToString(MessageDigest.getInstance("SHA-256").digest(payload.getBytes(StandardCharsets.UTF_8)))); diff --git a/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataRuntime.java b/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataRuntime.java index af8ac2f..cc2ef2b 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataRuntime.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataRuntime.java @@ -39,6 +39,7 @@ public class SecuredDataRuntime implements QSupplementalInstanceMetaData, AutoCl private final AtomicLong policyVersion = new AtomicLong(STARTUP_VERSION.incrementAndGet()); private final transient SecuredDataQBitConfig config; private final transient SecuredDataResolver resolver; + private final transient SecuredDataKeySource keys = new SecuredDataKeySource(); private final transient SecuredDataAccessLogger logger; private final transient SecuredDataAnomalies anomalies; private transient long cachedClassificationVersion; @@ -173,6 +174,7 @@ public class SecuredDataRuntime implements QSupplementalInstanceMetaData, AutoCl @Override public void close() { + keys.close(); logger.close(); } @@ -183,4 +185,12 @@ public class SecuredDataRuntime implements QSupplementalInstanceMetaData, AutoCl { return (anomalies); } + + /******************************************************************************* + ** Instance-scoped audit keys. + *******************************************************************************/ + public SecuredDataKeySource keys() + { + return (keys); + } } diff --git a/src/main/java/com/kingsrook/qbits/secureddata/app/SecuredDataHomeData.java b/src/main/java/com/kingsrook/qbits/secureddata/app/SecuredDataHomeData.java index 161b851..1a98d2c 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/app/SecuredDataHomeData.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/app/SecuredDataHomeData.java @@ -99,6 +99,14 @@ public class SecuredDataHomeData List.of("user_name", "table_name", "record_id", "action_name", "created_at"), List.of("User", "Table", "Record", "Action", "When")); data.recent("Anomaly flags", "No anomaly flags in the last 7 days", anomalies, List.of("user_name", "event_type", "table_name", "field_name", "created_at"), List.of("User", "Event", "Table", "Field", "When")); + for(QRecord rule : new QueryAction().execute(new QueryInput(SecuredDataTables.RETENTION_RULE)).getRecords()) + { + String warning = com.kingsrook.qbits.secureddata.retention.RetentionRules.backupWarning(rule); + if(warning != null) + { + data.backupWarnings.add(rule.getValueString("name") + ": " + warning); + } + } return (data); } @@ -182,4 +190,14 @@ public class SecuredDataHomeData { return (sections); } + + private final List backupWarnings = new ArrayList<>(); + + /******************************************************************************* + ** One warning for each policy outlived by known physical backups. + *******************************************************************************/ + public List getBackupWarnings() + { + return (backupWarnings); + } } diff --git a/src/main/java/com/kingsrook/qbits/secureddata/app/SecuredDataHomeView.java b/src/main/java/com/kingsrook/qbits/secureddata/app/SecuredDataHomeView.java index 3b1b255..6a3253c 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/app/SecuredDataHomeView.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/app/SecuredDataHomeView.java @@ -53,6 +53,10 @@ public final class SecuredDataHomeView .append("\">
").append(value) .append("
").append(esc(label)).append("
")); html.append(""); + for(String warning : data.getBackupWarnings()) + { + html.append("

").append(esc(warning)).append("

"); + } for(SecuredDataHomeData.Section section : data.getSections()) { html.append("

") diff --git a/src/main/java/com/kingsrook/qbits/secureddata/breakglass/BreakGlassService.java b/src/main/java/com/kingsrook/qbits/secureddata/breakglass/BreakGlassService.java index 1a5a87c..60a8b1f 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/breakglass/BreakGlassService.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/breakglass/BreakGlassService.java @@ -28,10 +28,10 @@ import com.kingsrook.qbits.secureddata.SecuredDataResolver; import com.kingsrook.qbits.secureddata.SecuredDataRuntime; import com.kingsrook.qbits.secureddata.metadata.SecuredDataTables; import com.kingsrook.qbits.secureddata.permissions.SecuredDataPermissions; -import com.kingsrook.qbits.secureddata.standin.DataClassification; import com.kingsrook.qbits.secureddata.support.ActivityLog; -import com.kingsrook.qbits.secureddata.support.CoreMetadataBridge; +import com.kingsrook.qbits.secureddata.support.SecuredDataMetadata; import com.kingsrook.qbits.secureddata.support.SystemWrites; +import com.kingsrook.qqq.backend.core.actions.ai.DataClassification; import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; import com.kingsrook.qqq.backend.core.alerts.AlertSeverity; import com.kingsrook.qqq.backend.core.alerts.AlertSignal; @@ -235,7 +235,7 @@ public class BreakGlassService { if(QContext.getQSession() == null || QContext.getQSession().getUser() == null || "anonymous".equals(SecuredDataResolver.userId()) || SecuredDataResolver.userId().isBlank() - || "assistant".equals(CoreMetadataBridge.channel(QContext.getQSession()))) + || "assistant".equals(SecuredDataMetadata.channel(QContext.getQSession()))) { throw (new QPermissionDeniedException("Permission denied.")); } diff --git a/src/main/java/com/kingsrook/qbits/secureddata/metadata/RetentionTables.java b/src/main/java/com/kingsrook/qbits/secureddata/metadata/RetentionTables.java new file mode 100644 index 0000000..37b8f1b --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/secureddata/metadata/RetentionTables.java @@ -0,0 +1,134 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.secureddata.metadata; + +import java.util.List; +import com.kingsrook.qbits.secureddata.SecuredDataQBitConfig; +import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel; +import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.SectionFactory; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import static com.kingsrook.qbits.secureddata.metadata.SecuredDataTables.AUDIT_KEY; +import static com.kingsrook.qbits.secureddata.metadata.SecuredDataTables.LEGAL_HOLD; +import static com.kingsrook.qbits.secureddata.metadata.SecuredDataTables.RETENTION_RULE; +import static com.kingsrook.qbits.secureddata.metadata.SecuredDataTables.WIPE_MARKER; +import static com.kingsrook.qbits.secureddata.metadata.SecuredDataTables.WIPE_RUN; +import static com.kingsrook.qbits.secureddata.metadata.SecuredDataTables.base; +import static com.kingsrook.qbits.secureddata.metadata.SecuredDataTables.field; + +/******************************************************************************* + ** Retention policies, value-free history, legal holds and mutable audit keys. + *******************************************************************************/ +public final class RetentionTables +{ + public static final String KEY_LOCK = "secured_data_internal_key"; + + /** A key value no session is ever given, so the lock denies every user (validator requires a non-empty override list). */ + public static final String KEY_LOCK_NEVER_GRANTED = "secured_data_never_granted"; + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private RetentionTables() + { + } + + /******************************************************************************* + ** No backup or WORM registration is made for the key table. + *******************************************************************************/ + public static List tables(SecuredDataQBitConfig config) + { + QTableMetaData rule = base(config, RETENTION_RULE, "Retention rules") + .withField(field("name", QFieldType.STRING).withMaxLength(255).withIsRequired(true)) + .withField(field("table_name", QFieldType.STRING).withMaxLength(255).withIsRequired(true).withPossibleValueSourceName("tables")) + .withField(field("field_names", QFieldType.TEXT).withIsRequired(true)) + .withField(field("age_field_name", QFieldType.STRING).withMaxLength(255).withIsRequired(true).withPossibleValueSourceName("secured_data_fields")) + .withField(field("age_days", QFieldType.INTEGER).withIsRequired(true)) + .withField(field("action", QFieldType.STRING).withMaxLength(40).withDefaultValue("NULL_WITH_MARKER")) + .withField(field("overwrite_text", QFieldType.TEXT).withDefaultValue("[wiped per policy]")) + .withField(field("audit_treatment", QFieldType.STRING).withMaxLength(30).withDefaultValue("SHREDDABLE")) + .withField(field("archive_storage_name", QFieldType.STRING).withMaxLength(255)) + .withField(field("batch_size", QFieldType.INTEGER).withDefaultValue(500)) + .withField(field("is_active", QFieldType.BOOLEAN).withDefaultValue(true)) + .withField(field("last_run_at", QFieldType.DATE_TIME).withIsEditable(false)) + .withField(field("last_run_count", QFieldType.INTEGER).withIsEditable(false)) + .withField(field("create_date", QFieldType.DATE_TIME)) + .withField(field("modify_date", QFieldType.DATE_TIME)) + .withField(field("modified_by", QFieldType.STRING).withMaxLength(255)); + rule.withSection(SectionFactory.defaultT1("id", "name", "table_name", "field_names")); + rule.withSection(SectionFactory.defaultT2("age_field_name", "age_days", "action", "overwrite_text", "audit_treatment", "archive_storage_name", "batch_size", "is_active")); + rule.withSection(SectionFactory.defaultT3("last_run_at", "last_run_count", "create_date", "modify_date", "modified_by")); + rule.withSection(SectionFactory.customT2("backups", new com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon("backup")) + .withLabel("Backups").withWidgetName(com.kingsrook.qbits.secureddata.retention.RetentionWidgets.BACKUPS)); + rule.withSection(SectionFactory.customT2("markers", new com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon("delete")) + .withLabel("Wipe markers").withWidgetName(com.kingsrook.qbits.secureddata.retention.RetentionWidgets.MARKERS)); + QTableMetaData marker = base(config, WIPE_MARKER, "Wipe marker") + .withField(field("table_name", QFieldType.STRING).withMaxLength(255).withIsRequired(true)) + .withField(field("record_id", QFieldType.STRING).withMaxLength(255).withIsRequired(true)) + .withField(field("field_name", QFieldType.STRING).withMaxLength(255).withIsRequired(true)) + .withField(field("rule_id", QFieldType.INTEGER).withIsRequired(true)) + .withField(field("action", QFieldType.STRING).withMaxLength(40).withIsRequired(true)) + .withField(field("wiped_at", QFieldType.DATE_TIME).withIsRequired(true)) + .withField(field("run_id", QFieldType.INTEGER).withIsRequired(true)) + .withUniqueKey(new UniqueKey().withFieldNames(List.of("table_name", "record_id", "field_name"))); + marker.withSection(SectionFactory.defaultT1("id", "table_name", "record_id", "field_name", "rule_id", "action", "wiped_at", "run_id")); + QTableMetaData hold = base(config, LEGAL_HOLD, "Legal hold") + .withField(field("table_name", QFieldType.STRING).withMaxLength(255).withIsRequired(true).withPossibleValueSourceName("tables")) + .withField(field("record_id", QFieldType.STRING).withMaxLength(255)) + .withField(field("reason", QFieldType.TEXT).withIsRequired(true)) + .withField(field("placed_by", QFieldType.STRING).withMaxLength(255)) + .withField(field("placed_at", QFieldType.DATE_TIME)) + .withField(field("released_by", QFieldType.STRING).withMaxLength(255)) + .withField(field("released_at", QFieldType.DATE_TIME)) + .withField(field("status", QFieldType.STRING).withMaxLength(20).withDefaultValue("ACTIVE")); + hold.withSection(SectionFactory.defaultT1("id", "table_name", "record_id", "reason", "status")); + hold.withSection(SectionFactory.defaultT3("placed_by", "placed_at", "released_by", "released_at")); + QTableMetaData run = base(config, WIPE_RUN, "Wipe run") + .withField(field("rule_id", QFieldType.INTEGER).withIsRequired(true)) + .withField(field("started_at", QFieldType.DATE_TIME).withIsRequired(true)) + .withField(field("finished_at", QFieldType.DATE_TIME)) + .withField(field("status", QFieldType.STRING).withMaxLength(30)) + .withField(field("eligible_count", QFieldType.INTEGER)) + .withField(field("held_count", QFieldType.INTEGER)) + .withField(field("wiped_count", QFieldType.INTEGER)) + .withField(field("error_count", QFieldType.INTEGER)) + .withField(field("record_ids", QFieldType.STRING).withMaxLength(4000)) + .withField(field("message", QFieldType.STRING).withMaxLength(500)); + run.withSection(SectionFactory.defaultT1("id", "rule_id", "status", "eligible_count", "held_count", "wiped_count", "error_count")); + run.withSection(SectionFactory.defaultT3("started_at", "finished_at", "record_ids", "message")); + QTableMetaData key = base(config, AUDIT_KEY, "Audit key") + .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.NONE)) + .withoutCapabilities(Capability.TABLE_EXPORT) + .withField(field("table_name", QFieldType.STRING).withMaxLength(255).withIsRequired(true)) + .withField(field("record_id", QFieldType.STRING).withMaxLength(255).withIsRequired(true)) + .withField(field("field_name", QFieldType.STRING).withMaxLength(255).withIsRequired(true)) + .withField(field("wrapped_key", QFieldType.TEXT).withIsRequired(true).withIsSensitive(true).withIsHidden(true) + .withFieldSecurityLock(new FieldSecurityLock().withSecurityKeyType(KEY_LOCK).withDefaultBehavior(FieldSecurityLock.Behavior.DENY).withOverrideValues(List.of(KEY_LOCK_NEVER_GRANTED)))) + .withField(field("created_at", QFieldType.DATE_TIME).withIsRequired(true)) + .withUniqueKey(new UniqueKey().withFieldNames(List.of("table_name", "record_id", "field_name"))); + key.withSection(SectionFactory.defaultT1("id", "table_name", "record_id", "field_name", "created_at")); + return (List.of(rule, marker, hold, run, key)); + } +} diff --git a/src/main/java/com/kingsrook/qbits/secureddata/metadata/SecuredDataEnums.java b/src/main/java/com/kingsrook/qbits/secureddata/metadata/SecuredDataEnums.java index f776f30..234a1b8 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/metadata/SecuredDataEnums.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/metadata/SecuredDataEnums.java @@ -23,9 +23,9 @@ package com.kingsrook.qbits.secureddata.metadata; import java.util.ArrayList; import java.util.List; -import com.kingsrook.qbits.secureddata.standin.DataClassification; import com.kingsrook.qbits.secureddata.standin.FieldAccessTier; import com.kingsrook.qbits.secureddata.standin.FieldMaskStyle; +import com.kingsrook.qqq.backend.core.actions.ai.DataClassification; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; diff --git a/src/main/java/com/kingsrook/qbits/secureddata/metadata/SecuredDataTableCustomizers.java b/src/main/java/com/kingsrook/qbits/secureddata/metadata/SecuredDataTableCustomizers.java index 9fd19ce..eb1b439 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/metadata/SecuredDataTableCustomizers.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/metadata/SecuredDataTableCustomizers.java @@ -68,6 +68,20 @@ public class SecuredDataTableCustomizers implements TableCustomizerInterface record.getValues().forEach(effective::setValue); validate(table, record, effective); Instant now = SecuredDataRuntime.current().config().getClock().instant(); + if(SecuredDataTables.LEGAL_HOLD.equals(table)) + { + if(old == null) + { + record.setValue("placed_at", now); + record.setValue("placed_by", SecuredDataResolver.userId()); + } + if("RELEASED".equals(effective.getValueString("status"))) + { + record.setValue("released_at", now); + record.setValue("released_by", SecuredDataResolver.userId()); + } + continue; + } if(old == null) { record.setValue("create_date", now); @@ -86,7 +100,7 @@ public class SecuredDataTableCustomizers implements TableCustomizerInterface *******************************************************************************/ private static void assertWrite(String table, boolean mutation) throws QPermissionDeniedException { - if((mutation && SecuredDataTables.ACCESS_LOG.equals(table)) + if((mutation && List.of(SecuredDataTables.ACCESS_LOG, SecuredDataTables.WIPE_RUN).contains(table)) || (SecuredDataTables.SYSTEM_WRITTEN.contains(table) && !SystemWrites.active() && !(QContext.getQSession() instanceof QSystemUserSession))) { throw (new QPermissionDeniedException("Secured-data activity is system-written; access logs are insert-only.")); @@ -106,7 +120,25 @@ public class SecuredDataTableCustomizers implements TableCustomizerInterface { record.addError(new BadInputStatusMessage("Select an existing table.")); } - if(SecuredDataTables.CLASSIFICATION.equals(table)) + if(SecuredDataTables.RETENTION_RULE.equals(table)) + { + try + { + com.kingsrook.qbits.secureddata.retention.RetentionRules.validate(effective); + } + catch(IllegalArgumentException e) + { + record.addError(new BadInputStatusMessage(e.getMessage())); + } + } + else if(SecuredDataTables.LEGAL_HOLD.equals(table)) + { + if(!List.of("ACTIVE", "RELEASED").contains(effective.getValueString("status"))) + { + record.addError(new BadInputStatusMessage("Select ACTIVE or RELEASED.")); + } + } + else if(SecuredDataTables.CLASSIFICATION.equals(table)) { String field = effective.getValueString("field_name"); if(field != null && field.isBlank()) diff --git a/src/main/java/com/kingsrook/qbits/secureddata/metadata/SecuredDataTables.java b/src/main/java/com/kingsrook/qbits/secureddata/metadata/SecuredDataTables.java index 698ddd8..86a9965 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/metadata/SecuredDataTables.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/metadata/SecuredDataTables.java @@ -25,7 +25,7 @@ import java.util.ArrayList; import java.util.List; import com.kingsrook.qbits.secureddata.SecuredDataQBitConfig; import com.kingsrook.qbits.secureddata.permissions.SecuredDataPermissions; -import com.kingsrook.qbits.secureddata.support.CoreMetadataBridge; +import com.kingsrook.qbits.secureddata.support.SecuredDataMetadata; import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.ChildRecordListRenderer; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel; @@ -58,9 +58,14 @@ public final class SecuredDataTables public static final String REVEAL_GRANT = "secured_data_reveal_grant"; public static final String BREAK_GLASS = "secured_data_break_glass"; public static final String ACCESS_LOG = "secured_data_access_log"; - public static final List ALL = List.of(CLASSIFICATION, POLICY, REVEAL_GRANT, BREAK_GLASS, ACCESS_LOG); - public static final List MAIN = List.of(CLASSIFICATION, POLICY, ACCESS_LOG, BREAK_GLASS); - public static final List SYSTEM_WRITTEN = List.of(REVEAL_GRANT, BREAK_GLASS, ACCESS_LOG); + public static final String RETENTION_RULE = "secured_data_retention_rule"; + public static final String WIPE_MARKER = "secured_data_wipe_marker"; + public static final String LEGAL_HOLD = "secured_data_legal_hold"; + public static final String WIPE_RUN = "secured_data_wipe_run"; + public static final String AUDIT_KEY = "secured_data_audit_key"; + public static final List ALL = List.of(CLASSIFICATION, POLICY, REVEAL_GRANT, BREAK_GLASS, ACCESS_LOG, RETENTION_RULE, WIPE_MARKER, LEGAL_HOLD, WIPE_RUN, AUDIT_KEY); + public static final List MAIN = List.of(CLASSIFICATION, POLICY, ACCESS_LOG, BREAK_GLASS, RETENTION_RULE); + public static final List SYSTEM_WRITTEN = List.of(REVEAL_GRANT, BREAK_GLASS, ACCESS_LOG, WIPE_MARKER, WIPE_RUN, AUDIT_KEY); public static final String REVEALS_JOIN = "secured_data_access_log_reveals"; public static final String REVEALS_WIDGET = "securedDataReveals"; @@ -90,7 +95,9 @@ public final class SecuredDataTables *******************************************************************************/ public static List tables(SecuredDataQBitConfig config) { - return (List.of(classification(config), policy(config), revealGrant(config), breakGlass(config), accessLog(config))); + List tables = new ArrayList<>(List.of(classification(config), policy(config), revealGrant(config), breakGlass(config), accessLog(config))); + tables.addAll(RetentionTables.tables(config)); + return (tables); } @@ -98,13 +105,13 @@ public final class SecuredDataTables /******************************************************************************* ** Common identity, audit and administration rules. *******************************************************************************/ - private static QTableMetaData base(SecuredDataQBitConfig config, String name, String label) + static QTableMetaData base(SecuredDataQBitConfig config, String name, String label) { QTableMetaData table = new QTableMetaData().withName(name).withLabel(label).withBackendName(config.getBackendName()) .withPrimaryKeyField("id").withField(new QFieldMetaData("id", QFieldType.INTEGER).withBackendName("id").withIsEditable(false)) .withPermissionRules(SecuredDataPermissions.rules()).withIcon(new QIcon("shield")) .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD)); - CoreMetadataBridge.classifyTable(table); + SecuredDataMetadata.classifyTable(table); if(ACCESS_LOG.equals(name)) { table.withoutCapabilities(Capability.TABLE_UPDATE, Capability.TABLE_DELETE); @@ -117,7 +124,7 @@ public final class SecuredDataTables /******************************************************************************* ** Every database column has an explicit snake_case name. *******************************************************************************/ - private static QFieldMetaData field(String name, QFieldType type) + static QFieldMetaData field(String name, QFieldType type) { QFieldMetaData field = new QFieldMetaData(name, type).withBackendName(name).withLabel(name.replace('_', ' ')); // TODO(contingent): dependsOn("table_name") once the core RC has it @@ -253,9 +260,10 @@ public final class SecuredDataTables .withField(field("anomaly_flag", QFieldType.BOOLEAN).withDefaultValue(false)) .withField(field("break_glass_id", QFieldType.INTEGER)) .withField(field("created_at", QFieldType.DATE_TIME).withIsRequired(true)) - .withField(field("search_value_fingerprints", QFieldType.TEXT)); + .withField(field("search_value_fingerprints", QFieldType.TEXT)) + .withField(field("rule_id", QFieldType.INTEGER)); table.withSection(SectionFactory.defaultT1("id", "event_type", "user_id", "user_name")); - table.withSection(SectionFactory.defaultT2("table_name", "record_id", "field_name", "record_count", "action_name", "reason", "reason_text", "client_channel", "outcome", "anomaly_flag", "break_glass_id", "search_value_fingerprints")); + table.withSection(SectionFactory.defaultT2("table_name", "record_id", "field_name", "record_count", "action_name", "reason", "reason_text", "client_channel", "outcome", "anomaly_flag", "break_glass_id", "rule_id", "search_value_fingerprints")); table.withSection(SectionFactory.defaultT3("created_at")); table.withSection(SectionFactory.customT2("reveals", new QIcon("visibility")).withLabel("Reveals").withWidgetName(REVEALS_WIDGET)); return (table); diff --git a/src/main/java/com/kingsrook/qbits/secureddata/permissions/SecuredDataPermissionChecker.java b/src/main/java/com/kingsrook/qbits/secureddata/permissions/SecuredDataPermissionChecker.java index a78cbc2..af4cded 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/permissions/SecuredDataPermissionChecker.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/permissions/SecuredDataPermissionChecker.java @@ -22,7 +22,7 @@ package com.kingsrook.qbits.secureddata.permissions; import com.kingsrook.qbits.secureddata.metadata.SecuredDataTables; -import com.kingsrook.qbits.secureddata.support.CoreMetadataBridge; +import com.kingsrook.qbits.secureddata.support.SecuredDataMetadata; import com.kingsrook.qqq.backend.core.actions.permissions.CustomPermissionChecker; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; @@ -45,7 +45,7 @@ public class SecuredDataPermissionChecker implements CustomPermissionChecker public void checkPermissionsThrowing(AbstractActionInput input, MetaDataWithPermissionRules metadata) throws QPermissionDeniedException { boolean write = input instanceof InsertInput || input instanceof UpdateInput || input instanceof DeleteInput; - if(QContext.getQSession() == null || "assistant".equals(CoreMetadataBridge.channel(QContext.getQSession())) + if(QContext.getQSession() == null || "assistant".equals(SecuredDataMetadata.channel(QContext.getQSession())) || !QContext.getQSession().hasPermission(SecuredDataPermissions.ADMIN) || (write && SecuredDataTables.SYSTEM_WRITTEN.contains(metadata.getName()))) { diff --git a/src/main/java/com/kingsrook/qbits/secureddata/processes/BreakGlassProcesses.java b/src/main/java/com/kingsrook/qbits/secureddata/processes/BreakGlassProcesses.java index dfb8047..a810409 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/processes/BreakGlassProcesses.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/processes/BreakGlassProcesses.java @@ -28,8 +28,8 @@ import java.util.Map; import com.kingsrook.qbits.secureddata.SecuredDataQBitConfig; import com.kingsrook.qbits.secureddata.breakglass.BreakGlassService; import com.kingsrook.qbits.secureddata.metadata.SecuredDataTables; -import com.kingsrook.qbits.secureddata.standin.DataClassification; import com.kingsrook.qbits.secureddata.support.SystemWrites; +import com.kingsrook.qqq.backend.core.actions.ai.DataClassification; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; diff --git a/src/main/java/com/kingsrook/qbits/secureddata/retention/ArchiveSinkInterface.java b/src/main/java/com/kingsrook/qbits/secureddata/retention/ArchiveSinkInterface.java new file mode 100644 index 0000000..2a13177 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/secureddata/retention/ArchiveSinkInterface.java @@ -0,0 +1,36 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.secureddata.retention; + +import com.kingsrook.qqq.backend.core.exceptions.QException; + +/******************************************************************************* + ** Host adapter to qbit-object-storage. Receives only AES-GCM ciphertext. + ** The host should use a locked WORM bucket when available. Return after durable write. + *******************************************************************************/ +@FunctionalInterface +public interface ArchiveSinkInterface +{ + /******************************************************************************* + ** Store an encrypted JSON record at an opaque, unique object key. + *******************************************************************************/ + void put(String storageName, String objectKey, byte[] ciphertext) throws QException; +} diff --git a/src/main/java/com/kingsrook/qbits/secureddata/retention/RetentionRules.java b/src/main/java/com/kingsrook/qbits/secureddata/retention/RetentionRules.java new file mode 100644 index 0000000..224289e --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/secureddata/retention/RetentionRules.java @@ -0,0 +1,150 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.secureddata.retention; + +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import com.kingsrook.qbits.secureddata.SecuredDataClassifications; +import com.kingsrook.qbits.secureddata.SecuredDataRuntime; +import com.kingsrook.qbits.secureddata.metadata.SecuredDataTables; +import com.kingsrook.qbits.secureddata.support.SystemWrites; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + +/******************************************************************************* + ** Shared validation and effective audit treatment; no record values are queried. + *******************************************************************************/ +public final class RetentionRules +{ + public static final Set ACTIONS = Set.of("NULL_WITH_MARKER", "OVERWRITE_TEXT", "KEYED_HASH", "DELETE_RECORD", "ARCHIVE_THEN_WIPE"); + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private RetentionRules() + { + } + + /******************************************************************************* + ** Validated comma selection; preserve order and reject duplicates. + *******************************************************************************/ + public static List fields(QRecord rule) + { + String names = rule.getValueString("field_names"); + return (names == null ? List.of() : Arrays.stream(names.split(",", -1)).map(String::trim).toList()); + } + + /******************************************************************************* + ** Fresh rules in the current transaction; edits take effect without a stale cache. + *******************************************************************************/ + public static List active(String table, String field) + { + return (SystemWrites.query(SecuredDataTables.RETENTION_RULE, new QQueryFilter() + .withCriteria(new QFilterCriteria("is_active", QCriteriaOperator.EQUALS, true)) + .withCriteria(new QFilterCriteria("table_name", QCriteriaOperator.EQUALS, table))) + .stream().filter(rule -> fields(rule).contains(field)).toList()); + } + + /******************************************************************************* + ** A changed-only policy takes precedence over derivatives. + *******************************************************************************/ + public static boolean changedOnly(String table, String field) + { + return (active(table, field).stream().anyMatch(rule -> "CHANGED_ONLY".equals(rule.getValueString("audit_treatment")))); + } + + /******************************************************************************* + ** Validate both edits and persisted policies before any execution. + *******************************************************************************/ + public static void validate(QRecord rule) + { + QTableMetaData table = QContext.getQInstance().getTable(rule.getValueString("table_name")); + if(table == null || SecuredDataTables.ALL.contains(table.getName())) + { + throw (new IllegalArgumentException("Select a host table for retention.")); + } + List fields = fields(rule); + if(fields.isEmpty() || fields.size() != Set.copyOf(fields).size() || fields.stream().anyMatch(f -> table.getField(f) == null || f.equals(table.getPrimaryKeyField()))) + { + throw (new IllegalArgumentException("Select unique, non-key fields of the selected table.")); + } + var age = table.getField(rule.getValueString("age_field_name")); + if(age == null || !Set.of(QFieldType.DATE, QFieldType.DATE_TIME).contains(age.getType())) + { + throw (new IllegalArgumentException("Age must use a date or date-time field of the selected table.")); + } + if(rule.getValueInteger("age_days") == null || rule.getValueInteger("age_days") < 1 || rule.getValueInteger("batch_size") == null || rule.getValueInteger("batch_size") < 1) + { + throw (new IllegalArgumentException("Age days and batch size must be positive.")); + } + String action = rule.getValueString("action"); + if(!ACTIONS.contains(action) || !Set.of("SHREDDABLE", "CHANGED_ONLY").contains(rule.getValueString("audit_treatment"))) + { + throw (new IllegalArgumentException("Select a supported action and audit treatment.")); + } + for(String name : fields) + { + var field = table.getField(name); + if(Set.of("OVERWRITE_TEXT", "KEYED_HASH").contains(action) && !Set.of(QFieldType.STRING, QFieldType.TEXT).contains(field.getType())) + { + throw (new IllegalArgumentException("Overwrite and keyed hash require STRING or TEXT fields.")); + } + if("KEYED_HASH".equals(action) && SecuredDataClassifications.lowEntropy(table, name)) + { + throw (new IllegalArgumentException("Keyed hash is forbidden for low-entropy fields.")); + } + if("KEYED_HASH".equals(action) && field.getMaxLength() != null && field.getMaxLength() < 200) + { + throw (new IllegalArgumentException("Keyed hash requires capacity for a 200-character token.")); + } + if(Set.of("NULL_WITH_MARKER", "ARCHIVE_THEN_WIPE").contains(action) && field.getIsRequired()) + { + throw (new IllegalArgumentException("Null wipe requires nullable fields.")); + } + String overwrite = rule.getValueString("overwrite_text"); + if("OVERWRITE_TEXT".equals(action) && (overwrite == null || field.getMaxLength() != null && overwrite.length() > field.getMaxLength())) + { + throw (new IllegalArgumentException("Overwrite text must fit every target field.")); + } + } + if("ARCHIVE_THEN_WIPE".equals(action) && (rule.getValueString("archive_storage_name") == null || rule.getValueString("archive_storage_name").isBlank())) + { + throw (new IllegalArgumentException("Archive then wipe requires an archive storage name.")); + } + } + + /******************************************************************************* + ** Known backup window; unknown is explicit rather than assumed safe. + *******************************************************************************/ + public static String backupWarning(QRecord rule) + { + Integer days = SecuredDataRuntime.current().config().getBackupRetentionDays(); + Integer age = rule.getValueInteger("age_days"); + return (days != null && age != null && days > age ? "Backups keep data for " + days + " days, longer than this rule's " + age + + " days. Wiped values remain recoverable from backups until they expire." : null); + } +} diff --git a/src/main/java/com/kingsrook/qbits/secureddata/retention/RetentionWidgets.java b/src/main/java/com/kingsrook/qbits/secureddata/retention/RetentionWidgets.java new file mode 100644 index 0000000..44fe1b9 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/secureddata/retention/RetentionWidgets.java @@ -0,0 +1,163 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.secureddata.retention; + +import java.util.List; +import java.util.Set; +import com.kingsrook.qbits.secureddata.SecuredDataRuntime; +import com.kingsrook.qbits.secureddata.metadata.SecuredDataTables; +import com.kingsrook.qbits.secureddata.permissions.SecuredDataPermissions; +import com.kingsrook.qbits.secureddata.support.SystemWrites; +import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer; +import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.ChildRecordListRenderer; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + +/******************************************************************************* + ** Value-free policy panels and host-opt-in record marker lists. + *******************************************************************************/ +public final class RetentionWidgets +{ + public static final String BACKUPS = "securedDataBackups"; + public static final String MARKERS = "securedDataRuleMarkers"; + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private RetentionWidgets() + { + } + + /******************************************************************************* + ** Markers belong to retention rules and are never editable in child lists. + *******************************************************************************/ + public static QJoinMetaData markersJoin() + { + return (new QJoinMetaData().withName("secured_data_rule_markers").withType(JoinType.ONE_TO_MANY) + .withLeftTable(SecuredDataTables.RETENTION_RULE).withRightTable(SecuredDataTables.WIPE_MARKER).withJoinOn(new JoinOn("id", "rule_id"))); + } + + /******************************************************************************* + ** Standard rule child list. + *******************************************************************************/ + public static QWidgetMetaData markers() + { + return (ChildRecordListRenderer.widgetMetaDataBuilder(markersJoin()).withName(MARKERS).withLabel("Wipe markers") + .withCanAddChildRecord(false).withMaxRows(50).getWidgetMetaData().withPermissionRules(SecuredDataPermissions.rules())); + } + + /******************************************************************************* + ** Known physical backup retention and its residual recovery risk. + *******************************************************************************/ + public static QWidgetMetaData backups() + { + return (new QWidgetMetaData().withName(BACKUPS).withLabel("Backups").withType(WidgetType.HTML.getType()).withGridColumns(12) + .withCodeReference(new QCodeReference(Backups.class)).withPermissionRules(SecuredDataPermissions.rules())); + } + + /******************************************************************************* + ** Host adds this metadata and a section referencing its name on its own table. + ** The immutable widget configuration selects the table, not request parameters. + *******************************************************************************/ + public static QWidgetMetaData wipedFields(QTableMetaData table) + { + return (new QWidgetMetaData().withName("securedDataWipedFields_" + table.getName()).withLabel("Wiped fields").withType(WidgetType.HTML.getType()) + .withGridColumns(12).withDefaultValue("hostTable", table.getName()).withCodeReference(new QCodeReference(WipedFields.class))); + } + + /******************************************************************************* + ** Render the rule through normal user permissions. + *******************************************************************************/ + public static class Backups extends AbstractWidgetRenderer + { + @Override + public RenderWidgetOutput render(RenderWidgetInput input) throws QException + { + List rules = new QueryAction().execute(new QueryInput(SecuredDataTables.RETENTION_RULE) + .withFilter(new QQueryFilter().withCriteria(new QFilterCriteria("id", QCriteriaOperator.EQUALS, input.getQueryParams().get("id"))))).getRecords(); + StringBuilder html = new StringBuilder("
"); + Integer days = SecuredDataRuntime.current().config().getBackupRetentionDays(); + html.append("

Physical backup retention: ").append(days == null ? "unknown" : days + " days").append(".

"); + for(QRecord rule : rules) + { + String warning = RetentionRules.backupWarning(rule); + if(warning != null) + { + html.append("

").append(escape(warning)).append("

"); + } + } + html.append("

CNPG physical backups and WAL cannot exclude audit keys. A backup and the KEK can recover wiped data until that backup expires.

"); + return (new RenderWidgetOutput(new RawHTML(null, html.toString()))); + } + } + + /******************************************************************************* + ** Authorize the host record with an ID-only query before looking up markers. + *******************************************************************************/ + public static class WipedFields extends AbstractWidgetRenderer + { + @Override + public RenderWidgetOutput render(RenderWidgetInput input) throws QException + { + String tableName = (String) input.getWidgetMetaData().getDefaultValues().get("hostTable"); + var table = com.kingsrook.qqq.backend.core.context.QContext.getQInstance().getTable(tableName); + String id = input.getQueryParams().get("id"); + List visible = new QueryAction().execute(new QueryInput(tableName).withFieldNamesToInclude(Set.of(table.getPrimaryKeyField())) + .withFilter(new QQueryFilter().withCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.EQUALS, id)))).getRecords(); + StringBuilder html = new StringBuilder("
    "); + if(!visible.isEmpty()) + { + for(QRecord marker : SystemWrites.query(SecuredDataTables.WIPE_MARKER, new QQueryFilter() + .withCriteria(new QFilterCriteria("table_name", QCriteriaOperator.EQUALS, tableName)) + .withCriteria(new QFilterCriteria("record_id", QCriteriaOperator.EQUALS, id)))) + { + List rules = SystemWrites.query(SecuredDataTables.RETENTION_RULE, new QQueryFilter().withCriteria(new QFilterCriteria("id", QCriteriaOperator.EQUALS, marker.getValue("rule_id")))); + String age = rules.isEmpty() ? "unknown" : rules.getFirst().getValueString("age_days"); + html.append("
  • ").append(escape(marker.getValueString("field_name"))).append(": Wiped per policy (").append(escape(age)).append(" days)
    • "); + } + } + return (new RenderWidgetOutput(new RawHTML(null, html.append("
").toString()))); + } + } + + /******************************************************************************* + ** Escape user-managed names and all text before HTML rendering. + *******************************************************************************/ + private static String escape(String text) + { + return (text == null ? "" : text.replace("&", "&").replace("<", "<").replace(">", ">").replace("\"", """).replace("'", "'")); + } +} diff --git a/src/main/java/com/kingsrook/qbits/secureddata/retention/WipeProcesses.java b/src/main/java/com/kingsrook/qbits/secureddata/retention/WipeProcesses.java new file mode 100644 index 0000000..1ad5697 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/secureddata/retention/WipeProcesses.java @@ -0,0 +1,258 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.secureddata.retention; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.secureddata.SecuredDataQBitConfig; +import com.kingsrook.qbits.secureddata.metadata.SecuredDataTables; +import com.kingsrook.qbits.secureddata.support.SystemWrites; +import com.kingsrook.qqq.backend.core.actions.permissions.CustomPermissionChecker; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; + +/******************************************************************************* + ** Standard Review/Result action and a daily system scheduler entry point. + *******************************************************************************/ +public final class WipeProcesses +{ + public static final String WIPE = "secured_data_wipe"; + public static final String DAILY = "secured_data_wipe_daily"; + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private WipeProcesses() + { + } + + /******************************************************************************* + ** Processes are table actions or hidden schedules, never navigation entries. + *******************************************************************************/ + public static List processes(SecuredDataQBitConfig config) + { + Map values = new LinkedHashMap<>(); + values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, SecuredDataTables.RETENTION_RULE); + values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, SecuredDataTables.WIPE_RUN); + values.put(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION, true); + values.put(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true); + values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, "Eligible records will be wiped; active legal holds are preserved."); + QPermissionRules permissions = new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION) + .withPermissionBaseName("secured_data").withCustomPermissionChecker(new QCodeReference(Permission.class)); + QProcessMetaData wipe = StreamedETLWithFrontendProcess.defineProcessMetaData(Extract.class, Review.class, Load.class, values) + .withName(WIPE).withLabel("Run wipe now").withTableName(SecuredDataTables.RETENTION_RULE).withMinInputRecords(1).withPermissionRules(permissions); + QProcessMetaData daily = new QProcessMetaData().withName(DAILY).withLabel("Daily retention wipe").withIsHidden(true).withPermissionRules(permissions) + .withStep(new QBackendStepMetaData().withName("wipe").withCode(new QCodeReference(Daily.class))); + if(config.getSchedulerName() != null) + { + daily.withSchedule(new QScheduleMetaData().withSchedulerName(config.getSchedulerName()).withRepeatSeconds(config.getWipeIntervalSeconds()).withInitialDelaySeconds(30)); + } + return (List.of(wipe, daily)); + } + + /******************************************************************************* + ** Explicit retention permission is independent of classified-value access. + *******************************************************************************/ + public static class Permission implements CustomPermissionChecker + { + @Override + public void checkPermissionsThrowing(AbstractActionInput input, MetaDataWithPermissionRules metadata) throws QPermissionDeniedException + { + WipeService.checkPermission(); + } + } + + /******************************************************************************* + ** Select policies using the standard table-selection filter, then stream IDs. + *******************************************************************************/ + public static class Extract extends ExtractViaQueryStep + { + @Override + public Integer doCount(RunBackendStepInput input) + { + return (null); + } + + @Override + public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + WipeService.checkPermission(); + for(QRecord rule : SystemWrites.query(SecuredDataTables.RETENTION_RULE, getQueryFilter(input))) + { + new WipeService().extract(rule, packets -> + { + for(QRecord packet : packets) + { + getRecordPipe().addRecord(packet); + } + }); + } + } + } + + /******************************************************************************* + ** Counts per policy and hold status; no record or field values in summaries. + *******************************************************************************/ + public static class Review extends AbstractTransformStep implements ProcessSummaryProviderInterface + { + private final Map lines = new LinkedHashMap<>(); + + @Override + public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + WipeService.checkPermission(); + for(QRecord packet : input.getRecords()) + { + String id = packet.getValueString("rule_id"); + boolean held = Boolean.TRUE.equals(packet.getValueBoolean("held")); + String label = held ? "held" : "to wipe"; + lines.computeIfAbsent(id + label, key -> line(id, held, false)).incrementCountAndAddPrimaryKey(null); + output.getRecords().add(packet); + } + } + + @Override + public ArrayList getProcessSummary(RunBackendStepOutput output, boolean result) + { + return (new ArrayList<>(lines.values())); + } + } + + /******************************************************************************* + ** Group only the current ETL page, splitting again by each policy's batch size. + *******************************************************************************/ + public static class Load extends AbstractLoadStep implements ProcessSummaryProviderInterface + { + private final Map lines = new LinkedHashMap<>(); + + @Override + public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + WipeService.checkPermission(); + Map> rules = new LinkedHashMap<>(); + for(QRecord packet : input.getRecords()) + { + rules.computeIfAbsent(packet.getValue("rule_id"), ignored -> new ArrayList<>()).add(packet.getValue("id")); + } + for(var entry : rules.entrySet()) + { + int size = WipeService.rule(entry.getKey()).getValueInteger("batch_size"); + for(int offset = 0; offset < entry.getValue().size(); offset += size) + { + QRecord run = new WipeService().batch(entry.getKey(), entry.getValue().subList(offset, Math.min(offset + size, entry.getValue().size()))); + output.getRecords().add(run); + for(boolean held : List.of(false, true)) + { + String id = String.valueOf(entry.getKey()); + ProcessSummaryLine line = lines.computeIfAbsent(id + held, ignored -> line(id, held, true)); + for(int n = 0; n < run.getValueInteger(held ? "held_count" : "wiped_count"); n++) + { + line.incrementCountAndAddPrimaryKey(null); + } + } + } + } + } + + @Override + public ArrayList getProcessSummary(RunBackendStepOutput output, boolean result) + { + return (new ArrayList<>(lines.values())); + } + } + + /******************************************************************************* + ** Scheduler identity must already be system; never elevate a user invocation. + *******************************************************************************/ + public static class Daily implements BackendStep + { + @Override + public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + if(!(QContext.getQSession() instanceof QSystemUserSession)) + { + throw (new QPermissionDeniedException("Daily retention requires the system session.")); + } + for(QRecord rule : SystemWrites.query(SecuredDataTables.RETENTION_RULE, new QQueryFilter().withCriteria(new QFilterCriteria("is_active", QCriteriaOperator.EQUALS, true)))) + { + List errors = new ArrayList<>(); + new WipeService().extract(rule, packets -> + { + try + { + output.getRecords().add(new WipeService().batch(rule.getValue("id"), packets.stream().map(row -> row.getValue("id")).toList())); + } + catch(QException e) + { + errors.add(e); + } + }); + if(!errors.isEmpty()) + { + throw (new QException("Daily retention had rolled-back batches.")); + } + } + } + } + + /******************************************************************************* + ** Standard future/past summary text. + *******************************************************************************/ + private static ProcessSummaryLine line(String id, boolean held, boolean result) + { + ProcessSummaryLine line = new ProcessSummaryLine(held ? Status.INFO : Status.OK, 0, null); + String future = "eligible records for rule " + id + (held ? " held by legal hold" : " to wipe"); + String past = "records for rule " + id + (held ? " held by legal hold" : " wiped"); + line.setSingularFutureMessage(future); + line.setPluralFutureMessage(future); + line.setSingularPastMessage(past); + line.setPluralPastMessage(past); + return (line); + } +} diff --git a/src/main/java/com/kingsrook/qbits/secureddata/retention/WipeService.java b/src/main/java/com/kingsrook/qbits/secureddata/retention/WipeService.java new file mode 100644 index 0000000..5ffabbc --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/secureddata/retention/WipeService.java @@ -0,0 +1,426 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.secureddata.retention; + +import java.io.Serializable; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; +import com.kingsrook.qbits.secureddata.CoreAuditBridge; +import com.kingsrook.qbits.secureddata.SecuredDataKeySource; +import com.kingsrook.qbits.secureddata.SecuredDataRuntime; +import com.kingsrook.qbits.secureddata.metadata.SecuredDataTables; +import com.kingsrook.qbits.secureddata.support.ActivityLog; +import com.kingsrook.qbits.secureddata.support.SystemWrites; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; + +/******************************************************************************* + ** ID-only extraction and atomic, system-owned batches. No host values are placed + ** in process records, run messages, access logs or exceptions. + *******************************************************************************/ +public final class WipeService +{ + /******************************************************************************* + ** Checked-exception-aware streaming boundary. + *******************************************************************************/ + @FunctionalInterface + public interface PageConsumer + { + /******************************************************************************* + ** Consume one bounded page. + *******************************************************************************/ + void accept(List records) throws QException; + } + + public static final String PERMISSION = "secured_data.retention"; + + /******************************************************************************* + ** Enforce at every callable boundary, including process resumption. + *******************************************************************************/ + public static void checkPermission() throws QPermissionDeniedException + { + if(!(QContext.getQSession() instanceof QSystemUserSession) + && (QContext.getQSession() == null || !QContext.getQSession().hasPermission(PERMISSION) || "assistant".equals(QContext.getQSession().getClientChannelName()))) + { + throw (new QPermissionDeniedException("Permission denied: secured_data.retention is required.")); + } + } + + /******************************************************************************* + ** Reload the rule instead of trusting process-state configuration. + *******************************************************************************/ + public static QRecord rule(Serializable id) throws QException + { + List rows = SystemWrites.query(SecuredDataTables.RETENTION_RULE, new QQueryFilter().withCriteria(new QFilterCriteria("id", QCriteriaOperator.EQUALS, id))); + if(rows.isEmpty()) + { + throw (new QException("Retention rule no longer exists.")); + } + QRecord rule = rows.getFirst(); + RetentionRules.validate(rule); + return (rule); + } + + /******************************************************************************* + ** A bounded keyset scan. Per-field predicates return IDs only, even for PHI. + ** Packets carry IDs, field names and a hold indicator, never stored values. + *******************************************************************************/ + public void extract(QRecord rule, PageConsumer consumer) throws QException + { + checkPermission(); + RetentionRules.validate(rule); + if(!Boolean.TRUE.equals(rule.getValueBoolean("is_active"))) + { + return; + } + String tableName = rule.getValueString("table_name"); + QTableMetaData table = QContext.getQInstance().getTable(tableName); + String pkName = table.getPrimaryKeyField(); + Serializable after = null; + while(true) + { + QQueryFilter filter = eligible(rule, null).withOrderBy(new QFilterOrderBy(pkName)).withLimit(rule.getValueInteger("batch_size")); + if(after != null) + { + filter.withCriteria(new QFilterCriteria(pkName, QCriteriaOperator.GREATER_THAN, after)); + } + List ids = query(tableName, filter, Set.of(pkName)); + if(ids.isEmpty()) + { + return; + } + List packets = new ArrayList<>(); + for(QRecord id : ids) + { + Serializable pk = id.getValue(pkName); + packets.add(new QRecord().withValue("id", pk).withValue("rule_id", rule.getValue("id")) + .withValue("held", held(tableName, pk))); + } + consumer.accept(packets); + after = ids.getLast().getValue(pkName); + } + } + + /******************************************************************************* + ** Candidate fields are tested in SQL, not fetched. + *******************************************************************************/ + private static QQueryFilter eligible(QRecord rule, String field) + { + QTableMetaData table = QContext.getQInstance().getTable(rule.getValueString("table_name")); + Instant cutoff = SecuredDataRuntime.current().config().getClock().instant().minus(rule.getValueInteger("age_days"), ChronoUnit.DAYS); + Serializable age = table.getField(rule.getValueString("age_field_name")).getType() == QFieldType.DATE ? cutoff.atZone(ZoneOffset.UTC).toLocalDate() : cutoff; + QQueryFilter filter = new QQueryFilter().withCriteria(new QFilterCriteria(rule.getValueString("age_field_name"), QCriteriaOperator.LESS_THAN, age)); + QQueryFilter nonnull = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR); + for(String name : field == null ? RetentionRules.fields(rule) : List.of(field)) + { + nonnull.withCriteria(new QFilterCriteria(name, QCriteriaOperator.IS_NOT_BLANK)); + } + return (filter.withSubFilter(nonnull)); + } + + /******************************************************************************* + ** Active record and whole-table holds pause all policies. + *******************************************************************************/ + public static boolean held(String table, Serializable pk) + { + return (!SystemWrites.query(SecuredDataTables.LEGAL_HOLD, new QQueryFilter() + .withCriteria(new QFilterCriteria("table_name", QCriteriaOperator.EQUALS, table)) + .withCriteria(new QFilterCriteria("status", QCriteriaOperator.EQUALS, "ACTIVE")) + .withSubFilter(new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withCriteria(new QFilterCriteria("record_id", QCriteriaOperator.IS_BLANK)) + .withCriteria(new QFilterCriteria("record_id", QCriteriaOperator.EQUALS, String.valueOf(pk))))).isEmpty()); + } + + /******************************************************************************* + ** Explicit projection and transaction for every host read. + *******************************************************************************/ + private static List query(String table, QQueryFilter filter, Set fields) throws QException + { + return (SystemWrites.call(() -> new QueryAction().execute(new QueryInput(table).withFilter(filter).withFieldNamesToInclude(fields) + .withShouldFetchHeavyFields(true).withTransaction(QContext.getQBackendTransaction())).getRecords())); + } + + /******************************************************************************* + ** Revalidate eligibility and holds inside the batch, then commit all effects. + ** Cross-backend and nontransactional deployments are refused before any writes. + *******************************************************************************/ + public QRecord batch(Serializable ruleId, List ids) throws QException + { + checkPermission(); + QRecord rule = rule(ruleId); + QTableMetaData table = QContext.getQInstance().getTable(rule.getValueString("table_name")); + if(ids.size() > rule.getValueInteger("batch_size")) + { + throw (new QException("Wipe batch exceeds the policy batch size.")); + } + if(QContext.getQBackendTransaction() != null) + { + throw (new QException("Wipe batches must own their transaction.")); + } + for(String name : List.of(SecuredDataTables.RETENTION_RULE, SecuredDataTables.WIPE_MARKER, SecuredDataTables.LEGAL_HOLD, + SecuredDataTables.WIPE_RUN, SecuredDataTables.AUDIT_KEY, SecuredDataTables.ACCESS_LOG, "audit", "auditDetail")) + { + QTableMetaData metadata = QContext.getQInstance().getTable(name); + if(metadata != null && !Objects.equals(metadata.getBackendName(), table.getBackendName())) + { + throw (new QException("Wipe, audit and secured-data tables must share one transactional backend.")); + } + } + if("ARCHIVE_THEN_WIPE".equals(rule.getValueString("action")) && SecuredDataRuntime.current().config().getArchiveSink() == null) + { + throw (new QException("ARCHIVE_THEN_WIPE requires a host ArchiveSinkInterface backed by object storage.")); + } + try(QBackendTransaction transaction = QBackendTransaction.openFor(new UpdateInput(table.getName()))) + { + if(transaction == null || transaction.getClass() == QBackendTransaction.class) + { + throw (new QException("Wipe requires a backend with rollback-capable write transactions.")); + } + QContext.setTransaction(transaction); + try + { + QRecord result = SystemWrites.call(() -> apply(rule, table, ids, transaction)); + transaction.commit(); + return (result); + } + catch(Exception e) + { + transaction.rollback(); + throw (new QException("Wipe batch failed and was rolled back; no values are included in this message.")); + } + finally + { + SecuredDataRuntime.current().keys().close(); + QContext.clearTransaction(); + } + } + } + + /******************************************************************************* + ** Batch body. Markers, key deletion, audit, access log and run row share tx. + *******************************************************************************/ + private QRecord apply(QRecord initialRule, QTableMetaData table, List ids, QBackendTransaction transaction) throws QException + { + QRecord rule = rule(initialRule.getValue("id")); + Instant started = SecuredDataRuntime.current().config().getClock().instant(); + List wiped = new ArrayList<>(); + int held = 0; + Map> fieldsById = new LinkedHashMap<>(); + if(Boolean.TRUE.equals(rule.getValueBoolean("is_active"))) + { + for(String field : RetentionRules.fields(rule)) + { + QQueryFilter filter = eligible(rule, field).withCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, ids)); + for(QRecord row : query(table.getName(), filter, Set.of(table.getPrimaryKeyField()))) + { + fieldsById.computeIfAbsent(row.getValue(table.getPrimaryKeyField()), ignored -> new ArrayList<>()).add(field); + } + } + } + List markers = new ArrayList<>(); + Map> logged = new LinkedHashMap<>(); + for(var entry : fieldsById.entrySet()) + { + Serializable pk = entry.getKey(); + if(held(table.getName(), pk)) + { + held++; + continue; + } + List fields = entry.getValue(); + String action = rule.getValueString("action"); + if("DELETE_RECORD".equals(action)) + { + fields = new ArrayList<>(table.getFields().keySet()); + fields.remove(table.getPrimaryKeyField()); + } + change(rule, table, pk, fields, transaction); + for(String field : fields) + { + List keys = query(SecuredDataTables.AUDIT_KEY, SecuredDataKeySource.recordField(table.getName(), field, pk), Set.of("id")); + for(QRecord key : keys) + { + var deleted = new DeleteAction().execute(new DeleteInput(SecuredDataTables.AUDIT_KEY).withPrimaryKeys(List.of(key.getValue("id"))).withTransaction(transaction)); + if(deleted.getDeletedRecordCount() != 1) + { + throw (new QException("Audit key deletion failed.")); + } + CoreAuditBridge.detail(table, pk, field, rule.getValueString("id"), true, transaction); + } + markers.add(new QRecord().withValue("table_name", table.getName()).withValue("record_id", String.valueOf(pk)).withValue("field_name", field) + .withValue("rule_id", rule.getValue("id")).withValue("action", action).withValue("wiped_at", started)); + logged.computeIfAbsent(field, ignored -> new ArrayList<>()).add(pk); + } + wiped.add(pk); + } + QRecord run = SystemWrites.insertOne(SecuredDataTables.WIPE_RUN, new QRecord().withValue("rule_id", rule.getValue("id")) + .withValue("started_at", started).withValue("finished_at", SecuredDataRuntime.current().config().getClock().instant()) + .withValue("status", "COMPLETE").withValue("eligible_count", fieldsById.size()).withValue("held_count", held) + .withValue("wiped_count", wiped.size()).withValue("error_count", 0).withValue("record_ids", boundedIds(wiped)) + .withValue("message", "Policy batch completed."), transaction); + for(QRecord marker : markers) + { + marker.setValue("run_id", run.getValue("id")); + List previous = SystemWrites.query(SecuredDataTables.WIPE_MARKER, SecuredDataKeySource.recordField(marker.getValueString("table_name"), marker.getValueString("field_name"), marker.getValue("record_id"))); + if(previous.isEmpty()) + { + SystemWrites.insertOne(SecuredDataTables.WIPE_MARKER, marker, transaction); + } + else + { + marker.setValue("id", previous.getFirst().getValue("id")); + update(SecuredDataTables.WIPE_MARKER, marker, transaction); + } + } + for(var entry : logged.entrySet()) + { + for(int offset = 0; offset < entry.getValue().size(); offset += 50) + { + List chunk = entry.getValue().subList(offset, Math.min(offset + 50, entry.getValue().size())); + SystemWrites.insertOne(SecuredDataTables.ACCESS_LOG, ActivityLog.row("WIPE", table.getName(), "ALLOWED") + .withValue("rule_id", rule.getValue("id")).withValue("field_name", entry.getKey()).withValue("record_id", boundedIds(chunk)) + .withValue("record_count", chunk.size()).withValue("action_name", rule.getValue("action")), transaction); + } + } + update(SecuredDataTables.RETENTION_RULE, new QRecord().withValue("id", rule.getValue("id")).withValue("last_run_at", started).withValue("last_run_count", wiped.size()), transaction); + return (run); + } + + /******************************************************************************* + ** The two value transformations read only the fields they transform; all + ** other operations work entirely by ID. Archive receives encrypted JSON only. + *******************************************************************************/ + private static void change(QRecord rule, QTableMetaData table, Serializable pk, List fields, QBackendTransaction transaction) throws QException + { + String action = rule.getValueString("action"); + if("DELETE_RECORD".equals(action)) + { + if(new DeleteAction().execute(new DeleteInput(table.getName()).withPrimaryKeys(List.of(pk)).withTransaction(transaction)).getDeletedRecordCount() != 1) + { + throw (new QException("Record deletion failed.")); + } + return; + } + QRecord stored = null; + if(Set.of("KEYED_HASH", "ARCHIVE_THEN_WIPE").contains(action)) + { + stored = query(table.getName(), new QQueryFilter().withCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.EQUALS, pk)), new LinkedHashSet<>(fields)).getFirst(); + } + if("ARCHIVE_THEN_WIPE".equals(action)) + { + String objectKey = "secured-data/" + UUID.randomUUID() + ".aesgcm"; + Map values = new LinkedHashMap<>(); + for(String field : fields) + { + values.put(field, stored.getValue(field)); + } + byte[] json = JsonUtils.toJson(values).getBytes(StandardCharsets.UTF_8); + try + { + byte[] encrypted = SecuredDataKeySource.encrypt(json, objectKey); + SecuredDataRuntime.current().config().getArchiveSink().put(rule.getValueString("archive_storage_name"), objectKey, encrypted); + } + finally + { + Arrays.fill(json, (byte) 0); + values.clear(); + } + } + QRecord patch = new QRecord().withValue(table.getPrimaryKeyField(), pk); + for(String field : fields) + { + Serializable value = switch(action) + { + case "OVERWRITE_TEXT" -> rule.getValue("overwrite_text"); + case "KEYED_HASH" -> SecuredDataRuntime.current().keys().instanceToken(table, table.getField(field), pk, stored.getValue(field)); + default -> null; + }; + patch.setValue(field, value); + } + UpdateInput input = new UpdateInput(table.getName()).withRecord(patch).withTransaction(transaction); + CoreAuditBridge.flag(input, rule.getValueString("id")); + check(new UpdateAction().execute(input).getRecords()); + if(!CoreAuditBridge.available()) + { + for(String field : fields) + { + CoreAuditBridge.detail(table, pk, field, rule.getValueString("id"), false, transaction); + } + } + } + + /******************************************************************************* + ** Check record-level backend failures, which need not throw exceptions. + *******************************************************************************/ + private static void check(List records) throws QException + { + if(records.isEmpty() || records.stream().anyMatch(row -> row.getErrors() != null && !row.getErrors().isEmpty())) + { + throw (new QException("Wipe write failed.")); + } + } + + /******************************************************************************* + ** System metadata updates still validate in the transaction. + *******************************************************************************/ + private static void update(String table, QRecord row, QBackendTransaction transaction) throws QException + { + check(new UpdateAction().execute(new UpdateInput(table).withRecord(row).withTransaction(transaction)).getRecords()); + } + + /******************************************************************************* + ** Bounded identifiers, never values. Full record evidence is in audit rows. + *******************************************************************************/ + private static String boundedIds(List ids) + { + String joined = ids.stream().limit(50).map(String::valueOf).collect(Collectors.joining(",")); + return (joined.length() > 4000 ? joined.substring(0, 4000) : joined); + } +} diff --git a/src/main/java/com/kingsrook/qbits/secureddata/reveal/RevealService.java b/src/main/java/com/kingsrook/qbits/secureddata/reveal/RevealService.java index 4b8448b..91491d5 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/reveal/RevealService.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/reveal/RevealService.java @@ -32,7 +32,7 @@ import com.kingsrook.qbits.secureddata.SecuredDataRuntime; import com.kingsrook.qbits.secureddata.metadata.SecuredDataTables; import com.kingsrook.qbits.secureddata.standin.FieldAccessTier; import com.kingsrook.qbits.secureddata.support.ActivityLog; -import com.kingsrook.qbits.secureddata.support.CoreMetadataBridge; +import com.kingsrook.qbits.secureddata.support.SecuredDataMetadata; import com.kingsrook.qbits.secureddata.support.SystemWrites; import com.kingsrook.qqq.backend.core.actions.permissions.CorePermissionEnforcer; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; @@ -77,7 +77,7 @@ public class RevealService private QRecord policy(String tableName, String fieldName) throws QException { requireTarget(tableName, fieldName); - if("assistant".equals(CoreMetadataBridge.channel(QContext.getQSession()))) + if("assistant".equals(SecuredDataMetadata.channel(QContext.getQSession()))) { throw (new QPermissionDeniedException("Reveal is unavailable on the assistant channel.")); } @@ -120,7 +120,7 @@ public class RevealService SecuredDataRuntime runtime = SecuredDataRuntime.current(); synchronized(runtime) { - if("assistant".equals(CoreMetadataBridge.channel(QContext.getQSession()))) + if("assistant".equals(SecuredDataMetadata.channel(QContext.getQSession()))) { denied(tableName, primaryKey, fieldName, "DENIED"); throw (new QPermissionDeniedException("Reveal is unavailable on the assistant channel.")); diff --git a/src/main/java/com/kingsrook/qbits/secureddata/standin/AiFieldMetaData.java b/src/main/java/com/kingsrook/qbits/secureddata/standin/AiFieldMetaData.java deleted file mode 100644 index 980748e..0000000 --- a/src/main/java/com/kingsrook/qbits/secureddata/standin/AiFieldMetaData.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2025. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.kingsrook.qbits.secureddata.standin; - - -import com.kingsrook.qqq.backend.core.model.metadata.fields.QSupplementalFieldMetaData; - - -/******************************************************************************* - ** Temporary classification metadata alongside the released AI exposure metadata. - *******************************************************************************/ -// STAND-IN(core=QRun-IO/qqq#1142, removeBy=RC.6): copy of com.kingsrook.qqq.backend.core.actions.ai.AiFieldMetaData classification properties; delete when the pin moves to RC.6. -public class AiFieldMetaData extends QSupplementalFieldMetaData -{ - public static final String TYPE = "securedDataAi"; - private DataClassification classification; - - - - /******************************************************************************* - ** Metadata key, separate from RC.5's exposure-only AI metadata. - *******************************************************************************/ - @Override - public String getType() - { - return (TYPE); - } - - - - /******************************************************************************* - ** Classification. - *******************************************************************************/ - public DataClassification getClassification() - { - return (classification); - } - - - - /******************************************************************************* - ** Code classification until the released core supports it. - *******************************************************************************/ - public AiFieldMetaData withClassification(DataClassification classification) - { - this.classification = classification; - return (this); - } -} diff --git a/src/main/java/com/kingsrook/qbits/secureddata/standin/AiTableMetaData.java b/src/main/java/com/kingsrook/qbits/secureddata/standin/AiTableMetaData.java deleted file mode 100644 index 970b84d..0000000 --- a/src/main/java/com/kingsrook/qbits/secureddata/standin/AiTableMetaData.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2025. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.kingsrook.qbits.secureddata.standin; - - -import com.kingsrook.qqq.backend.core.model.metadata.tables.QSupplementalTableMetaData; - - -/******************************************************************************* - ** Temporary classification metadata alongside the released AI exposure metadata. - *******************************************************************************/ -// STAND-IN(core=QRun-IO/qqq#1142, removeBy=RC.6): copy of com.kingsrook.qqq.backend.core.actions.ai.AiTableMetaData classification properties; delete when the pin moves to RC.6. -public class AiTableMetaData extends QSupplementalTableMetaData -{ - public static final String TYPE = "securedDataAi"; - private DataClassification classification; - - - - /******************************************************************************* - ** Metadata key, separate from RC.5's exposure-only AI metadata. - *******************************************************************************/ - @Override - public String getType() - { - return (TYPE); - } - - - - /******************************************************************************* - ** Classification. - *******************************************************************************/ - public DataClassification getClassification() - { - return (classification); - } - - - - /******************************************************************************* - ** Code classification until the released core supports it. - *******************************************************************************/ - public AiTableMetaData withClassification(DataClassification classification) - { - this.classification = classification; - return (this); - } -} diff --git a/src/main/java/com/kingsrook/qbits/secureddata/support/UiApiBridge.java b/src/main/java/com/kingsrook/qbits/secureddata/standin/AuditValueProtectorInterface.java similarity index 52% rename from src/main/java/com/kingsrook/qbits/secureddata/support/UiApiBridge.java rename to src/main/java/com/kingsrook/qbits/secureddata/standin/AuditValueProtectorInterface.java index cdeaf74..42f74f9 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/support/UiApiBridge.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/standin/AuditValueProtectorInterface.java @@ -1,6 +1,6 @@ /* * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2025. Kingsrook, LLC + * Copyright (C) 2021-2026. Kingsrook, LLC * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States * contact@kingsrook.com * https://github.com/Kingsrook/ @@ -18,39 +18,33 @@ * limitations under the License. */ -package com.kingsrook.qbits.secureddata.support; +package com.kingsrook.qbits.secureddata.standin; + + +import java.io.Serializable; +import com.kingsrook.qbits.secureddata.standin.DMLType; +import com.kingsrook.qqq.backend.core.actions.ai.AiDataClassifications; +import com.kingsrook.qqq.backend.core.actions.ai.DataClassification; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.QInstance; /******************************************************************************* - ** Optional UI API registration, with no linkage to unreleased middleware on RC.5. + ** Application hook for audit tokens and effective classification. Never return plain values. *******************************************************************************/ -public final class UiApiBridge +// STAND-IN(core=QRun-IO/qqq#1182, removeBy=RC.7) +public interface AuditValueProtectorInterface { /******************************************************************************* - ** Utility class. + ** Protect the values in the current action's transaction before audit insertion. *******************************************************************************/ - private UiApiBridge() - { - } + ProtectedAuditValues protect(QTableMetaData table, QFieldMetaData field, Serializable primaryKey, Serializable oldValue, Serializable newValue, DMLType dmlType); /******************************************************************************* - ** STAND-IN(core=QRun-IO/qqq#1128, removeBy=RC.6): register directly at RC.6. + ** Override to use an application-owned classification source. *******************************************************************************/ - public static void register(QInstance instance) + default DataClassification classificationOf(QTableMetaData table, QFieldMetaData field) { - try - { - Class.forName("com.kingsrook.qqq.middleware.javalin.specs.extensions.MiddlewareSpecExtensionsMetaData"); - Class.forName("com.kingsrook.qbits.secureddata.uiapi.SecuredDataExtension").getMethod("register", QInstance.class).invoke(null, instance); - } - catch(ClassNotFoundException expected) - { - // RC.5 or a build without the uiapi source profile. - } - catch(ReflectiveOperationException e) - { - throw (new IllegalStateException("Unable to register secured-data UI API extension", e)); - } + return (AiDataClassifications.ofField(table, field.getName())); } } diff --git a/src/main/java/com/kingsrook/qbits/secureddata/standin/ClassifiedValueKeySource.java b/src/main/java/com/kingsrook/qbits/secureddata/standin/ClassifiedValueKeySource.java new file mode 100644 index 0000000..db81ced --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/secureddata/standin/ClassifiedValueKeySource.java @@ -0,0 +1,70 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.secureddata.standin; + + +import java.io.Serializable; +import java.util.Optional; + + +/******************************************************************************* + ** Supplies instance keys and optional per-record, per-field shreddable keys. + *******************************************************************************/ +// STAND-IN(core=QRun-IO/qqq#1182, removeBy=RC.7) +public interface ClassifiedValueKeySource +{ + /******************************************************************************* + ** The active instance key, if configured. + *******************************************************************************/ + Optional instanceKey(); + + /******************************************************************************* + ** Verification uses create=false and must never recreate a destroyed key. + *******************************************************************************/ + Optional keyForRecordField(String table, String field, Serializable pk, boolean create); + + /******************************************************************************* + ** Key bytes and the public identifier stored in a token. + *******************************************************************************/ + record KeyAndId(byte[] key, String id) + { + /******************************************************************************* + ** Own the key bytes; identifiers must keep tokens ASCII and under 200 characters. + *******************************************************************************/ + public KeyAndId + { + if(key == null || key.length == 0 || id == null || !id.matches("[A-Za-z0-9._-]{1,140}")) + { + throw new IllegalArgumentException("Invalid classified-value key or identifier"); + } + key = key.clone(); + } + + /******************************************************************************* + ** Do not expose mutable key material. + *******************************************************************************/ + @Override + public byte[] key() + { + return (key.clone()); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/secureddata/standin/ClassifiedValueTokens.java b/src/main/java/com/kingsrook/qbits/secureddata/standin/ClassifiedValueTokens.java new file mode 100644 index 0000000..8e85052 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/secureddata/standin/ClassifiedValueTokens.java @@ -0,0 +1,218 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.secureddata.standin; + + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import java.io.Serializable; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.security.GeneralSecurityException; +import java.security.MessageDigest; +import java.util.Base64; +import java.util.List; +import java.util.Optional; +import com.kingsrook.qbits.secureddata.standin.ClassifiedValueKeySource.KeyAndId; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** Shared, versioned audit/email value tokens. The MAC input is UTF-8. + *******************************************************************************/ +// STAND-IN(core=QRun-IO/qqq#1182, removeBy=RC.7) +public final class ClassifiedValueTokens +{ + + /******************************************************************************* + ** Static utility. + *******************************************************************************/ + private ClassifiedValueTokens() + { + } + + + + /******************************************************************************* + ** Bind a value to the table, field and primary key, preferring a shreddable key. + *******************************************************************************/ + public static String forRecordField(QTableMetaData table, QFieldMetaData field, Serializable pk, Serializable value) + { + if(value == null) + { + return null; + } + ClassifiedValueKeySource source = source(); + Optional key = source.keyForRecordField(table.getName(), field.getName(), pk, true); + return key.isPresent() ? token("shred", key.get(), binding(table, field, pk), value) + : instanceToken(source, binding(table, field, pk), value); + } + + + + /******************************************************************************* + ** Non-record bindings must start with a nonempty domain. + *******************************************************************************/ + public static String forContext(List bindingParts, Serializable value) + { + validateDomain(bindingParts); + return value == null ? null : instanceToken(source(), bindingParts, value); + } + + + + /******************************************************************************* + ** No derivative of the original value. + *******************************************************************************/ + public static String changed() + { + return "changed:v1"; + } + + + + /******************************************************************************* + ** A key-destruction event, never a derivative of the value. + *******************************************************************************/ + public static String shreddedToken(String ruleId) + { + if(ruleId == null || !ruleId.matches("[A-Za-z0-9._-]{1,180}")) + { + throw new IllegalArgumentException("Invalid shred rule identifier"); + } + return "shredded:v1:" + ruleId; + } + + + + /******************************************************************************* + ** Verify only keyed tokens, using existing keys and constant-time MAC comparison. + *******************************************************************************/ + public static boolean verifyRecordField(QTableMetaData table, QFieldMetaData field, Serializable pk, Serializable candidate, String token) + { + if(candidate == null || token == null) + { + return false; + } + ClassifiedValueKeySource source = source(); + if(token.startsWith("shred:v1:")) + { + return verify(source.keyForRecordField(table.getName(), field.getName(), pk, false), "shred", binding(table, field, pk), candidate, token); + } + return verify(source.instanceKey(), "hmac", binding(table, field, pk), candidate, token); + } + + + + /******************************************************************************* + ** Verify a non-record token without creating a key. + *******************************************************************************/ + public static boolean verifyContext(List bindingParts, Serializable candidate, String token) + { + validateDomain(bindingParts); + return candidate != null && token != null && verify(source().instanceKey(), "hmac", bindingParts, candidate, token); + } + + + + /******************************************************************************* + ** Canonical ValueUtils strings, with scale-independent plain decimal notation. + *******************************************************************************/ + public static String canonical(Serializable value) + { + return value instanceof BigDecimal decimal ? decimal.stripTrailingZeros().toPlainString() : ValueUtils.getValueAsString(value); + } + + + + /******************************************************************************* + ** Validate the context even for absent values. + *******************************************************************************/ + private static void validateDomain(List parts) + { + if(parts == null || parts.isEmpty() || parts.get(0) == null || parts.get(0).isBlank() || parts.stream().anyMatch(p -> p == null || p.indexOf('\u001F') >= 0)) + { + throw new IllegalArgumentException("A nonempty domain and unambiguous binding parts are required"); + } + } + + + + /******************************************************************************* + ** The agreed record binding does not prepend an additional domain. + *******************************************************************************/ + private static List binding(QTableMetaData table, QFieldMetaData field, Serializable pk) + { + return List.of(table.getName(), field.getName(), String.valueOf(pk)); + } + + + + /******************************************************************************* + ** Emit a keyed token or a value-free fallback. + *******************************************************************************/ + private static String instanceToken(ClassifiedValueKeySource source, List parts, Serializable value) + { + return source.instanceKey().map(key -> token("hmac", key, parts, value)).orElseGet(ClassifiedValueTokens::changed); + } + + + + /******************************************************************************* + ** Compare full tokens, including the key identifier and version. + *******************************************************************************/ + private static boolean verify(Optional key, String kind, List parts, Serializable value, String supplied) + { + return key.map(k -> MessageDigest.isEqual(token(kind, k, parts, value).getBytes(StandardCharsets.UTF_8), supplied.getBytes(StandardCharsets.UTF_8))).orElse(false); + } + + + + /******************************************************************************* + ** Encode the HMAC with URL-safe base64 and no padding. + *******************************************************************************/ + private static String token(String kind, KeyAndId key, List parts, Serializable value) + { + try + { + Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(key.key(), "HmacSHA256")); + byte[] digest = mac.doFinal((String.join("\u001F", parts) + "\u001F" + canonical(value)).getBytes(StandardCharsets.UTF_8)); + return kind + ":v1:" + key.id() + ":" + Base64.getUrlEncoder().withoutPadding().encodeToString(digest); + } + catch(GeneralSecurityException e) + { + throw new IllegalStateException("Unable to compute classified-value token", e); + } + } + + + + /******************************************************************************* + ** Use instance metadata, falling back to environment/property configuration. + *******************************************************************************/ + private static ClassifiedValueKeySource source() + { + return (com.kingsrook.qbits.secureddata.SecuredDataRuntime.current().keys()); + } +} diff --git a/src/main/java/com/kingsrook/qbits/secureddata/standin/ClientChannelSession.java b/src/main/java/com/kingsrook/qbits/secureddata/standin/ClientChannelSession.java deleted file mode 100644 index 763a5cb..0000000 --- a/src/main/java/com/kingsrook/qbits/secureddata/standin/ClientChannelSession.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2025. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.kingsrook.qbits.secureddata.standin; - - -import java.util.Locale; -import com.kingsrook.qqq.backend.core.model.session.QSession; - - -/******************************************************************************* - ** RC.5 host adapter for a trusted, server-supplied client channel. - *******************************************************************************/ -// STAND-IN(core=QRun-IO/qqq#1141, removeBy=RC.6): copy of com.kingsrook.qqq.backend.core.model.session.QSession client channel properties; delete when the pin moves to RC.6. -public class ClientChannelSession extends QSession -{ - private String clientChannel; - - - - /******************************************************************************* - ** Trusted host channel value; never take this from an unvalidated request. - *******************************************************************************/ - public ClientChannelSession withClientChannel(String clientChannel) - { - this.clientChannel = clientChannel; - return (this); - } - - - - /******************************************************************************* - ** Full channel including optional request detail. - *******************************************************************************/ - public String getClientChannel() - { - return (clientChannel); - } - - - - /******************************************************************************* - ** Normalized channel for policy matching. - *******************************************************************************/ - public String getClientChannelName() - { - return (clientChannel == null || clientChannel.isBlank() ? null : clientChannel.strip().split("/", 2)[0].toLowerCase(Locale.ROOT)); - } -} diff --git a/src/main/java/com/kingsrook/qbits/secureddata/standin/ClientChannelsMetaData.java b/src/main/java/com/kingsrook/qbits/secureddata/standin/ClientChannelsMetaData.java deleted file mode 100644 index f468f81..0000000 --- a/src/main/java/com/kingsrook/qbits/secureddata/standin/ClientChannelsMetaData.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2025. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.kingsrook.qbits.secureddata.standin; - - -import java.util.LinkedHashMap; -import java.util.Map; -import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.QSupplementalInstanceMetaData; - - -/******************************************************************************* - ** Temporary registry for the subset needed by this QBit. - *******************************************************************************/ -// STAND-IN(core=QRun-IO/qqq#1141, removeBy=RC.6): copy of com.kingsrook.qqq.backend.core.model.metadata.permissions.ClientChannelsMetaData registration subset; delete when the pin moves to RC.6. -public class ClientChannelsMetaData implements QSupplementalInstanceMetaData -{ - private final Map entries = new LinkedHashMap<>(); - - - - /******************************************************************************* - ** Registry name, distinct from the future core registry. - *******************************************************************************/ - @Override - public String getName() - { - return ("securedDataClientChannels"); - } - - - - /******************************************************************************* - ** Instance registry. - *******************************************************************************/ - public static ClientChannelsMetaData ofOrWithNew(QInstance instance) - { - return (QSupplementalInstanceMetaData.ofOrWithNew(instance, "securedDataClientChannels", ClientChannelsMetaData::new)); - } - - - - /******************************************************************************* - ** Register the QBit's entry. - *******************************************************************************/ - public ClientChannelsMetaData withDeniedPattern(String key, String value) - { - entries.put(key, value); - return (this); - } - - - - /******************************************************************************* - ** Read-only registration view. - *******************************************************************************/ - public Map getEntries() - { - return (Map.copyOf(entries)); - } -} diff --git a/src/test/java-uiapi/com/kingsrook/qbits/secureddata/SecuredDataUiApiContractTest.java b/src/main/java/com/kingsrook/qbits/secureddata/standin/DMLType.java similarity index 64% rename from src/test/java-uiapi/com/kingsrook/qbits/secureddata/SecuredDataUiApiContractTest.java rename to src/main/java/com/kingsrook/qbits/secureddata/standin/DMLType.java index 420bfa5..af749e4 100644 --- a/src/test/java-uiapi/com/kingsrook/qbits/secureddata/SecuredDataUiApiContractTest.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/standin/DMLType.java @@ -18,17 +18,13 @@ * limitations under the License. */ -package com.kingsrook.qbits.secureddata; +package com.kingsrook.qbits.secureddata.standin; -import java.nio.file.Path; -import com.kingsrook.qbits.secureddata.uiapi.SecuredDataExtension; -import org.junit.jupiter.api.Test; - -class SecuredDataUiApiContractTest +/******************************************************************************* + ** RC.6's nested DMLType is package-private; #1182 makes it public. + *******************************************************************************/ +// STAND-IN(core=QRun-IO/qqq#1182, removeBy=RC.7) +public enum DMLType { - @Test - void committedContractMatches() throws Exception - { - UiApiContractAssertions.assertExtensionMatchesCommitted(SecuredDataExtension.defineExtension(), Path.of("src/main/resources/openapi/qqq-v1/secured-data.yaml")); - } + INSERT, UPDATE, DELETE, OTHER } diff --git a/src/main/java/com/kingsrook/qbits/secureddata/standin/DataClassification.java b/src/main/java/com/kingsrook/qbits/secureddata/standin/DataClassification.java deleted file mode 100644 index 20138b8..0000000 --- a/src/main/java/com/kingsrook/qbits/secureddata/standin/DataClassification.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2026. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.kingsrook.qbits.secureddata.standin; - - -/******************************************************************************* - ** Data sensitivity, ordered from least to most sensitive. - ** Null means not classified and is less sensitive than every declared value. - *******************************************************************************/ -// STAND-IN(core=QRun-IO/qqq#1142, removeBy=RC.6): copy of com.kingsrook.qqq.backend.core.actions.ai.DataClassification; delete when the pin moves to RC.6. -public enum DataClassification -{ - /******************************************************************************* - ** May be shown to anyone. - *******************************************************************************/ - PUBLIC, - - /******************************************************************************* - ** Business-confidential information. - *******************************************************************************/ - CONFIDENTIAL, - - /******************************************************************************* - ** Personally identifiable information. - *******************************************************************************/ - PII, - - /******************************************************************************* - ** Protected health information under HIPAA. - *******************************************************************************/ - PHI; - - - - /******************************************************************************* - ** Whether this classification is at least as sensitive as other. - ** Every classification is at least as sensitive as null. - *******************************************************************************/ - public boolean isAtLeast(DataClassification other) - { - return (other == null || ordinal() >= other.ordinal()); - } - - - - /******************************************************************************* - ** The more sensitive classification; null loses to any declared value. - ** Returns null when both values are null. - *******************************************************************************/ - public static DataClassification max(DataClassification a, DataClassification b) - { - return (a == null ? b : a.isAtLeast(b) ? a : b); - } -} diff --git a/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldAccessDecision.java b/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldAccessDecision.java index 132c68b..27358cc 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldAccessDecision.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldAccessDecision.java @@ -26,7 +26,7 @@ package com.kingsrook.qbits.secureddata.standin; /******************************************************************************* ** A resolver decision; the reason must never contain a field value. *******************************************************************************/ -// STAND-IN(core=QRun-IO/qqq#1172, removeBy=RC.6): copy of com.kingsrook.qqq.backend.core.actions.permissions.fieldaccess.FieldAccessDecision; delete when the pin moves to RC.6. +// STAND-IN(core=QRun-IO/qqq#1172, removeBy=RC.7): copy of com.kingsrook.qqq.backend.core.actions.permissions.fieldaccess.FieldAccessDecision; delete when the pin moves to RC.7. public record FieldAccessDecision(FieldAccessTier tier, FieldMaskStyle maskStyle, String reason, boolean exactMatchAllowed) { public static final FieldAccessDecision FULL = new FieldAccessDecision(FieldAccessTier.FULL, null, null); diff --git a/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldAccessEvent.java b/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldAccessEvent.java index 93a775b..58c4ecb 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldAccessEvent.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldAccessEvent.java @@ -30,7 +30,7 @@ import java.util.List; /******************************************************************************* ** Value-free audit notification. Primary key samples contain at most 50 keys. *******************************************************************************/ -// STAND-IN(core=QRun-IO/qqq#1172, removeBy=RC.6): copy of com.kingsrook.qqq.backend.core.actions.permissions.fieldaccess.FieldAccessEvent; delete when the pin moves to RC.6. +// STAND-IN(core=QRun-IO/qqq#1172, removeBy=RC.7): copy of com.kingsrook.qqq.backend.core.actions.permissions.fieldaccess.FieldAccessEvent; delete when the pin moves to RC.7. public class FieldAccessEvent { private FieldAccessEventType type; diff --git a/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldAccessEventType.java b/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldAccessEventType.java index 331d0cd..87be7d3 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldAccessEventType.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldAccessEventType.java @@ -26,7 +26,7 @@ package com.kingsrook.qbits.secureddata.standin; /******************************************************************************* ** Kinds of field access reported to the resolver. *******************************************************************************/ -// STAND-IN(core=QRun-IO/qqq#1172, removeBy=RC.6): copy of com.kingsrook.qqq.backend.core.actions.permissions.fieldaccess.FieldAccessEventType; delete when the pin moves to RC.6. +// STAND-IN(core=QRun-IO/qqq#1172, removeBy=RC.7): copy of com.kingsrook.qqq.backend.core.actions.permissions.fieldaccess.FieldAccessEventType; delete when the pin moves to RC.7. public enum FieldAccessEventType { VIEW, EXACT_MATCH_FILTER, DENIED diff --git a/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldAccessOutcome.java b/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldAccessOutcome.java index fc6c634..199d128 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldAccessOutcome.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldAccessOutcome.java @@ -26,7 +26,7 @@ package com.kingsrook.qbits.secureddata.standin; /******************************************************************************* ** Outcome of an exact-match search. *******************************************************************************/ -// STAND-IN(core=QRun-IO/qqq#1172, removeBy=RC.6): copy of com.kingsrook.qqq.backend.core.actions.permissions.fieldaccess.FieldAccessOutcome; delete when the pin moves to RC.6. +// STAND-IN(core=QRun-IO/qqq#1172, removeBy=RC.7): copy of com.kingsrook.qqq.backend.core.actions.permissions.fieldaccess.FieldAccessOutcome; delete when the pin moves to RC.7. public enum FieldAccessOutcome { MATCH, NO_MATCH, REFUSED diff --git a/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldAccessResolverInterface.java b/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldAccessResolverInterface.java index da9916e..9acca83 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldAccessResolverInterface.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldAccessResolverInterface.java @@ -30,7 +30,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; /******************************************************************************* ** Session-aware policy, reveal grants and value-free access notifications. *******************************************************************************/ -// STAND-IN(core=QRun-IO/qqq#1172, removeBy=RC.6): copy of com.kingsrook.qqq.backend.core.actions.permissions.fieldaccess.FieldAccessResolverInterface; delete when the pin moves to RC.6. +// STAND-IN(core=QRun-IO/qqq#1172, removeBy=RC.7): copy of com.kingsrook.qqq.backend.core.actions.permissions.fieldaccess.FieldAccessResolverInterface; delete when the pin moves to RC.7. public interface FieldAccessResolverInterface { /******************************************************************************* diff --git a/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldAccessTier.java b/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldAccessTier.java index 7fea6a4..2c13a92 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldAccessTier.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldAccessTier.java @@ -27,7 +27,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperat /******************************************************************************* ** Effective field access, ordered from least to most restrictive. *******************************************************************************/ -// STAND-IN(core=QRun-IO/qqq#1172, removeBy=RC.6): copy of com.kingsrook.qqq.backend.core.actions.permissions.fieldaccess.FieldAccessTier; delete when the pin moves to RC.6. +// STAND-IN(core=QRun-IO/qqq#1172, removeBy=RC.7): copy of com.kingsrook.qqq.backend.core.actions.permissions.fieldaccess.FieldAccessTier; delete when the pin moves to RC.7. public enum FieldAccessTier { FULL, REVEALABLE, MASKED, HIDDEN; diff --git a/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldMaskStyle.java b/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldMaskStyle.java index 52f9493..25531d1 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldMaskStyle.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/standin/FieldMaskStyle.java @@ -26,7 +26,7 @@ package com.kingsrook.qbits.secureddata.standin; /******************************************************************************* ** Presentation rules for a restricted value. *******************************************************************************/ -// STAND-IN(core=QRun-IO/qqq#1172, removeBy=RC.6): copy of com.kingsrook.qqq.backend.core.actions.permissions.fieldaccess.FieldMaskStyle; delete when the pin moves to RC.6. +// STAND-IN(core=QRun-IO/qqq#1172, removeBy=RC.7): copy of com.kingsrook.qqq.backend.core.actions.permissions.fieldaccess.FieldMaskStyle; delete when the pin moves to RC.7. public enum FieldMaskStyle { AUTO, EMAIL, LAST_4, FIRST_LAST, FULL diff --git a/src/main/java/com/kingsrook/qbits/secureddata/standin/ProtectedAuditValues.java b/src/main/java/com/kingsrook/qbits/secureddata/standin/ProtectedAuditValues.java new file mode 100644 index 0000000..73dc40c --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/secureddata/standin/ProtectedAuditValues.java @@ -0,0 +1,30 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.secureddata.standin; + + +/******************************************************************************* + ** Tokens and a value-free message supplied before an audit is inserted. + *******************************************************************************/ +// STAND-IN(core=QRun-IO/qqq#1182, removeBy=RC.7) +public record ProtectedAuditValues(String oldToken, String newToken, String message) +{ +} diff --git a/src/main/java/com/kingsrook/qbits/secureddata/standin/TableFeaturePermissionsMetaData.java b/src/main/java/com/kingsrook/qbits/secureddata/standin/TableFeaturePermissionsMetaData.java deleted file mode 100644 index c7528dd..0000000 --- a/src/main/java/com/kingsrook/qbits/secureddata/standin/TableFeaturePermissionsMetaData.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2025. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.kingsrook.qbits.secureddata.standin; - - -import java.util.LinkedHashMap; -import java.util.Map; -import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.QSupplementalInstanceMetaData; - - -/******************************************************************************* - ** Temporary registry for the subset needed by this QBit. - *******************************************************************************/ -// STAND-IN(core=QRun-IO/qqq#1142, removeBy=RC.6): copy of com.kingsrook.qqq.backend.core.model.metadata.permissions.TableFeaturePermissionsMetaData registration subset; delete when the pin moves to RC.6. -public class TableFeaturePermissionsMetaData implements QSupplementalInstanceMetaData -{ - private final Map entries = new LinkedHashMap<>(); - - - - /******************************************************************************* - ** Registry name, distinct from the future core registry. - *******************************************************************************/ - @Override - public String getName() - { - return ("securedDataTableFeatures"); - } - - - - /******************************************************************************* - ** Instance registry. - *******************************************************************************/ - public static TableFeaturePermissionsMetaData ofOrWithNew(QInstance instance) - { - return (QSupplementalInstanceMetaData.ofOrWithNew(instance, "securedDataTableFeatures", TableFeaturePermissionsMetaData::new)); - } - - - - /******************************************************************************* - ** Register the QBit's entry. - *******************************************************************************/ - public TableFeaturePermissionsMetaData withPermission(String key, String value) - { - entries.put(key, value); - return (this); - } - - - - /******************************************************************************* - ** Read-only registration view. - *******************************************************************************/ - public Map getEntries() - { - return (Map.copyOf(entries)); - } -} diff --git a/src/main/java/com/kingsrook/qbits/secureddata/support/ActivityLog.java b/src/main/java/com/kingsrook/qbits/secureddata/support/ActivityLog.java index 89a1294..f350cde 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/support/ActivityLog.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/support/ActivityLog.java @@ -45,7 +45,7 @@ public final class ActivityLog return (new QRecord().withValue("event_type", event).withValue("table_name", table) .withValue("user_id", SecuredDataResolver.userId()) .withValue("user_name", QContext.getQSession().getUser() == null ? "system" : QContext.getQSession().getUser().getFullName()) - .withValue("client_channel", CoreMetadataBridge.channel(QContext.getQSession())) + .withValue("client_channel", SecuredDataMetadata.channel(QContext.getQSession())) .withValue("created_at", SecuredDataRuntime.current().config().getClock().instant()) .withValue("outcome", outcome).withValue("anomaly_flag", false)); } diff --git a/src/main/java/com/kingsrook/qbits/secureddata/support/CoreMetadataBridge.java b/src/main/java/com/kingsrook/qbits/secureddata/support/CoreMetadataBridge.java deleted file mode 100644 index 3eeb78c..0000000 --- a/src/main/java/com/kingsrook/qbits/secureddata/support/CoreMetadataBridge.java +++ /dev/null @@ -1,286 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2025. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.kingsrook.qbits.secureddata.support; - - -import java.lang.reflect.InvocationTargetException; -import com.kingsrook.qbits.secureddata.standin.ClientChannelsMetaData; -import com.kingsrook.qbits.secureddata.standin.DataClassification; -import com.kingsrook.qbits.secureddata.standin.TableFeaturePermissionsMetaData; -import com.kingsrook.qqq.backend.core.actions.ai.AiExposure; -import com.kingsrook.qqq.backend.core.actions.ai.AiFieldMetaData; -import com.kingsrook.qqq.backend.core.actions.ai.AiTableMetaData; -import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; -import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType; -import com.kingsrook.qqq.backend.core.context.QContext; -import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; -import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; -import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.model.session.QSession; - - -/******************************************************************************* - ** Adapters for #1141/#1142 APIs missing from the pinned released core. - *******************************************************************************/ -public final class CoreMetadataBridge -{ - private static final String PERMISSIONS = "com.kingsrook.qqq.backend.core.model.metadata.permissions."; - - - - /******************************************************************************* - ** Utility class. - *******************************************************************************/ - private CoreMetadataBridge() - { - } - - - - /******************************************************************************* - ** Normalized client channel, with RC.5 session-adapter support. - *******************************************************************************/ - public static String channel(QSession session) - { - return (session == null ? null : (String) optional(session, "getClientChannelName")); - } - - - - /******************************************************************************* - ** Read a code classification from released/future core or temporary metadata. - *******************************************************************************/ - public static DataClassification classification(QTableMetaData table, QFieldMetaData field) - { - Object ai = field == null ? AiTableMetaData.of(table) : AiFieldMetaData.of(field); - Object result = ai == null ? null : optional(ai, "getClassification"); - if(result != null) - { - return (DataClassification.valueOf(result.toString())); - } - Object fallback = field == null ? table.getSupplementalMetaData("securedDataAi") : field.getSupplementalMetaData("securedDataAi"); - return (fallback == null ? null : (DataClassification) optional(fallback, "getClassification")); - } - - - - /******************************************************************************* - ** Protect the QBit's own AI routing metadata on either core version. - *******************************************************************************/ - public static void classifyTable(QTableMetaData table) - { - AiTableMetaData ai = AiTableMetaData.ofOrWithNew(table); - ai.withExposure(AiExposure.NONE); - setClassification(ai, DataClassification.CONFIDENTIAL); - table.withSupplementalMetaData(new com.kingsrook.qbits.secureddata.standin.AiTableMetaData().withClassification(DataClassification.CONFIDENTIAL)); - } - - - - /******************************************************************************* - ** Seed opt-in name heuristics without overriding explicit code metadata. - *******************************************************************************/ - public static void classifyField(QFieldMetaData field, DataClassification classification) - { - AiFieldMetaData ai = AiFieldMetaData.of(field); - if(ai == null) - { - ai = new AiFieldMetaData(); - field.withSupplementalMetaData(ai); - } - setClassification(ai, classification); - field.withSupplementalMetaData(new com.kingsrook.qbits.secureddata.standin.AiFieldMetaData().withClassification(classification)); - } - - - - /******************************************************************************* - ** Copy the classification into core when that property exists. - *******************************************************************************/ - private static void setClassification(Object ai, DataClassification classification) - { - try - { - Class type = Class.forName("com.kingsrook.qqq.backend.core.actions.ai.DataClassification"); - Object value = type.getMethod("valueOf", String.class).invoke(null, classification.name()); - ai.getClass().getMethod("withClassification", type).invoke(ai, value); - } - catch(ClassNotFoundException | NoSuchMethodException expected) - { - // RC.5 has exposure-only metadata; the stand-in carries classification. - } - catch(ReflectiveOperationException e) - { - throw (new IllegalStateException("Unable to set core classification", e)); - } - } - - - - /******************************************************************************* - ** Register the assistant wall and both features. Catalog applies to all tables. - *******************************************************************************/ - public static void register(QInstance instance) - { - ClientChannelsMetaData.ofOrWithNew(instance).withDeniedPattern("assistant", "secured_data_*"); - registerChannel(instance); - for(String suffix : new String[] {"secured_data_full", "secured_data_reveal"}) - { - String label = "Secured data: " + suffix.substring("secured_data_".length()); - TableFeaturePermissionsMetaData.ofOrWithNew(instance).withPermission(suffix, label); - try - { - Class registry = Class.forName(PERMISSIONS + "TableFeaturePermissionsMetaData"); - Class entry = Class.forName(PERMISSIONS + "TableFeaturePermission"); - Object feature = entry.getConstructor().newInstance(); - entry.getMethod("withSuffix", String.class).invoke(feature, suffix); - entry.getMethod("withLabel", String.class).invoke(feature, label); - Object metadata = registry.getMethod("ofOrWithNew", QInstance.class).invoke(null, instance); - registry.getMethod("withPermission", entry).invoke(metadata, feature); - } - catch(ClassNotFoundException expected) - { - // RC.5 uses the stand-in catalog until core exposes the registry. - } - catch(ReflectiveOperationException e) - { - throw (new IllegalStateException("Unable to register table features", e)); - } - } - } - - - - /******************************************************************************* - ** Core channel enforcement when the #1141 API is installed. - *******************************************************************************/ - private static void registerChannel(QInstance instance) - { - try - { - Class type = Class.forName(PERMISSIONS + "ClientChannelsMetaData"); - Object metadata = type.getMethod("ofOrWithNew", QInstance.class).invoke(null, instance); - type.getMethod("withDeniedPattern", String.class, String.class).invoke(metadata, "assistant", "secured_data_*"); - } - catch(ClassNotFoundException expected) - { - // The RC.5 custom permission checker also enforces the local wall. - } - catch(ReflectiveOperationException e) - { - throw (new IllegalStateException("Unable to register client channel wall", e)); - } - } - - - - /******************************************************************************* - ** Use core's feature semantics, including table READ and permission base names. - *******************************************************************************/ - public static boolean hasFeature(QTableMetaData table, String suffix) - { - try - { - return ((Boolean) PermissionsHelper.class.getMethod("hasTableFeaturePermission", QTableMetaData.class, String.class).invoke(null, table, suffix)); - } - catch(NoSuchMethodException expected) - { - if(table.getIsHidden() || !PermissionsHelper.hasTablePermission(new GetInput(table.getName()), table.getName(), TablePermissionSubType.READ)) - { - return (false); - } - QPermissionRules rules = PermissionsHelper.getEffectivePermissionRules(table, QContext.getQInstance()); - String base = permissionBase(table); - return (rules.getLevel() == PermissionLevel.NOT_PROTECTED || QContext.getQSession().hasPermission(base + "." + suffix)); - } - catch(ReflectiveOperationException e) - { - throw (new IllegalStateException("Core feature permission check failed", e)); - } - } - - - - /******************************************************************************* - ** Missing properties are expected only on RC.5; invocation errors fail closed. - *******************************************************************************/ - private static Object optional(Object target, String name) - { - try - { - return (target.getClass().getMethod(name).invoke(target)); - } - catch(NoSuchMethodException expected) - { - return (null); - } - catch(IllegalAccessException | InvocationTargetException e) - { - throw (new IllegalStateException("Core metadata adapter failed", e)); - } - } - - - /******************************************************************************* - ** RC.5 keeps core's equivalent helper package-private. - *******************************************************************************/ - public static String permissionBase(QTableMetaData table) - { - QPermissionRules rules = PermissionsHelper.getEffectivePermissionRules(table, QContext.getQInstance()); - String base = rules.getPermissionBaseName(); - return (base == null || base.isBlank() ? table.getName() : base); - } - - - - /******************************************************************************* - ** Promote temporary host code classifications so future core AI routing sees them. - *******************************************************************************/ - public static void promoteClassifications(QInstance instance) - { - for(QTableMetaData table : instance.getTables().values()) - { - DataClassification tableClassification = classification(table, null); - if(tableClassification != null) - { - setClassification(AiTableMetaData.ofOrWithNew(table), tableClassification); - } - for(QFieldMetaData field : table.getFields().values()) - { - DataClassification fieldClassification = classification(table, field); - if(fieldClassification != null) - { - AiFieldMetaData ai = AiFieldMetaData.of(field); - if(ai == null) - { - ai = new AiFieldMetaData(); - field.withSupplementalMetaData(ai); - } - setClassification(ai, fieldClassification); - } - } - } - } - -} diff --git a/src/main/java/com/kingsrook/qbits/secureddata/support/SecuredDataMetadata.java b/src/main/java/com/kingsrook/qbits/secureddata/support/SecuredDataMetadata.java new file mode 100644 index 0000000..a5e35c5 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/secureddata/support/SecuredDataMetadata.java @@ -0,0 +1,124 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.secureddata.support; + +import com.kingsrook.qqq.backend.core.actions.ai.AiExposure; +import com.kingsrook.qqq.backend.core.actions.ai.AiFieldMetaData; +import com.kingsrook.qqq.backend.core.actions.ai.AiTableMetaData; +import com.kingsrook.qqq.backend.core.actions.ai.DataClassification; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.ClientChannelsMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.TableFeaturePermission; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.TableFeaturePermissionsMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; + +/******************************************************************************* + ** Secured-data defaults and registration using the released core metadata API. + *******************************************************************************/ +public final class SecuredDataMetadata +{ + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private SecuredDataMetadata() + { + } + + /******************************************************************************* + ** Normalized trusted client channel. + *******************************************************************************/ + public static String channel(QSession session) + { + return (session == null ? null : session.getClientChannelName()); + } + + /******************************************************************************* + ** Read code classification without applying field/table precedence. + *******************************************************************************/ + public static DataClassification classification(QTableMetaData table, QFieldMetaData field) + { + if(field == null) + { + AiTableMetaData ai = AiTableMetaData.of(table); + return (ai == null ? null : ai.getClassification()); + } + AiFieldMetaData ai = AiFieldMetaData.of(field); + return (ai == null ? null : ai.getClassification()); + } + + /******************************************************************************* + ** Protect the QBit's own AI routing metadata. + *******************************************************************************/ + public static void classifyTable(QTableMetaData table) + { + AiTableMetaData.ofOrWithNew(table).withExposure(AiExposure.NONE).withClassification(DataClassification.CONFIDENTIAL); + } + + /******************************************************************************* + ** Seed opt-in name heuristics without replacing other AI field metadata. + *******************************************************************************/ + public static void classifyField(QFieldMetaData field, DataClassification classification) + { + AiFieldMetaData ai = AiFieldMetaData.of(field); + if(ai == null) + { + ai = new AiFieldMetaData(); + field.withSupplementalMetaData(ai); + } + ai.withClassification(classification); + } + + /******************************************************************************* + ** Register the assistant wall and both table features. + *******************************************************************************/ + public static void register(QInstance instance) + { + ClientChannelsMetaData.ofOrWithNew(instance).withDeniedPattern("assistant", "secured_data_*"); + for(String suffix : new String[] {"secured_data_full", "secured_data_reveal"}) + { + TableFeaturePermissionsMetaData.ofOrWithNew(instance).withPermission(new TableFeaturePermission() + .withSuffix(suffix).withLabel("Secured data: " + suffix.substring("secured_data_".length()))); + } + } + + /******************************************************************************* + ** Use core feature semantics, including table READ and permission base names. + *******************************************************************************/ + public static boolean hasFeature(QTableMetaData table, String suffix) + { + return (PermissionsHelper.hasTableFeaturePermission(table, suffix)); + } + + /******************************************************************************* + ** Effective permission base name. + *******************************************************************************/ + public static String permissionBase(QTableMetaData table) + { + QPermissionRules rules = PermissionsHelper.getEffectivePermissionRules(table, QContext.getQInstance()); + String base = rules.getPermissionBaseName(); + return (base == null || base.isBlank() ? table.getName() : base); + } +} diff --git a/src/main/java/com/kingsrook/qbits/secureddata/support/SystemWrites.java b/src/main/java/com/kingsrook/qbits/secureddata/support/SystemWrites.java index a18d3c9..0e3de9c 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/support/SystemWrites.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/support/SystemWrites.java @@ -106,7 +106,7 @@ public final class SystemWrites { try { - return (call(() -> new QueryAction().execute(new QueryInput(table).withFilter(filter)).getRecords())); + return (call(() -> new QueryAction().execute(new QueryInput(table).withFilter(filter).withTransaction(QContext.getQBackendTransaction())).getRecords())); } catch(QException e) { diff --git a/src/main/java-uiapi/com/kingsrook/qbits/secureddata/uiapi/SecuredDataExtension.java b/src/main/java/com/kingsrook/qbits/secureddata/uiapi/SecuredDataExtension.java similarity index 95% rename from src/main/java-uiapi/com/kingsrook/qbits/secureddata/uiapi/SecuredDataExtension.java rename to src/main/java/com/kingsrook/qbits/secureddata/uiapi/SecuredDataExtension.java index 189aa0f..4ef4755 100644 --- a/src/main/java-uiapi/com/kingsrook/qbits/secureddata/uiapi/SecuredDataExtension.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/uiapi/SecuredDataExtension.java @@ -25,7 +25,7 @@ import com.kingsrook.qqq.middleware.javalin.specs.extensions.MiddlewareSpecExten import com.kingsrook.qqq.middleware.javalin.specs.extensions.MiddlewareSpecExtensionsMetaData; /******************************************************************************* - ** STAND-IN(core=QRun-IO/qqq#1128, removeBy=RC.6): move to main source root. + ** Secured-data extension for the released UI API. *******************************************************************************/ public final class SecuredDataExtension { @@ -47,7 +47,7 @@ public final class SecuredDataExtension } /******************************************************************************* - ** Called reflectively by the producer. + ** Register directly with the middleware extension catalog. *******************************************************************************/ public static void register(QInstance instance) { diff --git a/src/main/java-uiapi/com/kingsrook/qbits/secureddata/uiapi/SecuredDataRevealOptionsSpecV1.java b/src/main/java/com/kingsrook/qbits/secureddata/uiapi/SecuredDataRevealOptionsSpecV1.java similarity index 100% rename from src/main/java-uiapi/com/kingsrook/qbits/secureddata/uiapi/SecuredDataRevealOptionsSpecV1.java rename to src/main/java/com/kingsrook/qbits/secureddata/uiapi/SecuredDataRevealOptionsSpecV1.java diff --git a/src/main/java-uiapi/com/kingsrook/qbits/secureddata/uiapi/SecuredDataRevealSpecV1.java b/src/main/java/com/kingsrook/qbits/secureddata/uiapi/SecuredDataRevealSpecV1.java similarity index 100% rename from src/main/java-uiapi/com/kingsrook/qbits/secureddata/uiapi/SecuredDataRevealSpecV1.java rename to src/main/java/com/kingsrook/qbits/secureddata/uiapi/SecuredDataRevealSpecV1.java diff --git a/src/main/java-uiapi/com/kingsrook/qbits/secureddata/uiapi/SecuredDataSchemas.java b/src/main/java/com/kingsrook/qbits/secureddata/uiapi/SecuredDataSchemas.java similarity index 100% rename from src/main/java-uiapi/com/kingsrook/qbits/secureddata/uiapi/SecuredDataSchemas.java rename to src/main/java/com/kingsrook/qbits/secureddata/uiapi/SecuredDataSchemas.java diff --git a/src/main/java-uiapi/com/kingsrook/qbits/secureddata/uiapi/SecuredDataStatusSpecV1.java b/src/main/java/com/kingsrook/qbits/secureddata/uiapi/SecuredDataStatusSpecV1.java similarity index 100% rename from src/main/java-uiapi/com/kingsrook/qbits/secureddata/uiapi/SecuredDataStatusSpecV1.java rename to src/main/java/com/kingsrook/qbits/secureddata/uiapi/SecuredDataStatusSpecV1.java diff --git a/src/test/java/com/kingsrook/qbits/secureddata/CoreFieldAccessBridgeIT.java b/src/test/java/com/kingsrook/qbits/secureddata/CoreFieldAccessBridgeIT.java index cfa577a..bc56526 100644 --- a/src/test/java/com/kingsrook/qbits/secureddata/CoreFieldAccessBridgeIT.java +++ b/src/test/java/com/kingsrook/qbits/secureddata/CoreFieldAccessBridgeIT.java @@ -78,7 +78,7 @@ class CoreFieldAccessBridgeIT { available = false; } - Assumptions.assumeTrue(available, "QQQ RC.6 field-access core is not on this classpath"); + Assumptions.assumeTrue(available, "QQQ RC.7 field-access core is not on this classpath"); } diff --git a/src/test/java/com/kingsrook/qbits/secureddata/LogCapture.java b/src/test/java/com/kingsrook/qbits/secureddata/LogCapture.java new file mode 100644 index 0000000..9161535 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/secureddata/LogCapture.java @@ -0,0 +1,119 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.secureddata; + + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.Appender; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.appender.AbstractAppender; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.LoggerConfig; +import org.apache.logging.log4j.core.config.Property; + + +/******************************************************************************* + ** Captures every log event, at every level, from every logger while open. + ** Used by the canary test to prove no protected value reaches a log line. + *******************************************************************************/ +final class LogCapture implements AutoCloseable +{ + private final List lines = new ArrayList<>(); + private final Map originalLevels = new HashMap<>(); + private final LoggerContext context; + private final Appender appender; + + + + /******************************************************************************* + ** Attach to the root logger and force all configured loggers to ALL. + *******************************************************************************/ + LogCapture() + { + context = (LoggerContext) LogManager.getContext(false); + Configuration configuration = context.getConfiguration(); + appender = new AbstractAppender("secured-data-log-capture", null, null, true, Property.EMPTY_ARRAY) + { + @Override + public void append(LogEvent event) + { + StringBuilder line = new StringBuilder(event.getLoggerName()).append(' ').append(event.getMessage().getFormattedMessage()); + for(Throwable t = event.getThrown(); t != null; t = t.getCause()) + { + line.append(' ').append(t.getClass().getName()).append(' ').append(t.getMessage()); + } + synchronized(lines) + { + lines.add(line.toString()); + } + } + }; + appender.start(); + configuration.addAppender(appender); + List all = new ArrayList<>(configuration.getLoggers().values()); + all.add(configuration.getRootLogger()); + for(LoggerConfig config : all) + { + originalLevels.put(config.getName(), config.getLevel()); + config.setLevel(Level.ALL); + } + configuration.getRootLogger().addAppender(appender, Level.ALL, null); + context.updateLoggers(); + } + + + + /******************************************************************************* + ** Every captured line so far. + *******************************************************************************/ + List lines() + { + synchronized(lines) + { + return (new ArrayList<>(lines)); + } + } + + + + /******************************************************************************* + ** Detach and restore levels. + *******************************************************************************/ + @Override + public void close() + { + Configuration configuration = context.getConfiguration(); + configuration.getRootLogger().removeAppender(appender.getName()); + for(LoggerConfig config : configuration.getLoggers().values()) + { + config.setLevel(originalLevels.get(config.getName())); + } + configuration.getRootLogger().setLevel(originalLevels.get(configuration.getRootLogger().getName())); + context.updateLoggers(); + appender.stop(); + } +} diff --git a/src/test/java/com/kingsrook/qbits/secureddata/RetentionProofTest.java b/src/test/java/com/kingsrook/qbits/secureddata/RetentionProofTest.java new file mode 100644 index 0000000..0e473b3 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/secureddata/RetentionProofTest.java @@ -0,0 +1,371 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.secureddata; + + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import java.io.ByteArrayOutputStream; +import java.io.Serializable; +import java.nio.charset.StandardCharsets; +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.Base64; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import com.kingsrook.qbits.secureddata.metadata.SecuredDataTables; +import com.kingsrook.qbits.secureddata.retention.WipeService; +import com.kingsrook.qbits.secureddata.standin.ClassifiedValueKeySource.KeyAndId; +import com.kingsrook.qbits.secureddata.standin.ClassifiedValueTokens; +import com.kingsrook.qbits.secureddata.support.SystemWrites; +import com.kingsrook.qqq.backend.core.actions.reporting.ExportAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static com.kingsrook.qbits.secureddata.RetentionTestSupport.rows; +import static com.kingsrook.qbits.secureddata.RetentionTestSupport.rule; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** SECURED-DATA section 15 proofs: a planted canary never leaves the host table, + ** and shredded audit tokens cannot be brute-forced once the data key is gone. + ** Both need the core audit-protection hooks (#1182), so RC.6 skips them. + *******************************************************************************/ +class RetentionProofTest +{ + private static final SecureRandom RANDOM = new SecureRandom(); + private static final String OTHER_KEY = "/v79vLu6ubi3trW0s7KxsK+uraypqKempaSjoqGgn56dnJuamZiXlpU="; + + + + /******************************************************************************* + ** Skip unless core supplies the audit hooks. + *******************************************************************************/ + @BeforeEach + void requireCoreHooks() + { + Assumptions.assumeTrue(CoreAuditBridge.available(), "QQQ core audit-protection hooks (#1182) are not on this classpath"); + } + + + + /******************************************************************************* + ** Cleanup. + *******************************************************************************/ + @AfterEach + void cleanup() + { + RetentionTestSupport.close(); + } + + + + /******************************************************************************* + ** Letters g-v only, so a marker can never coincide with hex digests or tokens. + *******************************************************************************/ + private static String canary() + { + StringBuilder marker = new StringBuilder("CANARY-PHI-"); + for(int i = 0; i < 16; i++) + { + marker.append((char) ('g' + RANDOM.nextInt(16))); + } + return (marker.toString()); + } + + + + /******************************************************************************* + ** The marker, or any substring of it longer than four characters. + *******************************************************************************/ + private static List leaks(String haystack, String... markers) + { + List found = new ArrayList<>(); + for(String marker : markers) + { + for(int length = 5; length <= marker.length(); length++) + { + for(int start = 0; start + length <= marker.length(); start++) + { + if(haystack.contains(marker.substring(start, start + length))) + { + found.add("length " + length + " at " + start); + } + } + } + } + return (found); + } + + + + /******************************************************************************* + ** Canary through CRUD, a failing validation, a wipe run, a masked export and + ** the access log: it may appear only in the host table itself. + *******************************************************************************/ + @Test + void canaryNeverLeavesTheHostTable() throws Exception + { + String first = canary(); + String second = canary(); + String rejected = canary(); + try(LogCapture logs = new LogCapture()) + { + QInstance instance = RetentionTestSupport.setup(); + QLogger.getLogger(RetentionProofTest.class).info("log-capture-sentinel"); + assertThat(logs.lines()).anyMatch(line -> line.contains("log-capture-sentinel")); + SecuredDataTestSupport.policy("viewer", "PHI", "MASKED"); + SecuredDataTestSupport.policy("viewer", "PII", "MASKED"); + SecuredDataTestSupport.policy("secured_data.admin", "PHI", "FULL"); + SecuredDataTestSupport.policy("secured_data.admin", "PII", "FULL"); + QRecord rule = rule("NULL_WITH_MARKER", "secret,notes"); + + // CRUD with audits. + List errors = new ArrayList<>(); + QRecord inserted = new InsertAction().execute(new InsertInput("reading").withRecord(new QRecord().withValue("secret", first).withValue("notes", first) + .withValue("systolic", 120).withValue("reading_time", RetentionTestSupport.OLD))).getRecords().getFirst(); + assertThat(inserted.getErrors()).isNullOrEmpty(); + Serializable id = inserted.getValue("id"); + QRecord updated = new UpdateAction().execute(new UpdateInput("reading").withRecord(new QRecord().withValue("id", id).withValue("secret", second) + .withValue("notes", second))).getRecords().getFirst(); + assertThat(updated.getErrors()).isNullOrEmpty(); + + // A failing validation: the secret value is over the field's maximum length. + QRecord invalid = new InsertAction().execute(new InsertInput("reading").withRecord(new QRecord().withValue("secret", rejected + "x".repeat(300)) + .withValue("notes", rejected).withValue("reading_time", RetentionTestSupport.OLD))).getRecords().getFirst(); + assertThat(invalid.getErrors()).as("the invalid insert must be reported as an error").isNotEmpty(); + invalid.getErrors().forEach(error -> errors.add(String.valueOf(error.getMessage()))); + + // A masked user reads and exports; the access log records the views. + QContext.setQSession(SecuredDataTestSupport.user("viewer", "viewer", "reading.read")); + assertThat(new QueryAction().execute(new QueryInput("reading")).getRecords()).hasSize(1); + ByteArrayOutputStream export = new ByteArrayOutputStream(); + ExportInput exportInput = new ExportInput(); + exportInput.setTableName("reading"); + new ExportAction().execute(exportInput.withReportDestination(new ReportDestination().withReportFormat(ReportFormat.CSV).withReportOutputStream(export))); + String exported = export.toString(StandardCharsets.UTF_8); + assertThat(exported).isNotBlank(); + SecuredDataRuntime.current().logger().flush(); + + // Positive control: the scan does detect the planted value where it legitimately lives. + assertThat(leaks(JsonUtils.toJson(rows("reading")), second)).isNotEmpty(); + + // The wipe run, as the retention administrator. + QContext.setQSession(SecuredDataTestSupport.user("admin", "secured_data.admin", "secured_data.retention", "reading.read", "reading.write")); + QRecord run = new WipeService().batch(rule.getValue("id"), List.of(id)); + assertThat(run.getValueInteger("wiped_count")).isEqualTo(1); + assertThat(rows("reading").getFirst().getValue("secret")).isNull(); + assertThat(leaks(JsonUtils.toJson(rows("reading")), first, second)).as("the wiped host table").isEmpty(); + + List places = new ArrayList<>(); + Map haystacks = new LinkedHashMap<>(); + haystacks.put("export", exported); + haystacks.put("errors", String.join(" ", errors)); + haystacks.put("logs", String.join("\n", logs.lines())); + for(String table : List.of("audit", "auditDetail", SecuredDataTables.ACCESS_LOG, SecuredDataTables.WIPE_RUN, SecuredDataTables.WIPE_MARKER, + SecuredDataTables.AUDIT_KEY, SecuredDataTables.RETENTION_RULE, SecuredDataTables.LEGAL_HOLD, SecuredDataTables.POLICY, SecuredDataTables.CLASSIFICATION)) + { + haystacks.put(table, JsonUtils.toJson(rows(table))); + assertThat(instance.getTable(table)).isNotNull(); + } + haystacks.forEach((name, text) -> + { + if(!leaks(text, first, second, rejected).isEmpty()) + { + places.add(name); + } + }); + assertThat(places).as("places where the canary or a long fragment of it appeared").isEmpty(); + assertThat(haystacks.get("audit")).isNotBlank(); + assertThat(rows("auditDetail")).isNotEmpty(); + assertThat(rows(SecuredDataTables.ACCESS_LOG)).isNotEmpty(); + } + } + + + + /******************************************************************************* + ** HMAC-SHA256 over the agreed record binding, exactly as the token format does. + *******************************************************************************/ + private static String digest(byte[] key, QTableMetaData table, String field, Serializable pk, int value) throws Exception + { + Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(key, "HmacSHA256")); + byte[] out = mac.doFinal((String.join("\u001F", table.getName(), field, String.valueOf(pk)) + "\u001F" + value).getBytes(StandardCharsets.UTF_8)); + return (Base64.getUrlEncoder().withoutPadding().encodeToString(out)); + } + + + + /******************************************************************************* + ** After the wipe nothing can recover a systolic reading from its audit tokens, + ** not even with every key still reachable, while before the wipe the same + ** procedure does find the value (so the search genuinely detects a match). + *******************************************************************************/ + @Test + void shreddedAuditTokensCannotBeBruteForcedAfterTheWipe() throws Exception + { + QInstance instance = RetentionTestSupport.setup(); + System.setProperty("QQQ_AUDIT_HMAC_KEY", OTHER_KEY); + QTableMetaData table = instance.getTable("reading"); + SecuredDataTestSupport.policy("secured_data.admin", "PHI", "FULL"); + SecuredDataTestSupport.policy("secured_data.admin", "PII", "FULL"); + QRecord rule = rule("NULL_WITH_MARKER", "systolic"); + + List toInsert = new ArrayList<>(); + for(int systolic = 60; systolic <= 250; systolic++) + { + toInsert.add(new QRecord().withValue("systolic", systolic).withValue("secret", "s").withValue("reading_time", RetentionTestSupport.OLD)); + } + List inserted = new InsertAction().execute(new InsertInput("reading").withRecords(toInsert)).getRecords(); + assertThat(inserted).hasSize(191).allSatisfy(r -> assertThat(r.getErrors()).isNullOrEmpty()); + Map truth = new LinkedHashMap<>(); + inserted.forEach(r -> truth.put(r.getValue("id"), r.getValueInteger("systolic"))); + + // A control record the wipe will not touch: its data key stays reachable. + QRecord control = new InsertAction().execute(new InsertInput("reading").withRecord(new QRecord().withValue("systolic", 111) + .withValue("reading_time", SecuredDataTestSupport.NOW))).getRecords().getFirst(); + + Map> tokens = auditTokens(truth.keySet()); + assertThat(tokens).hasSize(191).allSatisfy((id, list) -> assertThat(list).isNotEmpty().allMatch(t -> t.startsWith("shred:v1:"))); + + // Before the wipe, the DEK verifies the true value and no other value. + var keys = SecuredDataRuntime.current().keys(); + for(var entry : truth.entrySet()) + { + Optional dek = keys.keyForRecordField("reading", "systolic", entry.getKey(), false); + assertThat(dek).isPresent(); + for(String token : tokens.get(entry.getKey())) + { + assertThat(ClassifiedValueTokens.verifyRecordField(table, table.getField("systolic"), entry.getKey(), entry.getValue(), token)).isTrue(); + assertThat(ClassifiedValueTokens.verifyRecordField(table, table.getField("systolic"), entry.getKey(), entry.getValue() + 1, token)).isFalse(); + assertThat(token).endsWith(digest(dek.get().key(), table, "systolic", entry.getKey(), entry.getValue())); + } + } + assertThat(rows(SecuredDataTables.AUDIT_KEY)).hasSize(192); + + // Wipe the old readings in policy-sized batches. + List ids = new ArrayList<>(truth.keySet()); + for(int offset = 0; offset < ids.size(); offset += 100) + { + new WipeService().batch(rule.getValue("id"), ids.subList(offset, Math.min(offset + 100, ids.size()))); + } + assertThat(rows("reading").stream().filter(r -> truth.containsKey(r.getValue("id")))).allSatisfy(r -> assertThat(r.getValue("systolic")).isNull()); + assertThat(rows(SecuredDataTables.AUDIT_KEY)).hasSize(1); + assertThat(rows(SecuredDataTables.AUDIT_KEY).getFirst().getValueString("record_id")).isEqualTo(String.valueOf(control.getValue("id"))); + + // Every key the test can still reach. + List reachable = new ArrayList<>(); + reachable.add(Base64.getDecoder().decode(OTHER_KEY)); + reachable.add(Base64.getDecoder().decode(RetentionTestSupport.KEY)); + reachable.add(keys.keyForRecordField("reading", "systolic", control.getValue("id"), false).orElseThrow().key()); + reachable.add(new byte[32]); + assertThat(reachable.get(2)).isNotEqualTo(new byte[32]); + + Map> after = auditTokens(truth.keySet()); + int matches = 0; + for(var entry : truth.entrySet()) + { + assertThat(after.get(entry.getKey())).containsAll(tokens.get(entry.getKey())); + for(String token : after.get(entry.getKey())) + { + String tail = token.substring(token.lastIndexOf(':') + 1); + for(int guess = 60; guess <= 250; guess++) + { + if(ClassifiedValueTokens.verifyRecordField(table, table.getField("systolic"), entry.getKey(), guess, token)) + { + matches++; + } + for(byte[] key : reachable) + { + if(digest(key, table, "systolic", entry.getKey(), guess).equals(tail)) + { + matches++; + } + } + } + } + } + assertThat(matches).as("brute-force matches after the wipe").isZero(); + assertThat(rows("auditDetail")).anySatisfy(row -> assertThat(row.getValueString("newValue")).startsWith("shredded:v1:")); + } + + + + /******************************************************************************* + ** Every systolic audit-detail value for the given records, by record id. + *******************************************************************************/ + private static Map> auditTokens(Iterable ids) + { + Map recordByAudit = new LinkedHashMap<>(); + for(QRecord audit : SystemWrites.query("audit", new QQueryFilter())) + { + recordByAudit.put(audit.getValue("id"), audit.getValueString("recordId")); + } + Map> result = new LinkedHashMap<>(); + for(Serializable id : ids) + { + result.put(id, new ArrayList<>()); + } + for(QRecord detail : SystemWrites.query("auditDetail", new QQueryFilter())) + { + if(!"systolic".equals(detail.getValueString("fieldName"))) + { + continue; + } + String record = recordByAudit.get(detail.getValue("auditId")); + for(Serializable id : ids) + { + if(String.valueOf(id).equals(record)) + { + for(String value : new String[] {detail.getValueString("oldValue"), detail.getValueString("newValue")}) + { + if(value != null && !value.isBlank()) + { + result.get(id).add(value); + } + } + } + } + } + return (result); + } +} diff --git a/src/test/java/com/kingsrook/qbits/secureddata/RetentionTestSupport.java b/src/test/java/com/kingsrook/qbits/secureddata/RetentionTestSupport.java new file mode 100644 index 0000000..d5cf045 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/secureddata/RetentionTestSupport.java @@ -0,0 +1,147 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.secureddata; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.time.Instant; +import java.util.List; +import java.util.UUID; +import com.kingsrook.qbits.secureddata.metadata.SecuredDataTables; +import com.kingsrook.qbits.secureddata.support.SystemWrites; +import com.kingsrook.qqq.backend.core.actions.ai.AiFieldMetaData; +import com.kingsrook.qqq.backend.core.actions.ai.DataClassification; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.model.audits.AuditsMetaDataProvider; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel; +import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails; + +final class RetentionTestSupport +{ + static String BACKEND = "retention_rdbms"; + static final String KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8="; + static final Instant OLD = SecuredDataTestSupport.NOW.minusSeconds(60L * 86400); + static QInstance setup() throws Exception + { + SecuredDataTestSupport.close(); + System.setProperty("QQQ_SECURED_DATA_KEK", KEY); + System.setProperty("QQQ_AUDIT_HMAC_KEY", KEY); + SecuredDataTestSupport.CLOCK.now = SecuredDataTestSupport.NOW; + String db = "retention_" + UUID.randomUUID().toString().replace("-", ""); + BACKEND = db; // pools are cached by backend name, so each instance needs its own + String url = "jdbc:h2:mem:" + db + ";MODE=PostgreSQL;DATABASE_TO_UPPER=FALSE;DB_CLOSE_DELAY=-1"; + RDBMSBackendMetaData backend = new RDBMSBackendMetaData().withName(BACKEND).withVendor("h2").withHostName("mem").withDatabaseName(db).withUsername("sa"); + backend.setJdbcUrl(url); + QInstance instance = SecuredDataTestSupport.base(); + instance.addBackend(backend); + QTableMetaData table = new QTableMetaData().withName("reading").withLabel("Reading").withBackendName(BACKEND).withPrimaryKeyField("id") + .withBackendDetails(new RDBMSTableBackendDetails().withTableName("reading")) + .withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_WRITE_PERMISSIONS)) + .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD)) + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("reading_time", QFieldType.DATE_TIME)) + .withField(new QFieldMetaData("reading_date", QFieldType.DATE)) + .withField(new QFieldMetaData("systolic", QFieldType.INTEGER).withSupplementalMetaData(new AiFieldMetaData().withClassification(DataClassification.PHI))) + .withField(new QFieldMetaData("secret", QFieldType.STRING).withMaxLength(255).withBehavior(com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior.ERROR).withSupplementalMetaData(new AiFieldMetaData().withClassification(DataClassification.PHI))) + .withField(new QFieldMetaData("notes", QFieldType.TEXT).withSupplementalMetaData(new AiFieldMetaData().withClassification(DataClassification.PII))); + table.getFields().values().forEach(f -> f.setBackendName(f.getName())); + instance.addTable(table); + new AuditsMetaDataProvider().defineAll(instance, BACKEND, t -> t.withBackendDetails(new RDBMSTableBackendDetails().withTableName(t.getName()))); + try(Connection connection = DriverManager.getConnection(url, "sa", "")) + { + for(QTableMetaData t : instance.getTables().values()) + { + StringBuilder sql = new StringBuilder("CREATE TABLE ").append(t.getName()).append(" ("); + for(QFieldMetaData field : t.getFields().values()) + { + field.setBackendName(field.getName()); + sql.append(field.getName()).append(" "); + if(field.getName().equals(t.getPrimaryKeyField())) + { + sql.append("INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY"); + } + else + { + sql.append(switch(field.getType()) + { + case INTEGER -> "INTEGER"; + case LONG -> "BIGINT"; + case DATE_TIME -> "TIMESTAMP WITH TIME ZONE"; + case DATE -> "DATE"; + case BOOLEAN -> "BOOLEAN"; + case DECIMAL -> "DECIMAL"; + default -> "TEXT"; + }); + } + sql.append(","); + } + sql.setLength(sql.length() - 1); + connection.createStatement().execute(sql.append(")").toString()); + } + for(String sql : Files.readString(Path.of("docs/ddl/secured-data-postgres.sql")).split(";")) + { + if(!sql.isBlank()) connection.createStatement().execute(sql); + } + } + new SecuredDataQBitProducer().withQBitConfig(new SecuredDataQBitConfig().withBackendName(BACKEND).withClock(SecuredDataTestSupport.CLOCK) + .withBackupRetentionDays(35).withAccessLogFlushIntervalMillis(60000) + .withTableMetaDataCustomizer((i,t) -> t.withBackendDetails(new RDBMSTableBackendDetails().withTableName(t.getName())))).produce(instance); + QContext.init(instance, SecuredDataTestSupport.user("admin", "secured_data.admin", "secured_data.retention", "reading.read", "reading.write")); + SecuredDataTestSupport.RUNTIMES.add(SecuredDataRuntime.current()); + new QInstanceValidator().validate(instance); + return instance; + } + static QRecord rule(String action, String fields) throws Exception + { + return SystemWrites.insertOne(SecuredDataTables.RETENTION_RULE, new QRecord().withValue("name", "30-day policy").withValue("table_name", "reading") + .withValue("field_names", fields).withValue("age_field_name", "reading_time").withValue("age_days", 30).withValue("action", action) + .withValue("archive_storage_name", "archive")); + } + static QRecord reading(Object value) throws Exception + { + return SystemWrites.insertOne("reading", new QRecord().withValue("systolic", (java.io.Serializable)value).withValue("secret", "test-secret") + .withValue("notes", "test-notes").withValue("reading_time", OLD).withValue("reading_date", OLD.atZone(java.time.ZoneOffset.UTC).toLocalDate())); + } + static List rows(String table) + { + return SystemWrites.query(table, new QQueryFilter()); + } + static void close() + { + SecuredDataTestSupport.close(); + System.clearProperty("QQQ_SECURED_DATA_KEK"); + System.clearProperty("QQQ_AUDIT_HMAC_KEY"); + System.clearProperty("QQQ_AUDIT_HMAC_KEY_ID"); + } +} diff --git a/src/test/java/com/kingsrook/qbits/secureddata/RetentionWipeTest.java b/src/test/java/com/kingsrook/qbits/secureddata/RetentionWipeTest.java new file mode 100644 index 0000000..fc5437b --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/secureddata/RetentionWipeTest.java @@ -0,0 +1,265 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.secureddata; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Base64; +import java.util.Set; +import java.nio.charset.StandardCharsets; +import com.kingsrook.qbits.secureddata.metadata.SecuredDataTables; +import com.kingsrook.qbits.secureddata.retention.RetentionRules; +import com.kingsrook.qbits.secureddata.retention.RetentionWidgets; +import com.kingsrook.qbits.secureddata.retention.WipeService; +import com.kingsrook.qbits.secureddata.retention.WipeProcesses; +import com.kingsrook.qbits.secureddata.standin.DMLType; +import com.kingsrook.qbits.secureddata.standin.ClassifiedValueTokens; +import com.kingsrook.qbits.secureddata.support.SystemWrites; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import static com.kingsrook.qbits.secureddata.RetentionTestSupport.*; +import static org.assertj.core.api.Assertions.*; + +class RetentionWipeTest +{ + @AfterEach void cleanup() { close(); } + + @ParameterizedTest + @ValueSource(strings={"NULL_WITH_MARKER", "OVERWRITE_TEXT", "KEYED_HASH", "DELETE_RECORD", "ARCHIVE_THEN_WIPE"}) + void eachActionIsTransactionalAndValueFree(String action) throws Exception + { + var instance = setup(); + List archiveJson = new ArrayList<>(); + SecuredDataRuntime.current().config().withArchiveSink((store, key, bytes) -> { + assertThat(store).isEqualTo("archive"); + assertThat(new String(bytes, StandardCharsets.UTF_8)).doesNotContain("test-secret"); + archiveJson.add(new String(SecuredDataKeySource.decrypt(bytes, key), StandardCharsets.UTF_8)); + }); + QRecord rule = rule(action, "secret"); + QRecord record = reading(120); + var keys = SecuredDataRuntime.current().keys(); + keys.keyForRecordField("reading", "secret", record.getValue("id"), true); + assertThat(rows(SecuredDataTables.AUDIT_KEY)).isNotEmpty(); + QRecord run = new WipeService().batch(rule.getValue("id"), List.of(record.getValue("id"))); + assertThat(run.getValueInteger("wiped_count")).isEqualTo(1); + assertThat(rows(SecuredDataTables.AUDIT_KEY)).isEmpty(); + assertThat(keys.keyForRecordField("reading", "secret", record.getValue("id"), false)).isEmpty(); + if(action.equals("DELETE_RECORD")) assertThat(rows("reading")).isEmpty(); + else { + QRecord remaining = rows("reading").getFirst(); + if(action.equals("OVERWRITE_TEXT")) assertThat(remaining.getValue("secret")).isEqualTo("[wiped per policy]"); + else if(action.equals("KEYED_HASH")) { + assertThat(remaining.getValueString("secret")).startsWith("hmac:v1:"); + assertThat(ClassifiedValueTokens.verifyRecordField(instance.getTable("reading"), instance.getTable("reading").getField("secret"), record.getValue("id"), "test-secret", remaining.getValueString("secret"))).isTrue(); + } + else assertThat(remaining.getValue("secret")).isNull(); + } + if(action.equals("ARCHIVE_THEN_WIPE")) assertThat(archiveJson).containsExactly("{\"secret\":\"test-secret\"}"); + else assertThat(archiveJson).isEmpty(); + assertThat(rows(SecuredDataTables.WIPE_MARKER)).isNotEmpty(); + assertThat(rows(SecuredDataTables.ACCESS_LOG).stream().filter(r -> "WIPE".equals(r.getValueString("event_type"))).toList()).isNotEmpty() + .allSatisfy(r -> { assertThat(r.getValueString("record_id")).isEqualTo(record.getValueString("id")); assertThat(r.getValueInteger("record_count")).isEqualTo(1); }); + assertThat(rows("auditDetail")).anySatisfy(r -> assertThat(r.getValueString("newValue")).isEqualTo("shredded:v1:" + rule.getValueString("id"))); + } + + @Test void recordAndTableLegalHoldsAndAgeAreRechecked() throws Exception + { + setup(); + QRecord rule = rule("NULL_WITH_MARKER", "systolic"); + QRecord one = reading(120), two = reading(130); + QRecord hold = SystemWrites.insertOne(SecuredDataTables.LEGAL_HOLD, new QRecord().withValue("table_name", "reading").withValue("record_id", one.getValueString("id")).withValue("reason", "Preservation")); + QRecord run = new WipeService().batch(rule.getValue("id"), List.of(one.getValue("id"), two.getValue("id"))); + assertThat(run.getValueInteger("held_count")).isEqualTo(1); + assertThat(run.getValueInteger("wiped_count")).isEqualTo(1); + SecuredDataTestSupport.update(SecuredDataTables.LEGAL_HOLD, new QRecord().withValue("id", hold.getValue("id")).withValue("record_id", null)); + assertThat(new WipeService().batch(rule.getValue("id"), List.of(one.getValue("id"))).getValueInteger("held_count")).isEqualTo(1); + SecuredDataTestSupport.update(SecuredDataTables.LEGAL_HOLD, new QRecord().withValue("id", hold.getValue("id")).withValue("status", "RELEASED")); + assertThat(new WipeService().batch(rule.getValue("id"), List.of(one.getValue("id"))).getValueInteger("wiped_count")).isEqualTo(1); + QRecord young = reading(140); + SecuredDataTestSupport.update("reading", new QRecord().withValue("id", young.getValue("id")).withValue("reading_time", SecuredDataTestSupport.NOW)); + assertThat(new WipeService().batch(rule.getValue("id"), List.of(young.getValue("id"))).getValueInteger("eligible_count")).isZero(); + } + + @Test void failureAfterFirstRecordRollsBackUpdatesKeysMarkersAuditsAndRun() throws Exception + { + var instance = setup(); + QRecord rule = rule("NULL_WITH_MARKER", "secret"); + QRecord first = reading(120), second = reading(130); + for(QRecord row : List.of(first, second)) SecuredDataRuntime.current().keys().keyForRecordField("reading", "secret", row.getValue("id"), true); + int audits = rows("auditDetail").size(); + FailSecond.id = second.getValue("id"); + instance.getTable("reading").withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, new QCodeReference(FailSecond.class)); + assertThatThrownBy(() -> new WipeService().batch(rule.getValue("id"), List.of(first.getValue("id"), second.getValue("id")))).hasMessageContaining("rolled back"); + assertThat(rows("reading")).allSatisfy(r -> assertThat(r.getValue("secret")).isEqualTo("test-secret")); + assertThat(rows(SecuredDataTables.AUDIT_KEY)).hasSize(2); + assertThat(rows(SecuredDataTables.WIPE_MARKER)).isEmpty(); + assertThat(rows(SecuredDataTables.WIPE_RUN)).isEmpty(); + assertThat(rows("auditDetail")).hasSize(audits); + } + + public static class FailSecond implements TableCustomizerInterface + { + static java.io.Serializable id; + @Override public List preInsertOrUpdate(AbstractActionInput input, List records, boolean preview, Optional> old) throws QException + { + if(records.stream().anyMatch(r -> id.equals(r.getValue("id")))) throw new QException("Injected failure"); + return records; + } + } + + @Test void reviewResultAndDailySchedule() throws Exception + { + setup(); + QRecord rule = rule("NULL_WITH_MARKER", "systolic"); + reading(120); + QRecord held = reading(130); + SystemWrites.insertOne(SecuredDataTables.LEGAL_HOLD, new QRecord().withValue("table_name", "reading").withValue("record_id", held.getValueString("id")).withValue("reason", "Preserve")); + var lines = ProcessRunner.run(WipeProcesses.WIPE, rule.getValue("id")); + assertThat(lines.review()).anyMatch(s -> s.contains("to wipe")).anyMatch(s -> s.contains("held")); + assertThat(lines.result()).anyMatch(s -> s.contains("wiped")).anyMatch(s -> s.contains("held")); + reading(150); + assertThatThrownBy(() -> new WipeProcesses.Daily().run(new RunBackendStepInput(), new RunBackendStepOutput())).hasMessageContaining("system session"); + QContext.setQSession(new QSystemUserSession(QContext.getQInstance())); + var output = new RunBackendStepOutput(); + new WipeProcesses.Daily().run(new RunBackendStepInput(), output); + assertThat(output.getRecords()).anySatisfy(r -> assertThat(r.getValueInteger("wiped_count")).isEqualTo(1)); + var scheduled = WipeProcesses.processes(new SecuredDataQBitConfig().withSchedulerName("test")).getLast(); + assertThat(scheduled.getSchedule().getRepeatSeconds()).isEqualTo(86400); + } + + @Test void permissionsKeyLockAndBackupsPanels() throws Exception + { + var instance = setup(); + QRecord rule = rule("NULL_WITH_MARKER", "secret"), record = reading(120); + SecuredDataRuntime.current().keys().keyForRecordField("reading", "secret", record.getValue("id"), true); + var keyTable = instance.getTable(SecuredDataTables.AUDIT_KEY); + assertThat(keyTable.getField("wrapped_key").getFieldSecurityLock().getOverrideValues()).containsExactly(com.kingsrook.qbits.secureddata.metadata.RetentionTables.KEY_LOCK_NEVER_GRANTED); + assertThat(keyTable.getDisabledCapabilities()).contains(com.kingsrook.qqq.backend.core.model.metadata.tables.Capability.TABLE_EXPORT); + assertThat(SecuredDataTables.MAIN).doesNotContain(SecuredDataTables.AUDIT_KEY); + assertThat(new QueryAction().execute(new QueryInput(SecuredDataTables.AUDIT_KEY)).getRecords()).allSatisfy(r -> assertThat(r.getValue("wrapped_key")).isNull()); + assertThat(RetentionRules.backupWarning(rule)).contains("35 days", "30 days", "recoverable"); + assertThat(com.kingsrook.qbits.secureddata.app.SecuredDataHomeView.render(com.kingsrook.qbits.secureddata.app.SecuredDataHomeData.load())).contains("Backups keep data for 35 days"); + RenderWidgetInput input = new RenderWidgetInput().withWidgetMetaData(RetentionWidgets.backups()); + input.getQueryParams().put("id", rule.getValueString("id")); + assertThat(new RetentionWidgets.Backups().render(input)).isNotNull(); + new WipeService().batch(rule.getValue("id"), List.of(record.getValue("id"))); + var widget = RetentionWidgets.wipedFields(instance.getTable("reading")); + input = new RenderWidgetInput().withWidgetMetaData(widget); input.getQueryParams().put("id", record.getValueString("id")); + assertThat(new RetentionWidgets.WipedFields().render(input)).isNotNull(); + QContext.setQSession(SecuredDataTestSupport.user("ordinary")); + assertThatThrownBy(() -> new WipeService().batch(rule.getValue("id"), List.of(record.getValue("id")))).hasMessageContaining("secured_data.retention"); + } + + @Test void changedOnlyAndWrapUnwrap() throws Exception + { + var instance = setup(); + QRecord rule = rule("NULL_WITH_MARKER", "secret"); + SecuredDataTestSupport.update(SecuredDataTables.RETENTION_RULE, new QRecord().withValue("id", rule.getValue("id")).withValue("audit_treatment", "CHANGED_ONLY")); + var table = instance.getTable("reading"); + var protectedValues = new SecuredDataAuditProtector().protect(table, table.getField("secret"), 1, "old", "new", DMLType.UPDATE); + assertThat(protectedValues.oldToken()).isEqualTo("changed:v1"); assertThat(protectedValues.newToken()).isEqualTo("changed:v1"); + assertThat(rows(SecuredDataTables.AUDIT_KEY)).isEmpty(); + byte[] plain = Base64.getDecoder().decode(KEY); + byte[] encrypted = SecuredDataKeySource.encrypt(plain, "binding"); + assertThat(SecuredDataKeySource.decrypt(encrypted, "binding")).isEqualTo(plain); + assertThatThrownBy(() -> SecuredDataKeySource.decrypt(encrypted, "wrong-binding")).isInstanceOf(IllegalStateException.class); + encrypted[15] ^= 1; + assertThatThrownBy(() -> SecuredDataKeySource.decrypt(encrypted, "binding")).isInstanceOf(IllegalStateException.class); + System.setProperty("QQQ_SECURED_DATA_KEK", "bad"); + assertThatThrownBy(() -> SecuredDataKeySource.encrypt(plain, "binding")).hasMessageContaining("QQQ_SECURED_DATA_KEK"); + } + + @Test void validatesPolicyAndRefusesMissingArchive() throws Exception + { + setup(); + assertThatThrownBy(() -> rule("KEYED_HASH", "systolic")).hasMessageContaining("insert failed"); + assertThatThrownBy(() -> rule("NULL_WITH_MARKER", "missing")).hasMessageContaining("insert failed"); + assertThatThrownBy(() -> rule("NULL_WITH_MARKER", "secret,secret")).hasMessageContaining("insert failed"); + var rule = rule("ARCHIVE_THEN_WIPE", "secret"); var record = reading(120); + assertThatThrownBy(() -> new WipeService().batch(rule.getValue("id"), List.of(record.getValue("id")))).hasMessageContaining("ArchiveSinkInterface"); + assertThat(rows("reading").getFirst().getValue("secret")).isEqualTo("test-secret"); + assertThatThrownBy(() -> new SecuredDataQBitConfig().withBackupRetentionDays(-1)).isInstanceOf(IllegalArgumentException.class); + } + + @Test void wrongKekCannotUnwrapAndKeyReturnedIsNeverZeroed() throws Exception + { + setup(); + rule("NULL_WITH_MARKER", "secret"); + QRecord record = reading(120); + var keys = SecuredDataRuntime.current().keys(); + Object pk = record.getValue("id"); + String binding = "readingsecret" + pk; + + // Regression: the source once returned the array it then zeroed in finally. + byte[] created = keys.keyForRecordField("reading", "secret", (java.io.Serializable) pk, true).orElseThrow().key(); + assertThat(created).hasSize(32).isNotEqualTo(new byte[32]); + QRecord stored = SystemWrites.call(() -> new QueryAction().execute(new QueryInput(SecuredDataTables.AUDIT_KEY) + .withFilter(SecuredDataKeySource.recordField("reading", "secret", (java.io.Serializable) pk)).withShouldOmitHiddenFields(false)).getRecords()).getFirst(); + byte[] wrapped = Base64.getDecoder().decode(stored.getValueString("wrapped_key")); + assertThat(created).isEqualTo(SecuredDataKeySource.decrypt(wrapped, binding)); + + // Lookups (uncached then cached) agree with the unwrapped DEK, and callers zeroing their copy cannot poison the cache. + byte[] looked = keys.keyForRecordField("reading", "secret", (java.io.Serializable) pk, false).orElseThrow().key(); + assertThat(looked).isNotEqualTo(new byte[32]).isEqualTo(created); + java.util.Arrays.fill(looked, (byte) 0); + byte[] cached = keys.keyForRecordField("reading", "secret", (java.io.Serializable) pk, false).orElseThrow().key(); + assertThat(cached).isEqualTo(created).isNotEqualTo(new byte[32]); + + // A different KEK, or a different binding, cannot unwrap. + System.setProperty("QQQ_SECURED_DATA_KEK", "/v79vLu6ubi3trW0s7KxsK+uraypqKempaSjoqGgn56dnJuamZiXlpU="); + assertThatThrownBy(() -> SecuredDataKeySource.decrypt(wrapped, binding)).isInstanceOf(IllegalStateException.class); + System.setProperty("QQQ_SECURED_DATA_KEK", KEY); + assertThatThrownBy(() -> SecuredDataKeySource.decrypt(wrapped, binding + "x")).isInstanceOf(IllegalStateException.class); + assertThat(SecuredDataKeySource.decrypt(wrapped, binding)).isEqualTo(created); + } + + @Test void keyedHashIsRefusedOnLowEntropyFields() throws Exception + { + setup(); + assertThat(rule("KEYED_HASH", "secret").getValue("id")).isNotNull(); + SystemWrites.insertOne(SecuredDataTables.CLASSIFICATION, new QRecord().withValue("table_name", "reading").withValue("field_name", "secret") + .withValue("classification", "PHI").withValue("low_cardinality", true)); + assertThatThrownBy(() -> rule("KEYED_HASH", "secret")).hasMessageContaining("insert failed"); + assertThatThrownBy(() -> RetentionRules.validate(new QRecord().withValue("table_name", "reading").withValue("field_names", "secret") + .withValue("age_field_name", "reading_time").withValue("age_days", 30).withValue("batch_size", 10).withValue("action", "KEYED_HASH") + .withValue("audit_treatment", "SHREDDABLE"))).hasMessageContaining("low-entropy"); + assertThat(rule("OVERWRITE_TEXT", "secret").getValue("id")).isNotNull(); + } +} diff --git a/src/test/java/com/kingsrook/qbits/secureddata/RevealAndBreakGlassTest.java b/src/test/java/com/kingsrook/qbits/secureddata/RevealAndBreakGlassTest.java index 9ab3520..2da5f8f 100644 --- a/src/test/java/com/kingsrook/qbits/secureddata/RevealAndBreakGlassTest.java +++ b/src/test/java/com/kingsrook/qbits/secureddata/RevealAndBreakGlassTest.java @@ -31,7 +31,7 @@ import com.kingsrook.qbits.secureddata.metadata.SecuredDataTables; import com.kingsrook.qbits.secureddata.processes.BreakGlassProcesses; import com.kingsrook.qbits.secureddata.reveal.RevealCappedException; import com.kingsrook.qbits.secureddata.reveal.RevealService; -import com.kingsrook.qbits.secureddata.standin.DataClassification; +import com.kingsrook.qqq.backend.core.actions.ai.DataClassification; import com.kingsrook.qbits.secureddata.standin.FieldAccessTier; import com.kingsrook.qbits.secureddata.support.SystemWrites; import com.kingsrook.qqq.backend.core.actions.tables.GetAction; diff --git a/src/test/java/com/kingsrook/qbits/secureddata/SecuredDataAccessLoggerTest.java b/src/test/java/com/kingsrook/qbits/secureddata/SecuredDataAccessLoggerTest.java index 08023cb..96ac9a2 100644 --- a/src/test/java/com/kingsrook/qbits/secureddata/SecuredDataAccessLoggerTest.java +++ b/src/test/java/com/kingsrook/qbits/secureddata/SecuredDataAccessLoggerTest.java @@ -32,7 +32,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.IntStream; import com.kingsrook.qbits.secureddata.metadata.SecuredDataTables; -import com.kingsrook.qbits.secureddata.standin.DataClassification; +import com.kingsrook.qqq.backend.core.actions.ai.DataClassification; import com.kingsrook.qbits.secureddata.standin.FieldAccessEvent; import com.kingsrook.qbits.secureddata.standin.FieldAccessEventType; import com.kingsrook.qbits.secureddata.standin.FieldAccessOutcome; diff --git a/src/test/java/com/kingsrook/qbits/secureddata/SecuredDataHomeTest.java b/src/test/java/com/kingsrook/qbits/secureddata/SecuredDataHomeTest.java index 553225e..c43793d 100644 --- a/src/test/java/com/kingsrook/qbits/secureddata/SecuredDataHomeTest.java +++ b/src/test/java/com/kingsrook/qbits/secureddata/SecuredDataHomeTest.java @@ -57,7 +57,7 @@ class SecuredDataHomeTest var app = instance.getApp("secured_data"); assertThat(app.getWidgets()).containsExactly(SecuredDataApp.HOME_WIDGET); assertThat(app.getChildren()).extracting(c -> c.getName()).containsExactly(SecuredDataTables.CLASSIFICATION, SecuredDataTables.POLICY, - SecuredDataTables.ACCESS_LOG, SecuredDataTables.BREAK_GLASS); + SecuredDataTables.ACCESS_LOG, SecuredDataTables.BREAK_GLASS, SecuredDataTables.RETENTION_RULE); assertThat(instance.getTable(SecuredDataTables.ACCESS_LOG).getSections()).anyMatch(s -> SecuredDataTables.REVEALS_WIDGET.equals(s.getWidgetName())); } diff --git a/src/test/java-uiapi/com/kingsrook/qbits/secureddata/SecuredDataHttpTest.java b/src/test/java/com/kingsrook/qbits/secureddata/SecuredDataHttpTest.java similarity index 100% rename from src/test/java-uiapi/com/kingsrook/qbits/secureddata/SecuredDataHttpTest.java rename to src/test/java/com/kingsrook/qbits/secureddata/SecuredDataHttpTest.java diff --git a/src/test/java/com/kingsrook/qbits/secureddata/SecuredDataQBitProducerTest.java b/src/test/java/com/kingsrook/qbits/secureddata/SecuredDataQBitProducerTest.java index 0f6e118..c3f3eac 100644 --- a/src/test/java/com/kingsrook/qbits/secureddata/SecuredDataQBitProducerTest.java +++ b/src/test/java/com/kingsrook/qbits/secureddata/SecuredDataQBitProducerTest.java @@ -26,11 +26,11 @@ import java.util.Map; import com.kingsrook.qbits.secureddata.metadata.FieldsOfTablePossibleValueSource; import com.kingsrook.qbits.secureddata.metadata.SecuredDataEnums; import com.kingsrook.qbits.secureddata.metadata.SecuredDataTables; -import com.kingsrook.qbits.secureddata.standin.ClientChannelsMetaData; -import com.kingsrook.qbits.secureddata.standin.DataClassification; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.ClientChannelsMetaData; +import com.kingsrook.qqq.backend.core.actions.ai.DataClassification; import com.kingsrook.qbits.secureddata.standin.FieldAccessTier; -import com.kingsrook.qbits.secureddata.standin.TableFeaturePermissionsMetaData; -import com.kingsrook.qbits.secureddata.support.CoreMetadataBridge; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.TableFeaturePermissionsMetaData; +import com.kingsrook.qbits.secureddata.support.SecuredDataMetadata; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.context.QContext; @@ -93,19 +93,19 @@ class SecuredDataQBitProducerTest for(String name : SecuredDataTables.ALL) { QTableMetaData table = instance.getTable(name); - assertThat(table.getAuditRules().getAuditLevel()).isEqualTo(AuditLevel.FIELD); + assertThat(table.getAuditRules().getAuditLevel()).isEqualTo(name.equals(SecuredDataTables.AUDIT_KEY) ? AuditLevel.NONE : AuditLevel.FIELD); assertThat(table.getSections()).isNotEmpty(); - assertThat(CoreMetadataBridge.classification(table, null)).isEqualTo(DataClassification.CONFIDENTIAL); + assertThat(SecuredDataMetadata.classification(table, null)).isEqualTo(DataClassification.CONFIDENTIAL); table.getFields().values().forEach(field -> assertThat(field.getBackendName()).isEqualTo(field.getName())); } assertThat(SecuredDataTables.esbTableNames()).containsExactlyElementsOf(SecuredDataTables.MAIN); assertThat(SecuredDataEnums.sources()).hasSize(6); - assertThat(ClientChannelsMetaData.ofOrWithNew(instance).getEntries()).containsEntry("assistant", "secured_data_*"); - assertThat(TableFeaturePermissionsMetaData.ofOrWithNew(instance).getEntries()).containsKeys("secured_data_full", "secured_data_reveal"); + assertThat(ClientChannelsMetaData.ofOrWithNew(instance).getDeniedPatterns().get("assistant")).contains("secured_data_*"); + assertThat(TableFeaturePermissionsMetaData.ofOrWithNew(instance).getPermissions()).extracting("suffix").contains("secured_data_full", "secured_data_reveal"); assertThat(instance.getTable(SecuredDataTables.ACCESS_LOG).getDisabledCapabilities()).contains(Capability.TABLE_UPDATE, Capability.TABLE_DELETE); assertThat(instance.getJoin(SecuredDataTables.REVEALS_JOIN)).isNotNull(); assertThat(instance.getWidget(SecuredDataTables.REVEALS_WIDGET)).isNotNull(); - assertThat(instance.getApp("secured_data").getChildren()).hasSize(4); + assertThat(instance.getApp("secured_data").getChildren()).hasSize(5); } /******************************************************************************* diff --git a/src/test/java/com/kingsrook/qbits/secureddata/SecuredDataResolverTest.java b/src/test/java/com/kingsrook/qbits/secureddata/SecuredDataResolverTest.java index 172ac75..c9910c5 100644 --- a/src/test/java/com/kingsrook/qbits/secureddata/SecuredDataResolverTest.java +++ b/src/test/java/com/kingsrook/qbits/secureddata/SecuredDataResolverTest.java @@ -23,11 +23,11 @@ package com.kingsrook.qbits.secureddata; import java.util.List; import com.kingsrook.qbits.secureddata.metadata.SecuredDataTables; -import com.kingsrook.qbits.secureddata.standin.AiTableMetaData; -import com.kingsrook.qbits.secureddata.standin.DataClassification; +import com.kingsrook.qqq.backend.core.actions.ai.AiTableMetaData; +import com.kingsrook.qqq.backend.core.actions.ai.DataClassification; import com.kingsrook.qbits.secureddata.standin.FieldAccessTier; import com.kingsrook.qbits.secureddata.standin.FieldMaskStyle; -import com.kingsrook.qbits.secureddata.support.CoreMetadataBridge; +import com.kingsrook.qbits.secureddata.support.SecuredDataMetadata; import com.kingsrook.qbits.secureddata.support.SystemWrites; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -271,7 +271,7 @@ class SecuredDataResolverTest { QFieldMetaData field = new QFieldMetaData(name, QFieldType.valueOf(type)).withMaxLength(255); person.addField(field); - CoreMetadataBridge.classifyField(field, DataClassification.PII); + SecuredDataMetadata.classifyField(field, DataClassification.PII); assertThat(resolver.resolve(person, field).exactMatchAllowed()).isFalse(); QRecord row = insert(SecuredDataTables.CLASSIFICATION, new QRecord().withValue("table_name", "person").withValue("field_name", name) .withValue("classification", "PII").withValue("low_cardinality", false).withValue("mask_style", "FULL")); diff --git a/src/test/java/com/kingsrook/qbits/secureddata/SecuredDataTestSupport.java b/src/test/java/com/kingsrook/qbits/secureddata/SecuredDataTestSupport.java index c60cd9f..3d87045 100644 --- a/src/test/java/com/kingsrook/qbits/secureddata/SecuredDataTestSupport.java +++ b/src/test/java/com/kingsrook/qbits/secureddata/SecuredDataTestSupport.java @@ -30,9 +30,8 @@ import java.util.Arrays; import java.util.List; import java.util.function.Consumer; import com.kingsrook.qbits.secureddata.metadata.SecuredDataTables; -import com.kingsrook.qbits.secureddata.standin.ClientChannelSession; -import com.kingsrook.qbits.secureddata.standin.DataClassification; -import com.kingsrook.qbits.secureddata.support.CoreMetadataBridge; +import com.kingsrook.qqq.backend.core.actions.ai.DataClassification; +import com.kingsrook.qbits.secureddata.support.SecuredDataMetadata; import com.kingsrook.qbits.secureddata.support.SystemWrites; import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; @@ -124,8 +123,8 @@ final class SecuredDataTestSupport .withField(new QFieldMetaData("email", QFieldType.STRING).withMaxLength(255)) .withField(new QFieldMetaData("ssn", QFieldType.STRING).withMaxLength(255)) .withField(new QFieldMetaData("notes", QFieldType.TEXT)); - CoreMetadataBridge.classifyField(table.getField("email"), DataClassification.PII); - CoreMetadataBridge.classifyField(table.getField("ssn"), DataClassification.PHI); + SecuredDataMetadata.classifyField(table.getField("email"), DataClassification.PII); + SecuredDataMetadata.classifyField(table.getField("ssn"), DataClassification.PHI); table.getFields().values().forEach(field -> field.setBackendName(field.getName())); return (table); } @@ -149,16 +148,7 @@ final class SecuredDataTestSupport *******************************************************************************/ static QSession channelUser(String channel, String... permissions) throws Exception { - QSession session; - try - { - session = new QSession(); - QSession.class.getMethod("withClientChannel", String.class).invoke(session, channel); - } - catch(NoSuchMethodException expected) - { - session = new ClientChannelSession().withClientChannel(channel); - } + QSession session = new QSession().withClientChannel(channel); List grants = new ArrayList<>(Arrays.asList(permissions)); grants.add("person.read"); return (session.withUser(new QUser().withIdReference("u1").withFullName("User One")).withPermissions(grants)); diff --git a/src/test/java-uiapi/com/kingsrook/qbits/secureddata/UiApiContractAssertions.java b/src/test/java/com/kingsrook/qbits/secureddata/SecuredDataUiApiContractTest.java similarity index 71% rename from src/test/java-uiapi/com/kingsrook/qbits/secureddata/UiApiContractAssertions.java rename to src/test/java/com/kingsrook/qbits/secureddata/SecuredDataUiApiContractTest.java index 47683c8..040bab1 100644 --- a/src/test/java-uiapi/com/kingsrook/qbits/secureddata/UiApiContractAssertions.java +++ b/src/test/java/com/kingsrook/qbits/secureddata/SecuredDataUiApiContractTest.java @@ -20,24 +20,25 @@ package com.kingsrook.qbits.secureddata; -import java.nio.file.Files; import java.nio.file.Path; -import com.kingsrook.qqq.middleware.javalin.specs.extensions.MiddlewareSpecExtension; +import java.nio.file.Files; import com.kingsrook.qqq.middleware.javalin.specs.v1.MiddlewareVersionV1; import com.kingsrook.qqq.middleware.javalin.tools.UiApiContract; import static org.assertj.core.api.Assertions.assertThat; +import com.kingsrook.qbits.secureddata.uiapi.SecuredDataExtension; +import org.junit.jupiter.api.Test; -/** STAND-IN(core=QRun-IO/qqq#1128, removeBy=RC.6): supplied worktree exposes UiApiContract, not the documented assertion facade. */ -final class UiApiContractAssertions +class SecuredDataUiApiContractTest { - static void assertExtensionMatchesCommitted(MiddlewareSpecExtension extension, Path path) throws Exception + @Test + void committedContractMatches() throws Exception { - String generated = UiApiContract.generateExtensionYaml(new MiddlewareVersionV1(), extension); + Path path = Path.of("src/main/resources/openapi/qqq-v1/secured-data.yaml"); + String generated = UiApiContract.generateExtensionYaml(new MiddlewareVersionV1(), SecuredDataExtension.defineExtension()); if(Boolean.getBoolean("qqq.uiApi.regenerate")) { - Files.createDirectories(path.getParent()); Files.writeString(path, generated); } - assertThat(UiApiContract.check(generated, Files.readString(path), null, 1, "Regenerate secured-data.yaml with -Dqqq.uiApi.regenerate=true")).isEmpty(); + assertThat(UiApiContract.check(generated, Files.readString(path), null, 1, "Regenerate secured-data.yaml")).isEmpty(); } } -- 2.54.0