Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
571f14ff82
|
||
|
|
0ce28313f9 | ||
|
|
a8980eec0f | ||
|
|
082f3b95f0 | ||
|
|
c681b2c0e7 | ||
|
|
558effe85f
|
||
|
|
93f6f86f2d | ||
|
|
7e15fb92e0 | ||
|
|
a5d7192b95
|
||
|
|
05fcbdd8b8
|
||
|
|
d78ecc1bc4 | ||
|
|
e81c8ba37e
|
||
|
|
a9e985a86c | ||
|
|
bde8ff416f
|
||
|
|
92c5fc9ca4 | ||
|
|
86f9929cd2
|
||
|
|
4842cb174d | ||
|
|
18ae80740b
|
||
|
|
3c083c996e | ||
|
|
6c3acb7e53
|
||
|
|
ff0e5cbe73 | ||
|
|
2b66492571
|
||
|
|
062a95e30a | ||
|
|
4d2a735992
|
||
|
|
a2fad5103d
|
||
|
|
4194837610
|
||
|
|
5736bf3e1f | ||
|
|
f14a24fdbf
|
||
|
|
0fb9e8912c | ||
|
|
626e153841 | ||
|
|
85de1daf79 | ||
|
|
be26a370ac | ||
|
|
3c6e7d57a2 | ||
|
|
4b6717ebdd
|
||
|
|
9c2766e324
|
||
|
|
19ac187886
|
||
|
|
f380b9b2a1
|
||
|
|
906df343b0 |
No files matched your search
@@ -1,65 +1,14 @@
|
||||
version: 2.1
|
||||
|
||||
setup: true
|
||||
orbs:
|
||||
qqq-orb: kingsrook/qqq-orb@0.6.8
|
||||
|
||||
commands:
|
||||
configure-package-install:
|
||||
steps:
|
||||
- run:
|
||||
name: Keep package installation unattended
|
||||
command: |
|
||||
# These build machines are discarded after the job.
|
||||
sudo install -d /etc/needrestart/conf.d
|
||||
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
|
||||
|
||||
qqq-orb: kingsrook/qqq-orb@0.7.12
|
||||
workflows:
|
||||
|
||||
test_only:
|
||||
setup:
|
||||
jobs:
|
||||
- qqq-orb/mvn_test_only:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [ qqq-maven-registry-credentials ]
|
||||
filters:
|
||||
branches:
|
||||
ignore: /(develop|release/.*|integration.*)/
|
||||
tags:
|
||||
only: []
|
||||
|
||||
publish_snapshot:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [ qqq-maven-registry-credentials ]
|
||||
branch_type: snapshot
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- develop
|
||||
|
||||
publish_release_candidate:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [ qqq-maven-registry-credentials ]
|
||||
branch_type: release_candidate
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- /release\/.*/
|
||||
|
||||
publish_release:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [ qqq-maven-registry-credentials ]
|
||||
branch_type: release
|
||||
filters:
|
||||
branches:
|
||||
ignore: /.*/
|
||||
tags:
|
||||
only: [ /^v\d+\.\d+\.\d+$/ ]
|
||||
- qqq-orb/generate_pipeline:
|
||||
orb_version: 0.7.12
|
||||
filters:
|
||||
branches:
|
||||
ignore: automation/qbit-rc-pending
|
||||
tags:
|
||||
only: /^v\d+\.\d+\.\d+$/
|
||||
@@ -0,0 +1,23 @@
|
||||
name: CodeQL
|
||||
'on':
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- develop
|
||||
- release/**
|
||||
- hotfix/**
|
||||
pull_request: {}
|
||||
schedule:
|
||||
- cron: 41 16 * * 0
|
||||
workflow_dispatch: {}
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
security-events: write
|
||||
packages: read
|
||||
concurrency:
|
||||
group: codeql-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
jobs:
|
||||
analyze:
|
||||
uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22
|
||||
@@ -1,25 +1,74 @@
|
||||
# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
|
||||
# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
|
||||
# the release line and the release/* push runs CircleCI publish_release_candidate.
|
||||
name: Release candidate
|
||||
|
||||
on:
|
||||
'on':
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: Print the plan without merging or creating branches
|
||||
description: Plan a cut without changing release state
|
||||
type: boolean
|
||||
default: false
|
||||
finalize:
|
||||
description: Retry finalization of an existing cut; do not cut another RC
|
||||
type: boolean
|
||||
default: false
|
||||
recover:
|
||||
description: Recover the same published RC after a deployment timeout; never
|
||||
reupload
|
||||
type: boolean
|
||||
default: false
|
||||
circle_build:
|
||||
description: Original Circle publisher build number (recovery only)
|
||||
type: string
|
||||
default: ''
|
||||
merge_sha:
|
||||
description: Existing cut merge_sha (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
bump_sha:
|
||||
description: Existing cut bump_sha (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
branch:
|
||||
description: Existing cut branch (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
version:
|
||||
description: Existing cut version (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
pull_request:
|
||||
types: [closed]
|
||||
branches: [develop]
|
||||
|
||||
types:
|
||||
- closed
|
||||
branches:
|
||||
- develop
|
||||
repository_dispatch:
|
||||
types:
|
||||
- qbit-rc-published
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
actions: read
|
||||
checks: read
|
||||
jobs:
|
||||
release-rc:
|
||||
if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
|
||||
uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
|
||||
if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
|
||||
|| (github.event_name == 'pull_request' && github.event.pull_request.merged
|
||||
== true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
|
||||
'release:rc'))
|
||||
uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
|
||||
with:
|
||||
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
|
||||
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
|
||||
}}
|
||||
finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
|
||||
== 'workflow_dispatch' && inputs.finalize) }}
|
||||
recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
|
||||
}}
|
||||
circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
|
||||
|| '' }}
|
||||
merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
|
||||
|| inputs.merge_sha || '' }}
|
||||
bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
|
||||
|| inputs.bump_sha || '' }}
|
||||
branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
|
||||
|| inputs.branch || '' }}
|
||||
version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
|
||||
|| inputs.version || '' }}
|
||||
secrets: inherit
|
||||
@@ -0,0 +1,16 @@
|
||||
profile: qbit
|
||||
orb_version: 0.7.12
|
||||
default_branch: develop
|
||||
java_version: '21'
|
||||
browsers: true
|
||||
api_checks: false
|
||||
docs_only: false
|
||||
release_on_main: false
|
||||
release_tag_policy: stable
|
||||
test_branch_policy: non-release-no-integration
|
||||
publish:
|
||||
snapshot: true
|
||||
release_candidate: true
|
||||
release: true
|
||||
asciidoc: false
|
||||
release_completion: true
|
||||
@@ -18,7 +18,19 @@ Use the Maven 3.9.11 wrapper (Java 21); Maven 3.10 cannot read the parent POM. O
|
||||
|
||||
## Architecture
|
||||
|
||||
Package: `com.kingsrook.qbits.wormaudit`. Entry point: `WormAuditQBitProducer implements QBitMetaDataProducer<WormAuditQBitConfig>`. Config exposes `tableMetaDataCustomizer` and `retentionYears` (default 6).
|
||||
Package: `com.kingsrook.qbits.wormaudit`. Entry point: `WormAuditQBitProducer implements QBitMetaDataProducer<WormAuditQBitConfig>`. Config requires the host `backendName`; core audit tables must exist before production.
|
||||
|
||||
- `ddl`: reviewed Liquibase PostgreSQL template and safe renderer; public sidecar tables and owner-only retention purge.
|
||||
- `guard`: immutable QQQ capabilities and value-free SQLSTATE mapping.
|
||||
- `canonical`, `chain`, `checkpoint`: strict JCS v1, transaction/advisory-locked writer, explicit detail coverage, pinned Ed25519 ranges.
|
||||
- `verify`: snapshot checks of content, continuity, coverage and checkpoints, starting at the purge anchor.
|
||||
- `processes`: standard streamed ETL action, scheduled chainer and daily QAlerts producer; never app navigation entries.
|
||||
- `model`: read-only tables, range record references and standard record child lists. No app-home widgets or export execution in phase (a).
|
||||
|
||||
PostgreSQL uses a transaction advisory lock; other backends require a single writer JVM.
|
||||
Use `./mvnw.cmd -o verify` offline. PostgreSQL tests skip without Docker locally, but fail whenever `CI` is set.
|
||||
Core RC.5 has no assistant deny API (reflective registration). Alert types use the
|
||||
`wormaudit.*` namespace and are declared through core alert-type metadata.
|
||||
|
||||
## Invariants and naming
|
||||
|
||||
|
||||
@@ -4,6 +4,41 @@ All notable changes are documented here, following [Keep a Changelog](https://ke
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Fixed
|
||||
|
||||
- `worm-audit:13` failed with `permission denied for function worm_audit_purge` when the migrating role owns the function but had revoked its own EXECUTE (a runtime role that owns the schema, as on CNPG; PostgreSQL refuses `CREATE OR REPLACE` then). The changeset now grants EXECUTE back to the migrating role only if it lacks it, replaces the function, and revokes it again in the same transaction. Found by Website-Backend's CNPG-shaped changelog test.
|
||||
- `worm-audit:13` had a comment line starting with `-- changeset`, which Liquibase formatted-SQL parsing rejects as a malformed directive. The comment is reworded, and `CanonicalAndCryptoTest` now requires every `-- changeset` line to be a real directive.
|
||||
- Chain writer database load. Each 30-second tick re-read the whole late-detail lookback (24 h of `audit_detail`) and probed the GIN array index once per detail (18.4 s and 1.78M buffer hits per tick on a 120k-detail production database), and re-scanned every settled audit header for the unchained anti-join. Coverage now lives in `worm_audit_chain_detail` (`worm-audit:13`): one insert-only row per covered detail, written by the chain trigger in the same statement as its link and backfilled from the existing chain, so the late-detail check is a primary-key probe. Each tick also reads only ids above per-JVM sweep floors (rows settled since the last complete sweep); after a restart and once an hour a tick sweeps everything, as before. Link hash inputs, `detail_ids`, the verifier and exports are unchanged.
|
||||
- The chain trigger's overlap check (`EXISTS ... && ...`) ran as a cached generic plan that sequentially scanned all of `worm_audit_chain` for every inserted link (55k sequential scans, 1.2B tuples read in production, about 80 ms per link and growing). The trigger now inserts the coverage rows and lets their primary key reject a detail covered twice.
|
||||
- Verification counts detail coverage by aggregating the chain's own detail lists once instead of one index probe per detail, and breaks when the coverage table disagrees with the chain.
|
||||
|
||||
- Startup on hosts whose default authentication requires a token (OAuth2, Auth0): `produce()` read the settings singleton under a plain tokenless `QSession`, which failed session validation (`JWTDecodeException: The token is null`) and stopped the host. The startup read now runs as the core `QSystemUserSession`, still honoring the persisted schedules. `TokenRequiredHostTest` runs production under a token-required test authentication module.
|
||||
|
||||
### Added
|
||||
|
||||
- Initial scaffold
|
||||
- Foreign keys (`worm-audit:11-12`), `ON DELETE RESTRICT` only: chain link -> core audit header, sealed export -> its first and last checkpoints, export verification -> its export, with their indexes. Link references (ranges, break link, purge anchor) stay unconstrained because the retention purge deletes the links they outlive. Pre-existing orphans are copied to `qqq_fk_quarantine` (`KEPT`), never modified, and leave that constraint `NOT VALID`. Re-apply the rendered DDL (idempotent).
|
||||
- Guided empty states: `help/qbit-worm-audit.yaml` (QQQ help-bundle format) gives the app and all six tables a summary and a "How rows get created" section, applied by the `WormAuditHelp` stand-in (`STAND-IN(core=QRun-IO/qqq#1198, removeBy=4.1.0-RC.7)`) without replacing host help. `WormAuditTablesGuardrailTest` pins runtime use, seeding or a documented reason, and help for every table; `WormAuditSettingsSeedTest` runs the script's settings seed on H2.
|
||||
- RC.2 settings: editable, FIELD-audited singleton with insert-only defaults, validation, separate edit permission and a 30-second effective-settings cache. Schedules load at startup.
|
||||
- GOVERNANCE exports with 30-day default retention; per-export retention verification, administrator-bypass diagnostics, home warnings and documented COMPLIANCE transition.
|
||||
- New `worm-audit:10` settings migration; historical changesets remain unchanged.
|
||||
|
||||
- Phase (c): configurable WORM Audit app home, full-width summary widget and optional host health card; permission-aware navigation and content-free, responsive light/dark summaries.
|
||||
- Chain backlog/settling counts, verification and export record links, sealed-range gaps/lag, retention eligibility and recent purge/refusal history.
|
||||
- PostgreSQL guard/ownership/purge-function diagnostics, signing/trust warnings, tri-state Object Lock capability and stale daily-verification warnings.
|
||||
- Test-scope Next UI demo with synthetic tokenized patient notes, genuine verification failure/recovery, signed checkpoints, sealed memory exports and documented simulated purges.
|
||||
- Home privacy/permission/empty-state tests, memory/H2 summary tests and Docker-gated PostgreSQL setup-warning acceptance.
|
||||
|
||||
- Phase (a): PostgreSQL append-only tables/triggers, safe DDL renderer, retention purge with immutable refusal/anchor records, and runtime grants.
|
||||
- Canonical audit v1 documents, contiguous hash chain with late-detail coverage, pinned Ed25519 checkpoints and binary Merkle roots.
|
||||
- Snapshot verification, streamed ETL Review/Result action, scheduled chaining/daily verification, and core QAlerts raise/resolve/heartbeat integration.
|
||||
- Read-only metadata, WORM Audit table navigation, record range references and child lists; reflective RC.6 assistant deny registration.
|
||||
- Offline memory/H2 acceptance, real pre-insert PHI canary hook, and PostgreSQL Testcontainers acceptance with CI-strict Docker availability.
|
||||
- Adoption, app-home, demo and operations documentation.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Keep runtime signing/trust objects and storage suppliers out of serialized metadata so signed-chain record child lists render successfully.
|
||||
|
||||
### Deprecated
|
||||
|
||||
- `WormAuditSettings.seed(config)` is deprecated for PostgreSQL hosts, whose database script (`worm-audit:10`) already seeds row 1; it remains supported for memory and H2 hosts. `read()` falls back to the configured values while the row is absent.
|
||||
@@ -18,7 +18,19 @@ Use the Maven 3.9.11 wrapper (Java 21); Maven 3.10 cannot read the parent POM. O
|
||||
|
||||
## Architecture
|
||||
|
||||
Package: `com.kingsrook.qbits.wormaudit`. Entry point: `WormAuditQBitProducer implements QBitMetaDataProducer<WormAuditQBitConfig>`. Config exposes `tableMetaDataCustomizer` and `retentionYears` (default 6).
|
||||
Package: `com.kingsrook.qbits.wormaudit`. Entry point: `WormAuditQBitProducer implements QBitMetaDataProducer<WormAuditQBitConfig>`. Config requires the host `backendName`; core audit tables must exist before production.
|
||||
|
||||
- `ddl`: reviewed Liquibase PostgreSQL template and safe renderer; public sidecar tables and owner-only retention purge.
|
||||
- `guard`: immutable QQQ capabilities and value-free SQLSTATE mapping.
|
||||
- `canonical`, `chain`, `checkpoint`: strict JCS v1, transaction/advisory-locked writer, explicit detail coverage, pinned Ed25519 ranges.
|
||||
- `verify`: snapshot checks of content, continuity, coverage and checkpoints, starting at the purge anchor.
|
||||
- `processes`: standard streamed ETL action, scheduled chainer and daily QAlerts producer; never app navigation entries.
|
||||
- `model`: read-only tables, range record references and standard record child lists. No app-home widgets or export execution in phase (a).
|
||||
|
||||
PostgreSQL uses a transaction advisory lock; other backends require a single writer JVM.
|
||||
Use `./mvnw.cmd -o verify` offline. PostgreSQL tests skip without Docker locally, but fail whenever `CI` is set.
|
||||
Core RC.5 has no assistant deny API (reflective registration). Alert types use the
|
||||
`wormaudit.*` namespace and are declared through core alert-type metadata.
|
||||
|
||||
## Invariants and naming
|
||||
|
||||
|
||||
@@ -1,49 +1,401 @@
|
||||
# QBit WORM Audit
|
||||
|
||||
Write-once, read-many (WORM) audit for QQQ applications. The QQQ core audit tables (`audit`, `audit_detail`) and this qbit's own tables become append-only at the database level, every audit row is linked into a hash chain with periodic signed checkpoints, a verification action and a daily job detect tampering, and new audit rows are sealed into an S3 bucket with Object Lock in COMPLIANCE mode (default retention 6 years, the HIPAA audit-record retention). Audit content is hashed and sealed exactly as core writes it; with qbit-secured-data, PHI field values in audit details are already HMACs, so no raw PHI reaches the chain or the sealed copies. This repository currently contains the compiling scaffold; the design is in [docs/DESIGN.md](docs/DESIGN.md) once the spec PR lands.
|
||||
Phases (a)–(c) provide PostgreSQL append-only enforcement, an asynchronous audit hash chain,
|
||||
Ed25519 checkpoints, live and sealed-export verification actions, and daily jobs with core QAlerts signals.
|
||||
It hashes exactly the audit values core stored; it does not read application records or
|
||||
secured-data keys. Sealed JSONL exports use qbit-object-storage 0.1.0-RC.2 and configurable
|
||||
Object Lock (GOVERNANCE for 30 days by default). The WORM Audit app home summarizes chain health, exports, retention and setup.
|
||||
|
||||
## Requirements
|
||||
|
||||
Java 21 and QQQ 4.1.0-RC.5 or later. PostgreSQL is the supported database for the append-only enforcement. The Maven wrapper pins Maven 3.9.11.
|
||||
|
||||
## Install
|
||||
|
||||
```xml
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qbits</groupId>
|
||||
<artifactId>qbit-worm-audit</artifactId>
|
||||
<version>0.1.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
```
|
||||
Requires Java 21, Maven wrapper 3.9.11 and QQQ 4.1.0-RC.5. PostgreSQL provides enforcement;
|
||||
memory/H2 support tamper-evidence tests without database enforcement.
|
||||
|
||||
## Tables and permissions
|
||||
|
||||
Planned tables (none are produced yet): `worm_audit_chain`, `worm_audit_checkpoint`, `worm_audit_export`, `worm_audit_verification`, `worm_audit_purge`. Backend names match table names. Permissions use the full `worm_audit` prefix.
|
||||
All sidecar tables use the configured host backend, snake_case fields, BIGINT identifiers,
|
||||
record sections, and full table-name permissions. Immutable sidecars do not audit themselves.
|
||||
The editable Settings singleton uses an INTEGER ID and FIELD-level core audit. The app lists
|
||||
Verifications, Sealed Exports, Checkpoints, Audit Chain, Purges, and Settings in that order.
|
||||
|
||||
| Table | Purpose | Permission |
|
||||
|---|---|---|
|
||||
| `worm_audit_verification` | Immutable USER/SCHEDULE results | `worm_audit_verification.read` |
|
||||
| `worm_audit_export` | Successfully sealed export catalog | `worm_audit_export.read` |
|
||||
| `worm_audit_checkpoint` | Signed ranges and Merkle roots | `worm_audit_checkpoint.read` |
|
||||
| `worm_audit_chain` | Contiguous links and explicit detail coverage | `worm_audit_chain.read` |
|
||||
| `worm_audit_purge` | Refusals, removal counts and retention anchors | `worm_audit_purge.read` |
|
||||
| `worm_audit_settings` | Editable singleton; every edit is audited and chained | `worm_audit_settings.read`; editing needs `worm_audit_settings.edit` **and** `worm_audit.settings` |
|
||||
|
||||
`worm_audit.verify` authorizes the standard streamed ETL **Verify audit chain** action.
|
||||
It also authorizes **Verify sealed export** on selected export records. `worm_audit.export`
|
||||
authorizes **Export now** on the export table. Processes are table/record actions or schedules;
|
||||
none is a navigation entry.
|
||||
Insert/update/delete capabilities are absent on immutable sidecar tables; Settings allows updates only; update/delete capabilities
|
||||
are removed from all four core audit tables. Internal appends use an explicit permission bypass.
|
||||
Checkpoint, export and verification record screens have a bounded chain-link child list;
|
||||
range endpoints and export/checkpoint references link to their records.
|
||||
|
||||
The table metadata customizer can configure host ESB publications. Quick Search registration
|
||||
is host-owned through `QuickSearchQBitConfig.searchableTables`; add the desired tables.
|
||||
Verification and export inserts use QQQ's record-change listener SPI,
|
||||
so an installed ESB publisher receives those events when the host configures its destinations
|
||||
on these two tables. The qbit defines no destination and configures no chain-link publication.
|
||||
|
||||
## Configuration
|
||||
|
||||
`WormAuditQBitConfig` exposes an optional `tableMetaDataCustomizer` and `retentionYears` (default 6).
|
||||
| Field | Default / contract |
|
||||
|---|---|
|
||||
| `backendName` | Required; core and sidecar audit tables must use this backend |
|
||||
| `appName` / `appLabel` | `worm_audit` / `WORM Audit` |
|
||||
| `parentAppName` | Optional host app, for example `admin` |
|
||||
| `tablePathPrefix` | `/app`; standard QQQ table/record route prefix |
|
||||
| `sealedStoreSupplier` | Optional host/test storage boundary; defaults to qbit-object-storage |
|
||||
| `retentionYears` | 6; compiled into the migration, not changed by a runtime flag |
|
||||
| `signingKey` | Optional `Supplier<SigningKey>`; `SigningKey(keyId, PrivateKey)` |
|
||||
| `trustedPublicKeys` | `Map<String, PublicKey>`; must contain the signing key's public key |
|
||||
| `chainIntervalSeconds` | 30 |
|
||||
| `settleHorizonSeconds` | 60 (zero allowed) |
|
||||
| `exportBucketAlias` | `worm_audit`; qbit-object-storage BACKUP bucket alias |
|
||||
| `exportPrefix` | `worm-audit`; slash-separated letters, digits, underscores or hyphens |
|
||||
| `exportCron` | `0 0 2 * * ?`; 02:00 UTC on the configured scheduler |
|
||||
| `batchSize` | 1,000 headers per tick / links per insert batch |
|
||||
| `lateDetailLookbackHours` | 24; positive hours, older omissions checked by full verification |
|
||||
| `checkpointEveryLinks` | 10,000; maximum links in each checkpoint |
|
||||
| `checkpointEveryMinutes` | 60; requires at least one uncheckpointed link |
|
||||
| `verifyCron` | `0 0 3 * * ?` (03:00 UTC, Quartz syntax) |
|
||||
| `schedulerName` | `worm_audit`; uses the host scheduler if present, otherwise registers Quartz |
|
||||
| `coreAuditTableNames` | Overrides keyed by `audit`, `auditDetail`, `auditTable`, `auditUser` |
|
||||
| `appRole` | `app` |
|
||||
| `tableMetaDataCustomizer` | Optional; append-only capability guard runs afterward |
|
||||
|
||||
A missing signing key produces no checkpoints and adds the warning **no signing key configured**
|
||||
to verification. It does not itself break the chain. Signing requires a matching public-key pin.
|
||||
Rotate by adding a new pin and changing the supplier; retain old pins for old checkpoints.
|
||||
Never store the private key in a database row or checked-in configuration.
|
||||
|
||||
## Settings
|
||||
|
||||
Open **WORM Audit → Settings**, record **1** (`worm_audit_settings`, labelled WORM Audit Settings).
|
||||
This is a standard editable QQQ table. New records and deletion are disabled; a pre-insert
|
||||
check and database constraint require ID 1. Every edit uses core FIELD-level auditing, so
|
||||
the change enters the audit chain. `modify_date` and `modified_by` are stamped by the server.
|
||||
Keep notes free of sensitive data; notes are audited too.
|
||||
|
||||
### Starter defaults and empty screens
|
||||
|
||||
The only seeded table is `worm_audit_settings`, by the database script (`WormAuditDdl.postgres(config)`, changeset `worm-audit:10`): `INSERT ... SELECT ... WHERE NOT EXISTS`, so it adds row 1 from the host's configured values only when absent and never overwrites an edit. There is no separate defaults changelog and no extra include line. If the row is absent the qbit runs on the configured values (`WormAuditSettings.read()` falls back), so nothing stops, but run the script to make the values visible and editable. The chain, checkpoint, export, verification and purge tables are logs written by the qbit's jobs and the database owner's purge function, and start empty.
|
||||
|
||||
Every table, and the app, ships help (`help/qbit-worm-audit.yaml`, the QQQ help-bundle format) saying what it is for and how its rows get created; the Next UI shows it as the empty state. A host's own help is never replaced. `WormAuditTablesGuardrailTest` and `WormAuditSettingsSeedTest` pin this.
|
||||
|
||||
| Field | Default | Validation / effect |
|
||||
|---|---|---|
|
||||
| `audit_retention_years` | 6 | 1–100; home next eligible purge date |
|
||||
| `export_lock_mode` | GOVERNANCE | GOVERNANCE or COMPLIANCE; new exports |
|
||||
| `export_retention_days` | 30 | 1–36500; new exports |
|
||||
| `checkpoint_every_links` | 10000 | Positive integer; new checkpoint ranges |
|
||||
| `checkpoint_every_minutes` | 60 | Positive integer; checkpoint cadence |
|
||||
| `chain_interval_seconds` | 30 | Positive integer; **takes effect after restart** |
|
||||
| `settle_horizon_seconds` | 60 | Nonnegative integer; chaining and verification |
|
||||
| `late_detail_lookback_hours` | 24 | Positive integer; late-detail polling |
|
||||
| `export_cron` | `0 0 2 * * ?` | Quartz cron, UTC; **takes effect after restart** |
|
||||
| `verify_cron` | `0 0 3 * * ?` | Quartz cron, UTC; **takes effect after restart** |
|
||||
| `notes` | Empty | Administrative notes |
|
||||
| `modify_date`, `modified_by` | Server supplied | Last edit time and user |
|
||||
|
||||
Config values supply defaults for missing rows or null fields; a missing settings table also
|
||||
falls back to config. Other database failures fail closed. `exportLockMode` and
|
||||
`exportRetentionDays` are the corresponding config defaults. Runtime reads cache the row
|
||||
for at most 30 seconds per application instance; local QQQ edits invalidate that cache.
|
||||
Each run reads effective settings. Startup schedules read the existing row, or config when
|
||||
absent. Historical checkpoints remain valid after changing their creation cadence.
|
||||
|
||||
Changeset `worm-audit:10` creates and seeds the table with `INSERT ... SELECT ... WHERE NOT
|
||||
EXISTS`; rerunning it never overwrites an administrator change. The renderer seeds from the
|
||||
host config. Memory and H2 hosts, whose settings table cannot take generic inserts, can still call
|
||||
`WormAuditSettings.seed(config)` for the same insert-only behavior during single-threaded
|
||||
startup (deprecated for PostgreSQL hosts, which get the row from the script).
|
||||
|
||||
`audit_retention_years` does **not** rewrite the database purge function. The home warns if
|
||||
it differs from the compiled interval; changing that interval requires a reviewed migration.
|
||||
The home also warns when export retention is shorter than audit retention and while
|
||||
GOVERNANCE permits administrator bypass deletion. The runtime role receives SELECT, INSERT
|
||||
and UPDATE on Settings, with no DELETE or TRUNCATE grant and no append-only triggers.
|
||||
|
||||
## Switching sealed exports to COMPLIANCE
|
||||
|
||||
Edit `export_lock_mode` to **COMPLIANCE** in Settings. New exports use COMPLIANCE once the
|
||||
settings cache refreshes (within 30 seconds); existing GOVERNANCE objects keep their mode
|
||||
unless an administrator explicitly extends their retention and upgrades their mode. The
|
||||
verifier continues to compare each object with its export row's recorded mode and date, so
|
||||
an out-of-band mode change is reported as a mismatch. The bucket default retention should
|
||||
match the chosen mode and retention days, although the exporter always sends explicit values.
|
||||
The settings change is audited and chained. **COMPLIANCE cannot be undone**: protected
|
||||
versions cannot be deleted or have retention shortened until expiry, including by administrators.
|
||||
|
||||
## App home
|
||||
|
||||
Open `/app/worm_audit` (or `/app/admin/worm_audit` with `parentAppName="admin"`).
|
||||
The full-width `wormAuditHome` HTML widget uses the standard `/qqq/v1/widget` API.
|
||||
Hosts can add `wormAuditHealthCard` to their own dashboard. Both widgets and the app
|
||||
require read permission on at least one WORM table, using the host's table permission rules.
|
||||
To embed WORM in an existing host app, set `withIncludeApp(false)` and configure
|
||||
`appName`, `appLabel` and optional `parentAppName` for that host app. The host owns
|
||||
its navigation, home widget placement and app help; the qbit does not create or replace
|
||||
the app or fill its help. Both widgets and all table help, permissions, guards,
|
||||
processes and schedules remain registered. The health card links to the configured
|
||||
host app. Standalone app creation remains enabled by default.
|
||||
The aggregate home shows only identifiers, counts, times, shortened hashes and reasons;
|
||||
it never renders audit messages or field values. Record links retain the target table's permissions.
|
||||
|
||||
- Chain status: live link count, latest link time, settled unchained backlog and settling
|
||||
count at the configured horizon, latest verification and first open chain break.
|
||||
A subsequent full scheduled CHAIN PASS clears a reported break; a selected USER PASS does not.
|
||||
- Export status: current lock mode and retention days, latest sealed range and its recorded retention, link/time lag, coverage holes
|
||||
between the purge anchor and sealed head, and bucket Object Lock capability. The unsealed
|
||||
trailing suffix is lag, not a gap. Provider failures show UNKNOWN with a value-free reason.
|
||||
- Retention: configured years, purge anchor, oldest live audit and its next eligible purge date.
|
||||
The date indicates retention eligibility; the database purge function still checks the sealed prefix.
|
||||
- Recent purges: the last ten requests, including refusals.
|
||||
- Setup warnings: missing signing/trusted keys, unsupported or unknown Object Lock,
|
||||
daily CHAIN verification never run or older than 26 hours. PostgreSQL also checks every
|
||||
protected table's row/truncate guards, runtime ownership and the purge function.
|
||||
|
||||
Sections and definition-list rows wrap at narrow widths and inherit the Next UI's light/dark colors.
|
||||
Empty states explicitly distinguish missing history from a passing check.
|
||||
|
||||
### Local demo
|
||||
|
||||
The test-scope server serves the API and bundled Next UI with a memory backend:
|
||||
|
||||
```powershell
|
||||
./mvnw.cmd -o test-compile exec:java "-Dexec.classpathScope=test" "-Dexec.mainClass=com.kingsrook.qbits.wormaudit.demo.WormAuditDemoServer" "-Dexec.args=8099"
|
||||
```
|
||||
|
||||
On macOS/Linux use `./mvnw` with the same arguments. Open <http://localhost:8099/app/worm_audit>.
|
||||
No Docker or credentials are required. This anonymous demo uses synthetic `patient_note`
|
||||
records audited at FIELD level through the `PhiCanaryTest` HMAC stand-in. It seeds two weeks
|
||||
of actual chainer ticks, signed checkpoints, daily verifier results and immutable memory exports.
|
||||
One temporary detail tamper produces a genuine FAIL before restoration and later PASSes.
|
||||
The PURGED/REFUSED receipts and memory-only prefix removal are explicitly marked demo data;
|
||||
PostgreSQL enforcement and real provider durability are not simulated production guarantees.
|
||||
|
||||
The demo includes a small unsealed suffix and unchained backlog, with no invented export gap.
|
||||
The final settling count naturally ages into the backlog while the demo is open.
|
||||
|
||||
## Adopting in another app
|
||||
|
||||
Add `com.kingsrook.qbits:qbit-worm-audit:0.1.0-SNAPSHOT`. Define the host's core audit tables
|
||||
with `AuditsMetaDataProvider` first, then install the qbit:
|
||||
|
||||
```java
|
||||
new WormAuditQBitProducer()
|
||||
.withQBitConfig(new WormAuditQBitConfig())
|
||||
.produce(qInstance);
|
||||
WormAuditQBitConfig config = new WormAuditQBitConfig()
|
||||
.withBackendName("main")
|
||||
.withAppRole("app")
|
||||
.withSigningKey(() -> new WormAuditQBitConfig.SigningKey("2026-10", privateKey))
|
||||
.withTrustedPublicKeys(Map.of("2026-10", publicKey));
|
||||
new WormAuditQBitProducer().withQBitConfig(config).produce(qInstance);
|
||||
String migration = WormAuditDdl.postgres(config);
|
||||
```
|
||||
|
||||
Production currently registers the QBit identity and config only. Database changesets, the object-storage bucket and the alert wiring are documented with each phase.
|
||||
1. Render and review the migration, then copy it into the host Liquibase changelog.
|
||||
The [default rendering](docs/ddl/worm-audit-postgres.sql) uses `audit`, `audit_detail`,
|
||||
`audit_table`, `audit_user`, runtime role `app`, and six years. The renderer resolves table
|
||||
backend names during production; explicit overrides win. Standalone rendering uses those
|
||||
defaults. Identifiers are validated and quoted; schema-qualified core table names are supported.
|
||||
2. Apply the DDL as the schema owner. It creates the five sidecar tables in `public`, functions,
|
||||
triggers, the NOLOGIN `worm_audit_purger` role, and runtime grants. Core column names must use
|
||||
the usual snake_case mappings (`audit_id`, `audit_table_id`, etc.). The migration is idempotent. Provision the purger role beforehand or give the migration role permission to create it.
|
||||
3. Run as a separate runtime role with SELECT/INSERT and sequence USAGE. Do not make it a member
|
||||
of the purger role or a schema/table owner. Use the host's PostgreSQL RDBMS strategy (RC.5's
|
||||
base strategy uses MySQL quoting; see the test host strategy in `RdbmsTestSupport`).
|
||||
4. Start QQQ's standard scheduler with its system session and grant users only the required read
|
||||
and verification permissions. Point the existing alerts implementation at core `QAlerts`.
|
||||
5. Configure ESB publications on verification/export in the table customizer. Register all five
|
||||
tables in the host Quick Search config with `SearchableTableConfig` / `SearchableFieldConfig`.
|
||||
Index identifiers and status fields; set basepull timestamps to `chained_at`, `created_at`,
|
||||
`sealed_at`, `finished_at`, and `requested_at` respectively (these tables have no `modifyDate`).
|
||||
|
||||
## Build
|
||||
The RC.6 assistant deny API is registered reflectively when present; absence in RC.5 is a no-op.
|
||||
The qbit declares `wormaudit.chain.break` (CRITICAL), `wormaudit.export.failed` (ERROR),
|
||||
and the daily verification/export heartbeat types through core alert-type metadata.
|
||||
Daily heartbeat monitors expect a 24-hour interval with 2 hours of grace (overdue at 26 hours).
|
||||
`QAlerts` dispatches to the installed sink or its logging fallback. Every scheduled verification sends
|
||||
`wormaudit.verify.daily`; FAIL raises with the first break and verification record reference;
|
||||
PASS resolves dedup key `chain`.
|
||||
|
||||
```bash
|
||||
./mvnw compile
|
||||
./mvnw test
|
||||
./mvnw verify
|
||||
## Chain and verification format
|
||||
|
||||
- JCS v1 uses UTF-16 key ordering, strict surrogate validation and UTC microsecond timestamps.
|
||||
IDs are integer longs except canonical `recordId`, which is a string. No floating-point values.
|
||||
- Link hashes concatenate the predecessor's **64 lowercase ASCII hex bytes**, one NUL byte,
|
||||
and canonical UTF-8 bytes. Genesis is 64 ASCII zeroes.
|
||||
- Every link names the detail IDs it covered, at most 250. A PostgreSQL insert guard and GIN
|
||||
overlap index reject duplicate coverage under the same advisory lock. Overflow and later details use
|
||||
`LATE_DETAIL` links with the same canonical header-and-detail-subset document. The canonical
|
||||
JSON is never persisted. A v1 content mismatch can identify the link, audit and covered detail
|
||||
IDs, but cannot identify one changed field among multiple details without original content.
|
||||
- PostgreSQL chaining uses transaction-scoped advisory lock `86753090421` shared with purge.
|
||||
Other backends use a JVM-wide lock and require one writer JVM. SQL backends select settled
|
||||
unchained headers with an anti-join; late commits are picked up on a later tick.
|
||||
- Merkle leaves are decoded 32-byte link hashes; parents hash the concatenated binary children;
|
||||
an odd node is promoted unchanged. Ed25519 signs JCS of the seven specified range/trust fields.
|
||||
- SQL verification uses a repeatable-read snapshot. Full runs check settled header/detail
|
||||
completeness; selected or filtered runs check their links, predecessor boundaries and
|
||||
intersecting checkpoints.
|
||||
|
||||
## Retention operations
|
||||
|
||||
Only the ops login in `worm_audit_purger` may execute:
|
||||
|
||||
```sql
|
||||
SELECT public.worm_audit_purge(now() - interval '6 years');
|
||||
```
|
||||
|
||||
Verification includes Checkstyle and JaCoCo gates of 70% instructions and 90% classes. On Windows, use `./mvnw.cmd`.
|
||||
A recent/null cutoff or an incomplete expired prefix records REFUSED and returns normally.
|
||||
Purge locks the audit tables against concurrent inserts, removes only a prefix closed over
|
||||
all links and details of its audits, and records its last link id/hash. Detail and chain deletion
|
||||
is allowed only under the function-owner identity, transaction-local purge flag, and compiled
|
||||
retention cutoff. An app session setting that flag gets no deletion authority.
|
||||
|
||||
Lookups cannot change id/name or be deleted/truncated. The runtime role may update only the
|
||||
lookup label/dates through column grants; table-level UPDATE stays revoked. With normal grants,
|
||||
PostgreSQL rejects prohibited runtime writes as SQLSTATE 42501 before executing a trigger.
|
||||
The second layer (triggers) returns `P0001` / `worm_audit:` even if someone grants wider DML
|
||||
rights. The qbit maps those errors to a value-free user-facing message.
|
||||
|
||||
Verification starts from the latest successful purge anchor. Checkpoint signatures still verify
|
||||
after purge, but Merkle recomputation of a purged/partly purged checkpoint requires its sealed
|
||||
export and is reported as a warning. Seal and verify exports before exercising retention
|
||||
removal in production. Superusers/owners can disable triggers;
|
||||
protect signing keys and keep public-key pins outside the database.
|
||||
|
||||
## Sealed exports
|
||||
|
||||
Install qbit-object-storage and configure `exportBucketAlias` (default `worm_audit`) as a
|
||||
`BucketRole.BACKUP` bucket. **Object Lock must be enabled when the bucket is created**.
|
||||
The exporter requires the bucket's `OBJECT_LOCK` capability to be `SUPPORTED`; run the
|
||||
object-storage read-only capability probe during setup. Unknown, partial and unsupported
|
||||
capabilities all refuse sealing. Every object receives the effective `export_lock_mode` and retention until sealing time plus
|
||||
`export_retention_days` (GOVERNANCE and 30 days by default), checked by HEAD after PUT. The
|
||||
export row records both values. Verification checks that historical mode and retain-until,
|
||||
regardless of subsequent settings changes. Missing GOVERNANCE objects are reported as breaks
|
||||
with a possible administrator bypass-removal explanation.
|
||||
|
||||
The runtime IAM policy needs `s3:PutObject`, `s3:GetObject`, `s3:ListBucket`,
|
||||
`s3:GetObjectRetention`, `s3:PutObjectRetention`, and
|
||||
`s3:GetBucketObjectLockConfiguration`. Grant no delete or governance-bypass access.
|
||||
The storage bucket's upload size limit must accommodate the largest daily JSONL file.
|
||||
Retention protects object versions; protect the storage volumes and maintain off-site copies too.
|
||||
|
||||
At 02:00 UTC, or through **Export now**, the exporter takes the chainer's writer lock and
|
||||
seals links after the previous export through the latest checkpoint. The first export starts
|
||||
after the purge anchor, or at genesis. No new checkpoint produces an explicit "Nothing to
|
||||
seal" Result. If retention has removed an unexported gap, or a range cannot be proven by
|
||||
complete signed checkpoints, sealing refuses rather than recording incomplete evidence.
|
||||
Other database backends require a single writer JVM, as in phase (a).
|
||||
|
||||
Keys use `<exportPrefix>/<yyyy>/<MM>/<dd>/<from>-<to>/` with a UTC date:
|
||||
|
||||
- `links.jsonl`: JCS lines containing `link` identifiers/hashes and the embedded canonical
|
||||
`content` document exactly as hashed.
|
||||
- `checkpoints.jsonl`: JCS checkpoint envelopes including signatures and public keys.
|
||||
- `manifest.json`: JCS `worm-audit-export/1` with `fromLinkId`, `toLinkId`, `linkCount`,
|
||||
`checkpointCount`, `files` (filename to SHA-256), `headHash`, `prevManifestSha256`,
|
||||
`qbitVersion`, and `createdAt`.
|
||||
|
||||
Files are streamed through temporary disk storage with 1,000-link pages and batched content
|
||||
fetches, then removed. Only ids, hashes and object metadata enter the export catalog.
|
||||
The `object_versions` text column pins each upload's returned version id in a JCS JSON
|
||||
object keyed by `checkpoints.jsonl`, `links.jsonl`, and `manifest.json`. Sealing refuses
|
||||
missing version ids or a different version in the retention HEAD response, leaving no
|
||||
catalog row and raising the export failure alert.
|
||||
Partial upload failures leave no catalog row; retained orphan objects may remain. Retrying
|
||||
can create new locked versions under the same keys. Failures raise `wormaudit.export.failed`
|
||||
with dedup key `export`; success resolves it. Every scheduled run emits
|
||||
`wormaudit.export.daily`.
|
||||
|
||||
**Verify sealed export** accepts one or several selected records. It downloads and checks
|
||||
the object hashes, retention, manifest chain, link hashes, pinned checkpoint signatures and
|
||||
Merkle roots, then compares the sealed content and link metadata against a live database
|
||||
snapshot. Differences name exact link/audit ids without printing audit values. Each export
|
||||
gets its own `SEALED_EXPORT` verification row. Purged prefixes verify from the sealed copy
|
||||
alone and receive an INFO line. Verification does not write restored audit rows.
|
||||
|
||||
Verification compares every current HEAD version to the catalog and reports an ERROR for
|
||||
any overwrite, even when the replacement bytes are identical. It then reads the recorded
|
||||
sealed versions, so retained evidence remains verifiable after an overwrite; the overall
|
||||
verification still fails because the overwrite is a break. Missing catalog versions fail
|
||||
closed. RC.2's `StorageBucket.get` has no version parameter, so the adapter uses the public
|
||||
`ObjectStorageRuntime.connection(...).client()` accessor for an explicit S3 version GET,
|
||||
reusing its connection, key normalization and rate limiter and checking the response version.
|
||||
No object-storage change is required; this adapter depends on RC.2's public internal runtime
|
||||
accessors. The storage credentials need `s3:GetObjectVersion` in addition to the existing
|
||||
upload, HEAD and listing permissions.
|
||||
|
||||
## Build and acceptance
|
||||
|
||||
```powershell
|
||||
./mvnw.cmd -o verify
|
||||
```
|
||||
|
||||
Checkstyle and JaCoCo gates (70% instructions, 90% classes) run without Docker. Memory and H2
|
||||
tests cover canonical vectors, signatures, rollback, exact coverage, late arrivals/details,
|
||||
verification findings, PHI tokenization, process summaries, permissions, and the alerts SPI.
|
||||
The PHI stand-in uses RC.5 `AuditActionCustomizerInterface.customizeInput` before insertion;
|
||||
classification is test-local because RC.5 lacks `DataClassification`.
|
||||
|
||||
PostgreSQL acceptance uses `postgres:16-alpine`, applies the DDL twice and tests grants,
|
||||
triggers, purge, real QQQ RDBMS chaining, superuser tamper, and 8 writers / 3 chainers.
|
||||
It skips automatically without Docker locally and fails if **any** `CI` environment value is set.
|
||||
On a Docker-enabled coordinator (image already available if running without network):
|
||||
|
||||
```powershell
|
||||
$env:CI = 'true'
|
||||
./mvnw.cmd -o '-Dtest=PostgresAcceptanceTest' test
|
||||
./mvnw.cmd -o verify
|
||||
```
|
||||
|
||||
MinIO export acceptance uses the same `RequiresDockerCondition`. It checks GOVERNANCE
|
||||
with 30-day retention and a COMPLIANCE export, denied version deletion (without bypass for GOVERNANCE), overwrite version preservation, export verification,
|
||||
exact live-tamper findings through the record action, and refusal of a bucket without lock:
|
||||
|
||||
```powershell
|
||||
$env:CI = 'true'
|
||||
./mvnw.cmd -o '-Dtest=MinioExportAcceptanceTest' test
|
||||
```
|
||||
|
||||
The default MinIO image is pinned to the same Chainguard digest as object-storage's tests.
|
||||
Override it with `-Dqqq.test.minioImage=<image>` when necessary. Images must already be
|
||||
available for a completely network-free Docker run.
|
||||
|
||||
POSIX equivalents: `CI=true ./mvnw -o -Dtest=MinioExportAcceptanceTest test`,
|
||||
`CI=true ./mvnw -o -Dtest=PostgresAcceptanceTest test` and
|
||||
`CI=true ./mvnw -o verify`. No commits, publishing or deployment are part of these commands.
|
||||
|
||||
## License
|
||||
|
||||
[Apache License 2.0](LICENSE). Copyright Kingsrook, LLC.
|
||||
|
||||
### Physical columns and bounded polling
|
||||
|
||||
`WormAuditDdl.postgres(config)` renders table names only. Core physical columns
|
||||
must be snake_case: `audit.id`, `audit.timestamp`, `audit_detail.id`,
|
||||
`audit_detail.audit_id`, `audit_table.id/name/label/create_date/modify_date`, and
|
||||
`audit_user.id/name/create_date/modify_date`. QInstance field backend-name
|
||||
customizations do not change this migration; adapt the reviewed SQL before
|
||||
applying it if your schema differs. UTC values in `timestamp without time zone`
|
||||
and `timestamptz` are both supported.
|
||||
|
||||
`lateDetailLookbackHours` defaults to **24** (positive hours). Each PostgreSQL
|
||||
poll checks uncovered details whose chained header timestamp lies in
|
||||
`[cutoff - lookback, cutoff)`. The daily full verifier reports omissions outside
|
||||
that window. Initial header details are always chained, including overflow
|
||||
chunks. Checkpoints contain at most `checkpointEveryLinks` links; multiple full
|
||||
ranges can be signed per tick. Verification uses 1,000-link keyset pages.
|
||||
|
||||
Full verification holds at most one 1,000-link page and one checkpoint range of hashes.
|
||||
SQL completeness uses bounded offender queries; other persistent backends use repeated bounded
|
||||
scans. Only the memory backend uses a full coverage index.
|
||||
Full verification retains at most 1,000 exact break diagnostics and 100 warnings,
|
||||
while persisting the total break count. SQL completeness counts all offenders
|
||||
and reports the first 100 header ids and first 100 detail ids. This keeps a badly
|
||||
corrupted chain from exhausting verifier memory.
|
||||
@@ -0,0 +1,11 @@
|
||||
# Upgrading qbit-worm-audit
|
||||
|
||||
## Detail coverage table (0.1.0-RC.4)
|
||||
|
||||
Apply changeset `worm-audit:13` from the rendered DDL (or your copy of it) before or with the new jar. It takes the chain advisory lock, creates `worm_audit_chain_detail` with its append-only guards and grants, backfills it from the existing chain, and replaces `worm_audit_reject()`, `worm_audit_detail_coverage()` and `worm_audit_purge()`. The jar requires the table: its chain writer reads it every tick. The GIN index `worm_audit_chain_detail_coverage` is no longer used but is kept so an RC.3 writer still running during a rolling deploy keeps an indexed probe; a later changeset drops it. Memory and H2 hosts need no migration beyond creating the new table (the writer fills it).
|
||||
|
||||
## Table help bundle (unreleased)
|
||||
|
||||
Nothing to do. The qbit applies `help/qbit-worm-audit.yaml` at the end of `produce()`, filling only help slots the host has not set. No schema or seed changes: the settings row is still seeded by the database script (changeset `worm-audit:10`) and nothing else is seeded.
|
||||
|
||||
`WormAuditSettings.seed(config)` is now `@Deprecated` for PostgreSQL hosts (the script seeds row 1). It still works and remains the way for memory and H2 hosts to create row 1. The settings reader falls back to the configured values while the row is absent.
|
||||
@@ -104,11 +104,24 @@ runtime role owns the audit tables (checked through `pg_tables.tableowner = curr
|
||||
The qbit ships `src/main/resources/com/kingsrook/qbits/wormaudit/ddl/postgres/worm-audit.sql`,
|
||||
a Liquibase formatted-SQL template with placeholders for the four core audit table names, the
|
||||
app role and the retention interval, plus a renderer (`WormAuditDdl.postgres(config)`) and the
|
||||
rendered default (`docs/ddl/worm-audit-postgres.sql`, snake_case names). Hosts copy the rendered
|
||||
rendered default (`docs/ddl/worm-audit-postgres.sql`, snake_case names). Core audit physical
|
||||
columns must use snake_case (for example `audit_table_id`, `audit_id`, `new_value`); hosts map
|
||||
QQQ field names to those columns. Hosts copy the rendered
|
||||
file into their changelog (kof22: `changesets/features/YYYY-MM-DD-worm-audit.sql`). It creates the
|
||||
qbit tables, the functions, the triggers, the purger role (`CREATE ROLE ... NOLOGIN` when it is
|
||||
missing) and the grants. It is idempotent (`CREATE OR REPLACE`, `DROP TRIGGER IF EXISTS`).
|
||||
|
||||
Foreign keys (`worm-audit:11-12`) are `ON DELETE RESTRICT` only, so nothing cascades into or out
|
||||
of evidence: `worm_audit_chain.audit_id` -> the core audit header (the purge deletes details, then
|
||||
links, then headers, in that order), `worm_audit_export.from/to_checkpoint_id` -> checkpoints and
|
||||
`worm_audit_verification.export_id` -> exports (neither is ever deleted). References to chain links
|
||||
(checkpoint, export and verification ranges, `first_break_link_id`, the purge anchor) are not
|
||||
foreign keys: the purge deletes the sealed prefix those rows outlive. On upgrade, rows whose parent
|
||||
is already missing are evidence and are never changed: they are copied to `qqq_fk_quarantine`
|
||||
(action `KEPT`) and that constraint stays `NOT VALID` (new rows are still checked). With the
|
||||
foreign key, a plain `TRUNCATE` of `audit` (or of checkpoints or exports) is refused by PostgreSQL
|
||||
before the guard trigger; `TRUNCATE ... CASCADE` reaches the guards, which refuse it.
|
||||
|
||||
### 4.5 Threat model summary
|
||||
|
||||
| Actor | Can modify audit rows? | Detected by |
|
||||
@@ -179,7 +192,13 @@ chain would need the canonical form in SQL. The design instead uses **one chaine
|
||||
`kind = LATE_DETAIL`, covering exactly those detail ids. Verification flags any detail not
|
||||
covered by exactly one link.
|
||||
- Unique constraints back the lock: `UNIQUE (audit_id) WHERE kind = 'AUDIT'`, and the detail
|
||||
coverage.
|
||||
coverage: `worm_audit_chain_detail (detail_id PRIMARY KEY, chain_id)`, one append-only row per
|
||||
covered detail, written by the chain insert trigger in the same statement as the link. The
|
||||
writer finds uncovered details with a primary-key probe of it; verification still counts
|
||||
coverage from the links' own `detail_ids` and breaks if the table disagrees.
|
||||
- Each tick reads only rows above per-JVM sweep floors: the highest audit and detail ids seen at
|
||||
least (settle horizon + 5 minutes) before the last complete sweep. After a restart and once an
|
||||
hour a tick ignores the floors, so a row whose transaction commits even later is still chained.
|
||||
|
||||
Why not per-partition chains: they make each partition's chain shorter but multiply heads,
|
||||
checkpoints and verification state, and they still need a writer per partition. One chainer
|
||||
@@ -217,15 +236,19 @@ standard streamed ETL:
|
||||
does not verify".
|
||||
|
||||
**Daily verification**: a scheduled job runs the same verifier at 03:00. A failure raises
|
||||
`QAlerts.raise(new AlertSignal("worm_audit.chain.break").withDedupKey("chain")...)` with the first
|
||||
`QAlerts.raise(new AlertSignal("wormaudit.chain.break").withDedupKey("chain")...)` with the first
|
||||
break in the title and the verification record as the record ref; a later pass resolves it. Every
|
||||
run sends `QAlerts.heartbeat("worm_audit.verify.daily")`. `QAlerts` is the core producer SPI
|
||||
run sends `QAlerts.heartbeat("wormaudit.verify.daily")`. `QAlerts` is the core producer SPI
|
||||
(QQQ 4.1.0-RC.5), so the qbit needs no dependency on qbit-alerts. With no alerts implementation
|
||||
installed the alerts go to the log.
|
||||
|
||||
## 6. Sealed exports
|
||||
|
||||
### 6.1 Export job
|
||||
|
||||
**Default superseded by Owner decisions (2026-10-14) below:** lock mode and export retention
|
||||
now come from Settings, defaulting to GOVERNANCE and 30 days. The original decision text is
|
||||
retained for history.
|
||||
Daily at 02:00 (configurable), and by the table action "Export now" on `worm_audit_export`:
|
||||
1. Range: links after the last export's `to_link_id`, up to the latest checkpoint. Exports are
|
||||
contiguous by construction. The app home flags a gap (a range not covered) and lag.
|
||||
@@ -236,14 +259,17 @@ Daily at 02:00 (configurable), and by the table action "Export now" on `worm_aud
|
||||
the previous export's manifest SHA-256 (the exports form their own chain), qbit version.
|
||||
3. Each object is put through qbit-object-storage (`SystemStorage`, bucket alias from config,
|
||||
default `worm_audit`) with `LockMode.COMPLIANCE` and `retainUntil = now + retentionYears`. The
|
||||
export refuses to run, and alerts `worm_audit.export.failed`, when the bucket does not report
|
||||
export refuses to run, and alerts `wormaudit.export.failed`, when the bucket does not report
|
||||
the `OBJECT_LOCK` capability or the put does not echo the lock.
|
||||
4. On success it inserts one `worm_audit_export` row (insert-only; a failed export leaves no row,
|
||||
and its alert carries the reason).
|
||||
and its alert carries the reason). Every scheduled run sends `wormaudit.export.daily`.
|
||||
|
||||
### 6.2 Restore and verify a sealed export
|
||||
"Verify sealed export", a record action on `worm_audit_export` (standard ETL), downloads the
|
||||
objects. It checks the manifest hashes, the lock mode and retain-until of each object, the link
|
||||
exact object versions pinned in the catalog's `object_versions` map. A new version under an
|
||||
existing key is an overwrite even when bytes and retention match: verification checks the latest
|
||||
version against the pin and rejects replacements, then reads and checks the pinned version.
|
||||
It checks the manifest hashes, the lock mode and retain-until of each object, the link
|
||||
hashes recomputed from the exported content, the checkpoint signatures, and the export's link to
|
||||
the previous manifest. Then it compares each exported link and content document with the live
|
||||
chain and the live audit rows. Result lines name each difference ("link 88,412: live content
|
||||
@@ -265,7 +291,7 @@ events integrations care about); chain links are high-volume and publish none.
|
||||
- **`worm_audit_checkpoint`** [Checkpoint]: `id`, `from_link_id`, `to_link_id`, `link_count`,
|
||||
`head_hash`, `merkle_root`, `algorithm`, `key_id`, `public_key`, `signature`, `created_at`.
|
||||
- **`worm_audit_export`** [Sealed Export]: `id`, `from_link_id`, `to_link_id`,
|
||||
`from_checkpoint_id`, `to_checkpoint_id`, `bucket_alias`, `object_prefix`, `manifest_key`,
|
||||
`from_checkpoint_id`, `to_checkpoint_id`, `bucket_alias`, `object_prefix`, `object_versions`, `manifest_key`,
|
||||
`manifest_sha256`, `prev_manifest_sha256`, `object_count`, `byte_count`, `lock_mode`,
|
||||
`retain_until`, `sealed_at`.
|
||||
- **`worm_audit_verification`** [Verification]: `id`, `kind` (CHAIN, SEALED_EXPORT), `trigger`
|
||||
@@ -341,7 +367,7 @@ registers as a `MiddlewareSpecExtension` under `/qqq/v1/qbit/worm-audit/...`
|
||||
qbit's tests until the core hook ships), the audit header, the details (old, new, message),
|
||||
the canonical content and the export JSONL contain neither the canary nor any substring of it
|
||||
longer than 4 characters.
|
||||
6. **Alerts**: a break raises `worm_audit.chain.break` through `QAlerts` (captured by a test
|
||||
6. **Alerts**: a break raises `wormaudit.chain.break` through `QAlerts` (captured by a test
|
||||
sink); a later pass resolves it.
|
||||
7. Coverage gates: 70% instructions, 90% classes.
|
||||
|
||||
@@ -367,7 +393,7 @@ registers as a `MiddlewareSpecExtension` under `/qqq/v1/qbit/worm-audit/...`
|
||||
- **Bucket:** `kof22-worm-audit` on the kof22 MinIO, created by `minio-bucket-init` with
|
||||
`mc mb --with-lock --ignore-existing` and `mc retention set --default COMPLIANCE 6y`. Object
|
||||
Lock can only be enabled at bucket creation.
|
||||
- **Policy:** `kof22-app` gets `s3:PutObject`, `s3:GetObject`, `s3:ListBucket`,
|
||||
- **Policy:** `kof22-app` gets `s3:PutObject`, `s3:GetObject`, `s3:GetObjectVersion`, `s3:ListBucket`,
|
||||
`s3:GetObjectRetention`, `s3:PutObjectRetention` (needed to set the lock on each put),
|
||||
`s3:GetBucketObjectLockConfiguration` on that bucket. No delete, and no
|
||||
`s3:BypassGovernanceRetention` (irrelevant under COMPLIANCE anyway).
|
||||
@@ -381,3 +407,33 @@ registers as a `MiddlewareSpecExtension` under `/qqq/v1/qbit/worm-audit/...`
|
||||
|
||||
These are owner-run (Praetor and cluster-gitops are applied by the owner). The qbit only
|
||||
documents them.
|
||||
|
||||
## Owner decisions (2026-10-14)
|
||||
|
||||
The 2026-10-12 decisions above remain the historical design. These decisions supersede the
|
||||
COMPLIANCE/six-year export default in §6.1 and the corresponding default assumptions in
|
||||
§§2, 4.5, 9, 11, 12 and the §14 infrastructure proposal.
|
||||
|
||||
- Settings live in the application as the standard QQQ `worm_audit_settings` table, record
|
||||
ID 1, labelled WORM Audit Settings and navigated as Settings after Purges. It allows edits,
|
||||
has no append-only triggers, and uses FIELD-level core audit so every change is chained.
|
||||
Reading needs `worm_audit_settings.read`; editing additionally needs
|
||||
`worm_audit_settings.edit` and `worm_audit.settings`. New IDs and deletion are refused.
|
||||
- Defaults are seeded insert-only in new changeset `worm-audit:10`; existing changesets are
|
||||
unchanged. An administrator's edits survive subsequent seeding. Config supplies defaults
|
||||
when a row/field/table is absent. Runtime reads cache at most 30 seconds, scoped to the
|
||||
QInstance. Chain interval and UTC Quartz export/verify schedules take effect after restart.
|
||||
- kof22 starts with **GOVERNANCE, 30 days** for sealed exports. An administrator with
|
||||
`s3:BypassGovernanceRetention` can remove those copies. This is deliberately reversible;
|
||||
it does not provide COMPLIANCE's irreversible retention. Runtime credentials still receive
|
||||
no delete or bypass permission. Bucket defaults should match the selected settings.
|
||||
- Changing Settings to COMPLIANCE affects new exports. Existing GOVERNANCE versions retain
|
||||
their original mode unless an administrator extends/upgrades them. Verification checks
|
||||
the catalog's recorded mode and date, so external mode changes are reported as mismatches.
|
||||
COMPLIANCE cannot be undone. The settings change itself is audited and chained.
|
||||
- Audit retention defaults to six years independently of export retention. The home warns
|
||||
about a shorter export horizon and a mismatch with the purge function's compiled interval.
|
||||
Updating the setting does not modify the purge function; that still requires a migration.
|
||||
|
||||
The README Settings table specifies all fields, defaults, bounds and timing. Signing keys,
|
||||
public-key pins, bucket alias, prefix and backend remain host configuration, never settings.
|
||||
@@ -0,0 +1,556 @@
|
||||
-- liquibase formatted sql
|
||||
|
||||
-- changeset worm-audit:1
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_chain (
|
||||
id bigint PRIMARY KEY,
|
||||
kind text NOT NULL,
|
||||
audit_id bigint NOT NULL,
|
||||
detail_ids text NOT NULL,
|
||||
prev_hash text NOT NULL,
|
||||
link_hash text NOT NULL,
|
||||
chained_at timestamptz NOT NULL,
|
||||
CHECK (id > 0),
|
||||
CHECK (kind IN ('AUDIT', 'LATE_DETAIL')),
|
||||
CHECK (prev_hash ~ '^[0-9a-f]{64}$' AND link_hash ~ '^[0-9a-f]{64}$'),
|
||||
CHECK (detail_ids = '' OR (detail_ids ~ '^(0|[1-9][0-9]*)(,(0|[1-9][0-9]*))*$' AND cardinality(string_to_array(detail_ids, ',')) <= 250))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_checkpoint (
|
||||
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
from_link_id bigint,
|
||||
to_link_id bigint,
|
||||
link_count bigint,
|
||||
head_hash text,
|
||||
merkle_root text,
|
||||
algorithm text,
|
||||
key_id text,
|
||||
public_key text,
|
||||
signature text,
|
||||
created_at timestamptz
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_export (
|
||||
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
from_link_id bigint,
|
||||
to_link_id bigint,
|
||||
from_checkpoint_id bigint,
|
||||
to_checkpoint_id bigint,
|
||||
bucket_alias text,
|
||||
object_prefix text,
|
||||
object_versions text,
|
||||
manifest_key text,
|
||||
manifest_sha256 text,
|
||||
prev_manifest_sha256 text,
|
||||
object_count bigint,
|
||||
byte_count bigint,
|
||||
lock_mode text,
|
||||
retain_until timestamptz,
|
||||
sealed_at timestamptz
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_verification (
|
||||
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
kind text,
|
||||
"trigger" text,
|
||||
started_at timestamptz,
|
||||
finished_at timestamptz,
|
||||
from_link_id bigint,
|
||||
to_link_id bigint,
|
||||
links_checked bigint,
|
||||
outcome text,
|
||||
break_count bigint,
|
||||
first_break_link_id bigint,
|
||||
first_break_audit_id bigint,
|
||||
first_break_reason text,
|
||||
export_id bigint,
|
||||
run_by text
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_purge (
|
||||
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
requested_at timestamptz,
|
||||
requested_by text,
|
||||
cutoff timestamptz,
|
||||
retention_interval text,
|
||||
outcome text,
|
||||
reason text,
|
||||
audits_deleted bigint,
|
||||
details_deleted bigint,
|
||||
links_deleted bigint,
|
||||
anchor_link_id bigint,
|
||||
anchor_hash text
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS worm_audit_chain_audit ON public.worm_audit_chain(audit_id) WHERE kind = 'AUDIT';
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_chain_audit_all ON public.worm_audit_chain(audit_id);
|
||||
|
||||
-- changeset worm-audit:2 splitStatements:false
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_reject() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp SET timezone = 'UTC' AS $$
|
||||
DECLARE
|
||||
old_time timestamptz;
|
||||
function_owner name;
|
||||
stack text;
|
||||
row_identity text := 'all';
|
||||
BEGIN
|
||||
IF TG_OP = 'DELETE' AND current_setting('worm_audit.purging', true) = 'on' THEN
|
||||
SELECT pg_get_userbyid(proowner) INTO function_owner FROM pg_proc
|
||||
WHERE oid = 'public.worm_audit_purge(timestamptz)'::regprocedure;
|
||||
GET DIAGNOSTICS stack = PG_CONTEXT;
|
||||
IF current_user = function_owner AND position('function worm_audit_purge(timestamp with time zone)' in stack) > 0 THEN
|
||||
IF TG_RELID = '"audit"'::regclass THEN
|
||||
old_time := OLD.timestamp;
|
||||
ELSIF TG_RELID = '"audit_detail"'::regclass THEN
|
||||
SELECT timestamp INTO old_time FROM "audit" WHERE id = OLD.audit_id;
|
||||
ELSIF TG_RELID = 'public.worm_audit_chain'::regclass THEN
|
||||
SELECT timestamp INTO old_time FROM "audit" WHERE id = OLD.audit_id;
|
||||
END IF;
|
||||
IF old_time < now() - interval '6 years' THEN
|
||||
RETURN OLD;
|
||||
END IF;
|
||||
END IF;
|
||||
END IF;
|
||||
IF TG_LEVEL = 'ROW' THEN
|
||||
row_identity := OLD.id::text;
|
||||
END IF;
|
||||
RAISE EXCEPTION USING ERRCODE = 'P0001',
|
||||
MESSAGE = format('worm_audit: audit rows are write-once; %s of "%s" is not allowed (id=%s). Expired rows are removed only by worm_audit_purge().',
|
||||
TG_OP, TG_TABLE_NAME, row_identity),
|
||||
HINT = 'see qbit-worm-audit docs/DESIGN.md section 4';
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:3 splitStatements:false
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_lookup_reject() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $$
|
||||
BEGIN
|
||||
IF TG_OP = 'UPDATE' AND NEW.id IS NOT DISTINCT FROM OLD.id AND NEW.name IS NOT DISTINCT FROM OLD.name THEN
|
||||
RETURN NEW;
|
||||
END IF;
|
||||
RAISE EXCEPTION USING ERRCODE = 'P0001',
|
||||
MESSAGE = format('worm_audit: audit rows are write-once; %s of "%s" is not allowed (id=%s). Expired rows are removed only by worm_audit_purge().',
|
||||
TG_OP, TG_TABLE_NAME, OLD.id),
|
||||
HINT = 'see qbit-worm-audit docs/DESIGN.md section 4';
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:4 splitStatements:false
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'worm_audit_purger') THEN
|
||||
CREATE ROLE worm_audit_purger NOLOGIN;
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:5 splitStatements:false
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_purge(p_cutoff timestamptz) RETURNS void
|
||||
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, public, pg_temp SET timezone = 'UTC' AS $$
|
||||
DECLARE
|
||||
boundary bigint;
|
||||
smaller bigint;
|
||||
anchor text;
|
||||
audits_count bigint;
|
||||
details_count bigint;
|
||||
links_count bigint;
|
||||
BEGIN
|
||||
IF p_cutoff IS NULL OR p_cutoff > now() - interval '6 years' THEN
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome, reason)
|
||||
VALUES(now(), session_user, p_cutoff, '6 years', 'REFUSED', 'Cutoff is within retention or null');
|
||||
RETURN;
|
||||
END IF;
|
||||
-- Same lock as the Java chainer; table locks also exclude concurrent audit inserts.
|
||||
PERFORM pg_advisory_xact_lock(86753090421);
|
||||
LOCK TABLE "audit", "audit_detail", public.worm_audit_chain IN SHARE ROW EXCLUSIVE MODE;
|
||||
SELECT coalesce(min(c.id) - 1, (SELECT max(id) FROM public.worm_audit_chain)) INTO boundary
|
||||
FROM public.worm_audit_chain c LEFT JOIN "audit" a ON a.id = c.audit_id
|
||||
WHERE a.id IS NULL OR a.timestamp IS NULL OR a.timestamp >= p_cutoff;
|
||||
-- Shrink to a prefix closed over every audit's links AND all its stored details.
|
||||
LOOP
|
||||
SELECT min(c.id) - 1 INTO smaller FROM public.worm_audit_chain c
|
||||
WHERE c.id <= boundary AND (
|
||||
EXISTS (SELECT FROM public.worm_audit_chain later WHERE later.audit_id = c.audit_id AND later.id > boundary)
|
||||
OR EXISTS (SELECT FROM "audit_detail" d WHERE d.audit_id = c.audit_id AND NOT EXISTS (
|
||||
SELECT FROM public.worm_audit_chain covered WHERE covered.audit_id = c.audit_id
|
||||
AND d.id::text = ANY(string_to_array(covered.detail_ids, ',')))));
|
||||
EXIT WHEN smaller IS NULL;
|
||||
boundary := smaller;
|
||||
END LOOP;
|
||||
SELECT link_hash INTO anchor FROM public.worm_audit_chain WHERE id = boundary;
|
||||
IF anchor IS NULL THEN
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome, reason)
|
||||
VALUES(now(), session_user, p_cutoff, '6 years', 'REFUSED', 'No complete expired chain prefix');
|
||||
RETURN;
|
||||
END IF;
|
||||
PERFORM set_config('worm_audit.purging', 'on', true);
|
||||
DELETE FROM "audit_detail" d WHERE EXISTS
|
||||
(SELECT FROM public.worm_audit_chain c WHERE c.audit_id = d.audit_id AND c.id <= boundary);
|
||||
GET DIAGNOSTICS details_count = ROW_COUNT;
|
||||
-- Keep ids locally because links must be deleted before their audit headers.
|
||||
WITH removed AS (DELETE FROM public.worm_audit_chain WHERE id <= boundary RETURNING audit_id),
|
||||
removed_audits AS (DELETE FROM "audit" WHERE id IN (SELECT audit_id FROM removed) RETURNING id)
|
||||
SELECT (SELECT count(*) FROM removed), (SELECT count(*) FROM removed_audits) INTO links_count, audits_count;
|
||||
PERFORM set_config('worm_audit.purging', 'off', true);
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome,
|
||||
audits_deleted, details_deleted, links_deleted, anchor_link_id, anchor_hash)
|
||||
VALUES(now(), session_user, p_cutoff, '6 years', 'PURGED', audits_count, details_count, links_count, boundary, anchor);
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:6
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON "audit";
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON "audit"
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON "audit";
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON "audit"
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON "audit" FROM "app";
|
||||
GRANT SELECT, INSERT ON "audit" TO "app";
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON "audit_detail";
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON "audit_detail"
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON "audit_detail";
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON "audit_detail"
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON "audit_detail" FROM "app";
|
||||
GRANT SELECT, INSERT ON "audit_detail" TO "app";
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_chain;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_chain
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_chain;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_chain
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_chain FROM "app";
|
||||
GRANT SELECT, INSERT ON public.worm_audit_chain TO "app";
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_checkpoint;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_checkpoint
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_checkpoint;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_checkpoint
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_checkpoint FROM "app";
|
||||
GRANT SELECT, INSERT ON public.worm_audit_checkpoint TO "app";
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_export;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_export
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_export;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_export
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_export FROM "app";
|
||||
GRANT SELECT, INSERT ON public.worm_audit_export TO "app";
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_verification;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_verification
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_verification;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_verification
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_verification FROM "app";
|
||||
GRANT SELECT, INSERT ON public.worm_audit_verification TO "app";
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_purge;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_purge
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_purge;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_purge
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_purge FROM "app";
|
||||
GRANT SELECT, INSERT ON public.worm_audit_purge TO "app";
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON "audit_table";
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON "audit_table"
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_lookup_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON "audit_table";
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON "audit_table"
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON "audit_table" FROM "app";
|
||||
GRANT SELECT, INSERT ON "audit_table" TO "app";
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON "audit_user";
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON "audit_user"
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_lookup_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON "audit_user";
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON "audit_user"
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON "audit_user" FROM "app";
|
||||
GRANT SELECT, INSERT ON "audit_user" TO "app";
|
||||
|
||||
-- Lookups may refresh display metadata, but never identities or names.
|
||||
GRANT UPDATE (label, create_date, modify_date) ON "audit_table" TO "app";
|
||||
GRANT UPDATE (create_date, modify_date) ON "audit_user" TO "app";
|
||||
|
||||
REVOKE ALL ON FUNCTION public.worm_audit_purge(timestamptz) FROM PUBLIC;
|
||||
REVOKE ALL ON FUNCTION public.worm_audit_purge(timestamptz) FROM "app";
|
||||
GRANT EXECUTE ON FUNCTION public.worm_audit_purge(timestamptz) TO worm_audit_purger;
|
||||
|
||||
-- changeset worm-audit:7 splitStatements:false
|
||||
DO $$
|
||||
DECLARE
|
||||
table_name text;
|
||||
sequence_name text;
|
||||
BEGIN
|
||||
FOREACH table_name IN ARRAY ARRAY['"audit"', '"audit_detail"', '"audit_table"', '"audit_user"',
|
||||
'public.worm_audit_checkpoint', 'public.worm_audit_export', 'public.worm_audit_verification', 'public.worm_audit_purge'] LOOP
|
||||
sequence_name := pg_get_serial_sequence(table_name, 'id');
|
||||
IF sequence_name IS NOT NULL THEN
|
||||
EXECUTE format('GRANT USAGE ON SEQUENCE %s TO "app"', sequence_name);
|
||||
END IF;
|
||||
END LOOP;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:8 splitStatements:false
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_detail_coverage() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $$
|
||||
BEGIN
|
||||
-- Serialize even direct SQL appends; the GIN index makes overlap checks indexed.
|
||||
PERFORM pg_advisory_xact_lock(86753090421);
|
||||
IF NEW.detail_ids <> '' AND (
|
||||
EXISTS (SELECT FROM unnest(string_to_array(NEW.detail_ids, ',')) AS d(id) GROUP BY d.id HAVING count(*) > 1)
|
||||
OR EXISTS (SELECT FROM public.worm_audit_chain c
|
||||
WHERE string_to_array(c.detail_ids, ',') && string_to_array(NEW.detail_ids, ','))) THEN
|
||||
RAISE EXCEPTION USING ERRCODE = '23505', MESSAGE = 'worm_audit: audit detail is already covered by the chain';
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:9
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_chain_detail_coverage ON public.worm_audit_chain USING gin (string_to_array(detail_ids, ','));
|
||||
DROP TRIGGER IF EXISTS worm_audit_detail_coverage_guard ON public.worm_audit_chain;
|
||||
CREATE TRIGGER worm_audit_detail_coverage_guard BEFORE INSERT ON public.worm_audit_chain
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_detail_coverage();
|
||||
|
||||
-- changeset worm-audit:10
|
||||
-- Editable singleton: deliberately excluded from append-only triggers.
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_settings (
|
||||
id integer PRIMARY KEY CHECK (id = 1),
|
||||
audit_retention_years integer NOT NULL DEFAULT 6 CHECK (audit_retention_years BETWEEN 1 AND 100),
|
||||
export_lock_mode text NOT NULL DEFAULT 'GOVERNANCE' CHECK (export_lock_mode IN ('GOVERNANCE', 'COMPLIANCE')),
|
||||
export_retention_days integer NOT NULL DEFAULT 30 CHECK (export_retention_days BETWEEN 1 AND 36500),
|
||||
checkpoint_every_links integer NOT NULL DEFAULT 10000 CHECK (checkpoint_every_links > 0),
|
||||
checkpoint_every_minutes integer NOT NULL DEFAULT 60 CHECK (checkpoint_every_minutes > 0),
|
||||
chain_interval_seconds integer NOT NULL DEFAULT 30 CHECK (chain_interval_seconds > 0),
|
||||
settle_horizon_seconds integer NOT NULL DEFAULT 60 CHECK (settle_horizon_seconds >= 0),
|
||||
late_detail_lookback_hours integer NOT NULL DEFAULT 24 CHECK (late_detail_lookback_hours > 0),
|
||||
export_cron text NOT NULL DEFAULT '0 0 2 * * ?',
|
||||
verify_cron text NOT NULL DEFAULT '0 0 3 * * ?',
|
||||
notes text,
|
||||
modify_date timestamptz NOT NULL DEFAULT now(),
|
||||
modified_by text NOT NULL DEFAULT current_user
|
||||
);
|
||||
REVOKE DELETE, TRUNCATE ON public.worm_audit_settings FROM "app";
|
||||
GRANT SELECT, INSERT, UPDATE ON public.worm_audit_settings TO "app";
|
||||
INSERT INTO public.worm_audit_settings
|
||||
(id, audit_retention_years, export_lock_mode, export_retention_days, checkpoint_every_links,
|
||||
checkpoint_every_minutes, chain_interval_seconds, settle_horizon_seconds, late_detail_lookback_hours,
|
||||
export_cron, verify_cron)
|
||||
SELECT 1, 6, 'GOVERNANCE', 30, 10000, 60, 30, 60, 24, '0 0 2 * * ?', '0 0 3 * * ?'
|
||||
WHERE NOT EXISTS (SELECT 1 FROM public.worm_audit_settings WHERE id = 1);
|
||||
|
||||
-- changeset worm-audit:11
|
||||
-- comment: Index the foreign key columns added by worm-audit:12 (worm_audit_chain.audit_id already leads worm_audit_chain_audit_all).
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_export_from_checkpoint ON public.worm_audit_export(from_checkpoint_id);
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_export_to_checkpoint ON public.worm_audit_export(to_checkpoint_id);
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_verification_export ON public.worm_audit_verification(export_id);
|
||||
|
||||
-- changeset worm-audit:12 splitStatements:false
|
||||
-- comment: Foreign keys, ON DELETE RESTRICT only: nothing cascades into or out of audit evidence. A chain link needs its audit header (the purge deletes links before headers); a sealed export needs its signed checkpoints and an export verification its export (neither is ever deleted). Links referenced by checkpoints, exports, verifications and purges are not foreign keys: the purge deletes the sealed prefix those rows outlive. Rows whose parent is already missing are evidence and are never changed: they are copied to qqq_fk_quarantine (action KEPT) and the constraint is left NOT VALID (enforced for new rows only).
|
||||
DO $$
|
||||
DECLARE
|
||||
fk record;
|
||||
orphans bigint;
|
||||
BEGIN
|
||||
CREATE TABLE IF NOT EXISTS qqq_fk_quarantine (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, table_name VARCHAR(100) NOT NULL, column_name VARCHAR(100) NOT NULL, row_id VARCHAR(100), missing_parent_id VARCHAR(100), action VARCHAR(20) NOT NULL, row_data TEXT, changeset VARCHAR(200), quarantined_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP);
|
||||
FOR fk IN SELECT * FROM (VALUES
|
||||
('fk_worm_audit_chain_audit_id', 'worm_audit_chain', 'audit_id', '"audit"'),
|
||||
('fk_worm_audit_export_from_checkpoint_id', 'worm_audit_export', 'from_checkpoint_id', 'public.worm_audit_checkpoint'),
|
||||
('fk_worm_audit_export_to_checkpoint_id', 'worm_audit_export', 'to_checkpoint_id', 'public.worm_audit_checkpoint'),
|
||||
('fk_worm_audit_verification_export_id', 'worm_audit_verification', 'export_id', 'public.worm_audit_export')
|
||||
) AS t(name, child, col, parent) LOOP
|
||||
IF NOT EXISTS (SELECT FROM pg_constraint WHERE conname = fk.name) THEN
|
||||
EXECUTE format('INSERT INTO qqq_fk_quarantine (table_name, column_name, row_id, missing_parent_id, action, row_data, changeset)'
|
||||
|| ' SELECT %L, %L, c.id::text, c.%I::text, ''KEPT'', row_to_json(c)::text, ''worm-audit:12'' FROM public.%I c'
|
||||
|| ' WHERE c.%I IS NOT NULL AND NOT EXISTS (SELECT FROM %s p WHERE p.id = c.%I)', fk.child, fk.col, fk.col, fk.child, fk.col, fk.parent, fk.col);
|
||||
GET DIAGNOSTICS orphans = ROW_COUNT;
|
||||
EXECUTE format('ALTER TABLE public.%I ADD CONSTRAINT %I FOREIGN KEY (%I) REFERENCES %s (id) ON DELETE RESTRICT NOT VALID', fk.child, fk.name, fk.col, fk.parent);
|
||||
IF orphans = 0 THEN
|
||||
EXECUTE format('ALTER TABLE public.%I VALIDATE CONSTRAINT %I', fk.child, fk.name);
|
||||
END IF;
|
||||
END IF;
|
||||
END LOOP;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:13 splitStatements:false
|
||||
-- comment: Normalized detail coverage. One insert-only row per audit_detail id covered by a chain link, written by the coverage trigger in the same statement as the link, so the chain writer probes a primary key instead of the GIN array index. Append-only like the chain, with no foreign keys (the owner cannot UPDATE WORM tables, so FK checks would be denied). The advisory lock keeps a running chainer from appending between the backfill and the new trigger body.
|
||||
DO $$
|
||||
BEGIN
|
||||
PERFORM pg_advisory_xact_lock(86753090421);
|
||||
END;
|
||||
$$;
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_chain_detail (
|
||||
detail_id bigint PRIMARY KEY,
|
||||
chain_id bigint NOT NULL,
|
||||
CHECK (detail_id >= 0),
|
||||
CHECK (chain_id > 0)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_chain_detail_chain ON public.worm_audit_chain_detail(chain_id);
|
||||
-- The first link wins if a tampered chain covers a detail twice; the verifier still reports the duplicate from the chain itself.
|
||||
INSERT INTO public.worm_audit_chain_detail(detail_id, chain_id)
|
||||
SELECT x.id::bigint, c.id FROM public.worm_audit_chain c CROSS JOIN LATERAL unnest(string_to_array(c.detail_ids, ',')) AS x(id)
|
||||
WHERE c.detail_ids <> '' ORDER BY c.id
|
||||
ON CONFLICT (detail_id) DO NOTHING;
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_reject() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp SET timezone = 'UTC' AS $fn$
|
||||
DECLARE
|
||||
old_time timestamptz;
|
||||
function_owner name;
|
||||
stack text;
|
||||
row_identity text := 'all';
|
||||
BEGIN
|
||||
IF TG_OP = 'DELETE' AND current_setting('worm_audit.purging', true) = 'on' THEN
|
||||
SELECT pg_get_userbyid(proowner) INTO function_owner FROM pg_proc
|
||||
WHERE oid = 'public.worm_audit_purge(timestamptz)'::regprocedure;
|
||||
GET DIAGNOSTICS stack = PG_CONTEXT;
|
||||
IF current_user = function_owner AND position('function worm_audit_purge(timestamp with time zone)' in stack) > 0 THEN
|
||||
IF TG_RELID = '"audit"'::regclass THEN
|
||||
old_time := OLD.timestamp;
|
||||
ELSIF TG_RELID = '"audit_detail"'::regclass THEN
|
||||
SELECT timestamp INTO old_time FROM "audit" WHERE id = OLD.audit_id;
|
||||
ELSIF TG_RELID = 'public.worm_audit_chain'::regclass THEN
|
||||
SELECT timestamp INTO old_time FROM "audit" WHERE id = OLD.audit_id;
|
||||
ELSIF TG_RELID = 'public.worm_audit_chain_detail'::regclass THEN
|
||||
SELECT a.timestamp INTO old_time FROM public.worm_audit_chain c JOIN "audit" a ON a.id = c.audit_id WHERE c.id = OLD.chain_id;
|
||||
END IF;
|
||||
IF old_time < now() - interval '6 years' THEN
|
||||
RETURN OLD;
|
||||
END IF;
|
||||
END IF;
|
||||
END IF;
|
||||
IF TG_LEVEL = 'ROW' THEN
|
||||
-- Record fields resolve per statement, so each table's key is read in its own branch.
|
||||
IF TG_RELID = 'public.worm_audit_chain_detail'::regclass THEN
|
||||
row_identity := OLD.detail_id::text;
|
||||
ELSE
|
||||
row_identity := OLD.id::text;
|
||||
END IF;
|
||||
END IF;
|
||||
RAISE EXCEPTION USING ERRCODE = 'P0001',
|
||||
MESSAGE = format('worm_audit: audit rows are write-once; %s of "%s" is not allowed (id=%s). Expired rows are removed only by worm_audit_purge().',
|
||||
TG_OP, TG_TABLE_NAME, row_identity),
|
||||
HINT = 'see qbit-worm-audit docs/DESIGN.md section 4';
|
||||
END;
|
||||
$fn$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_detail_coverage() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $fn$
|
||||
BEGIN
|
||||
-- Serialize even direct SQL appends, then record coverage in the same statement as the link.
|
||||
-- The primary key rejects a detail covered twice, within this link or by an earlier one. The
|
||||
-- previous EXISTS overlap check ran as a cached generic plan that chose a sequential scan of
|
||||
-- the whole chain for every inserted link.
|
||||
PERFORM pg_advisory_xact_lock(86753090421);
|
||||
IF NEW.detail_ids = '' THEN
|
||||
RETURN NEW;
|
||||
END IF;
|
||||
BEGIN
|
||||
INSERT INTO public.worm_audit_chain_detail(detail_id, chain_id)
|
||||
SELECT d, NEW.id FROM unnest(string_to_array(NEW.detail_ids, ',')::bigint[]) AS d;
|
||||
EXCEPTION WHEN unique_violation THEN
|
||||
RAISE EXCEPTION USING ERRCODE = '23505', MESSAGE = 'worm_audit: audit detail is already covered by the chain';
|
||||
END;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$fn$;
|
||||
|
||||
-- The array index worm_audit_chain_detail_coverage is no longer used by this release. It stays until
|
||||
-- a later migration drops it, so a previous-release writer still running during a rolling deploy keeps its indexed probe.
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_chain_detail;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_chain_detail
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_chain_detail;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_chain_detail
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_chain_detail FROM "app";
|
||||
GRANT SELECT, INSERT ON public.worm_audit_chain_detail TO "app";
|
||||
|
||||
-- Same function as worm-audit:5, except that coverage rows are locked and purged before their links.
|
||||
-- The migrating role may have revoked its own EXECUTE (a runtime role that owns the schema, as on CNPG);
|
||||
-- PostgreSQL then refuses CREATE OR REPLACE. Grant it back for this statement only, then restore.
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT has_function_privilege(current_user, 'public.worm_audit_purge(timestamptz)', 'EXECUTE') THEN
|
||||
EXECUTE format('GRANT EXECUTE ON FUNCTION public.worm_audit_purge(timestamptz) TO %I', current_user);
|
||||
PERFORM set_config('worm_audit.purge_regrant', 'on', true);
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_purge(p_cutoff timestamptz) RETURNS void
|
||||
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, public, pg_temp SET timezone = 'UTC' AS $fn$
|
||||
DECLARE
|
||||
boundary bigint;
|
||||
smaller bigint;
|
||||
anchor text;
|
||||
audits_count bigint;
|
||||
details_count bigint;
|
||||
links_count bigint;
|
||||
BEGIN
|
||||
IF p_cutoff IS NULL OR p_cutoff > now() - interval '6 years' THEN
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome, reason)
|
||||
VALUES(now(), session_user, p_cutoff, '6 years', 'REFUSED', 'Cutoff is within retention or null');
|
||||
RETURN;
|
||||
END IF;
|
||||
-- Same lock as the Java chainer; table locks also exclude concurrent audit inserts.
|
||||
PERFORM pg_advisory_xact_lock(86753090421);
|
||||
LOCK TABLE "audit", "audit_detail", public.worm_audit_chain, public.worm_audit_chain_detail IN SHARE ROW EXCLUSIVE MODE;
|
||||
SELECT coalesce(min(c.id) - 1, (SELECT max(id) FROM public.worm_audit_chain)) INTO boundary
|
||||
FROM public.worm_audit_chain c LEFT JOIN "audit" a ON a.id = c.audit_id
|
||||
WHERE a.id IS NULL OR a.timestamp IS NULL OR a.timestamp >= p_cutoff;
|
||||
-- Shrink to a prefix closed over every audit's links AND all its stored details.
|
||||
LOOP
|
||||
SELECT min(c.id) - 1 INTO smaller FROM public.worm_audit_chain c
|
||||
WHERE c.id <= boundary AND (
|
||||
EXISTS (SELECT FROM public.worm_audit_chain later WHERE later.audit_id = c.audit_id AND later.id > boundary)
|
||||
OR EXISTS (SELECT FROM "audit_detail" d WHERE d.audit_id = c.audit_id AND NOT EXISTS (
|
||||
SELECT FROM public.worm_audit_chain covered WHERE covered.audit_id = c.audit_id
|
||||
AND d.id::text = ANY(string_to_array(covered.detail_ids, ',')))));
|
||||
EXIT WHEN smaller IS NULL;
|
||||
boundary := smaller;
|
||||
END LOOP;
|
||||
SELECT link_hash INTO anchor FROM public.worm_audit_chain WHERE id = boundary;
|
||||
IF anchor IS NULL THEN
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome, reason)
|
||||
VALUES(now(), session_user, p_cutoff, '6 years', 'REFUSED', 'No complete expired chain prefix');
|
||||
RETURN;
|
||||
END IF;
|
||||
PERFORM set_config('worm_audit.purging', 'on', true);
|
||||
DELETE FROM "audit_detail" d WHERE EXISTS
|
||||
(SELECT FROM public.worm_audit_chain c WHERE c.audit_id = d.audit_id AND c.id <= boundary);
|
||||
GET DIAGNOSTICS details_count = ROW_COUNT;
|
||||
-- Coverage rows go before their links: the guard finds their age through the link.
|
||||
DELETE FROM public.worm_audit_chain_detail WHERE chain_id <= boundary;
|
||||
-- Keep ids locally because links must be deleted before their audit headers.
|
||||
WITH removed AS (DELETE FROM public.worm_audit_chain WHERE id <= boundary RETURNING audit_id),
|
||||
removed_audits AS (DELETE FROM "audit" WHERE id IN (SELECT audit_id FROM removed) RETURNING id)
|
||||
SELECT (SELECT count(*) FROM removed), (SELECT count(*) FROM removed_audits) INTO links_count, audits_count;
|
||||
PERFORM set_config('worm_audit.purging', 'off', true);
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome,
|
||||
audits_deleted, details_deleted, links_deleted, anchor_link_id, anchor_hash)
|
||||
VALUES(now(), session_user, p_cutoff, '6 years', 'PURGED', audits_count, details_count, links_count, boundary, anchor);
|
||||
END;
|
||||
$fn$;
|
||||
DO $$
|
||||
BEGIN
|
||||
IF current_setting('worm_audit.purge_regrant', true) = 'on' THEN
|
||||
EXECUTE format('REVOKE EXECUTE ON FUNCTION public.worm_audit_purge(timestamptz) FROM %I', current_user);
|
||||
PERFORM set_config('worm_audit.purge_regrant', 'off', true);
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
@@ -0,0 +1,80 @@
|
||||
# WORM Audit screenshots
|
||||
|
||||
## RC.2 settings captures (2026-10-14)
|
||||
Home (GOVERNANCE, 30-day exports) and the WORM Audit Settings record at 390 x 844 and 1440 x 1000,
|
||||
light and dark, captured with Playwright against `WormAuditDemoServer` on port 8099:
|
||||
`worm-audit-home-{390,1440}-{light,dark}.jpg` (replaced) and
|
||||
`worm-audit-settings-{390,1440}-{light,dark}.jpg` (new). Overflow check on all eight: no page
|
||||
horizontal overflow, no inner horizontal scrollers, no double scrollbars.
|
||||
|
||||
## Earlier captures
|
||||
|
||||
Refreshed after record-screen polish **2026-10-13 (America/Chicago)** from the rebuilt test-scope demo using
|
||||
packaged Next UI 1.0.0-RC.14 and connected Edge through the browser screenshot
|
||||
and viewport tools. All files are JPEG viewport captures, below 300 KB each.
|
||||
|
||||
```powershell
|
||||
./mvnw.cmd -o test-compile exec:java "-Dexec.classpathScope=test" "-Dexec.mainClass=com.kingsrook.qbits.wormaudit.demo.WormAuditDemoServer" "-Dexec.args=8105"
|
||||
```
|
||||
|
||||
Open <http://localhost:8105/app/worm_audit/>. Port 8105 isolates this rebuilt instance from earlier demos. The seed creates genuine verification
|
||||
results and exports over synthetic data. Records captured: FAIL verification #8,
|
||||
PASS verification #15, sealed export #15, and chain link #329.
|
||||
|
||||
Required matrix: home at **390, 1440, 1920 px**, and all four records at **390,
|
||||
1440 px**, each in **light and dark**. Heights are 844, 1000 and 1080 px respectively.
|
||||
The existing supplementary 390 px light purge-history capture was refreshed too.
|
||||
Theme changes used Preferences and were restored to light; the temporary viewport override was reset afterward.
|
||||
|
||||
Every image was visually inspected. At each capture, the live DOM was checked after responsive layout settled for
|
||||
page-level horizontal overflow (`document.documentElement.scrollWidth > innerWidth`)
|
||||
and all rendered inner horizontal scrollers (`scrollWidth > clientWidth + 1` with
|
||||
computed `overflow-x` of `auto` or `scroll`). The scan included content below the
|
||||
viewport, including loaded child lists. Mobile verification accordions were also explicitly expanded and checked after their child rows loaded. There were **zero** page overflow failures
|
||||
and **zero** inner horizontal scrollers. Each home/verification/export view had
|
||||
one vertical scroll container (main); chain records had none. No double scrollbars
|
||||
were found. Long hashes, object versions and failure diagnostics wrap on mobile.
|
||||
No CSS or DOM overrides were injected for the captures.
|
||||
|
||||
All five tables have explicit field labels and meaningful record-label formats.
|
||||
Verification uses full-width Result, Scope, Run, and Chain links sections. Every
|
||||
child-list widget and section declares 12 grid columns. Kind and outcome use
|
||||
readable possible values; child queries translate those values too. Hashes stay
|
||||
complete because RC.5 has no separate list-only field display format. Numeric
|
||||
chain-reference labels keep the export title concise.
|
||||
|
||||
Core RC.5 shares the table label with its sidebar tree node. Existing plural
|
||||
navigation labels are retained, with singular record titles and record-section
|
||||
labels. Separate singular table labels and plural sidebar labels require core
|
||||
support; no application-wide metadata customizer was replaced to work around it.
|
||||
|
||||
`./mvnw.cmd -o verify` passed: 79 tests, no failures or errors, and 9
|
||||
Docker-dependent skips. Checkstyle and JaCoCo coverage gates passed. Metadata
|
||||
tests cover every field label, record-label formats and fields, verification
|
||||
section coverage, and 12-column sections and child widgets.
|
||||
|
||||
| Screenshot | Viewport | Size | Page horizontal overflow | Inner horizontal scrollers | Double scrollbars |
|
||||
|---|---|---|---|---|---|
|
||||
| [worm-audit-chain-link-1440-dark](worm-audit-chain-link-1440-dark.jpg) | 1440 x 1000 | 41.1 KB | None | None | None |
|
||||
| [worm-audit-chain-link-1440-light](worm-audit-chain-link-1440-light.jpg) | 1440 x 1000 | 41.4 KB | None | None | None |
|
||||
| [worm-audit-chain-link-390-dark](worm-audit-chain-link-390-dark.jpg) | 390 x 844 | 21.8 KB | None | None | None |
|
||||
| [worm-audit-chain-link-390-light](worm-audit-chain-link-390-light.jpg) | 390 x 844 | 21.8 KB | None | None | None |
|
||||
| [worm-audit-home-1440-dark](worm-audit-home-1440-dark.jpg) | 1440 x 1000 | 73.6 KB | None | None | None |
|
||||
| [worm-audit-home-1440-light](worm-audit-home-1440-light.jpg) | 1440 x 1000 | 72.4 KB | None | None | None |
|
||||
| [worm-audit-home-1920-dark](worm-audit-home-1920-dark.jpg) | 1920 x 1080 | 84.7 KB | None | None | None |
|
||||
| [worm-audit-home-1920-light](worm-audit-home-1920-light.jpg) | 1920 x 1080 | 83.6 KB | None | None | None |
|
||||
| [worm-audit-home-390-dark](worm-audit-home-390-dark.jpg) | 390 x 844 | 30.0 KB | None | None | None |
|
||||
| [worm-audit-home-390-light](worm-audit-home-390-light.jpg) | 390 x 844 | 28.8 KB | None | None | None |
|
||||
| [worm-audit-home-390-purges-light](worm-audit-home-390-purges-light.jpg) | 390 x 844 | 29.5 KB | None | None | None |
|
||||
| [worm-audit-sealed-export-1440-dark](worm-audit-sealed-export-1440-dark.jpg) | 1440 x 1000 | 117.6 KB | None | None | None |
|
||||
| [worm-audit-sealed-export-1440-light](worm-audit-sealed-export-1440-light.jpg) | 1440 x 1000 | 118.4 KB | None | None | None |
|
||||
| [worm-audit-sealed-export-390-dark](worm-audit-sealed-export-390-dark.jpg) | 390 x 844 | 38.4 KB | None | None | None |
|
||||
| [worm-audit-sealed-export-390-light](worm-audit-sealed-export-390-light.jpg) | 390 x 844 | 37.3 KB | None | None | None |
|
||||
| [worm-audit-verification-fail-1440-dark](worm-audit-verification-fail-1440-dark.jpg) | 1440 x 1000 | 53.6 KB | None | None | None |
|
||||
| [worm-audit-verification-fail-1440-light](worm-audit-verification-fail-1440-light.jpg) | 1440 x 1000 | 54.1 KB | None | None | None |
|
||||
| [worm-audit-verification-fail-390-dark](worm-audit-verification-fail-390-dark.jpg) | 390 x 844 | 28.1 KB | None | None | None |
|
||||
| [worm-audit-verification-fail-390-light](worm-audit-verification-fail-390-light.jpg) | 390 x 844 | 27.8 KB | None | None | None |
|
||||
| [worm-audit-verification-pass-1440-dark](worm-audit-verification-pass-1440-dark.jpg) | 1440 x 1000 | 49.0 KB | None | None | None |
|
||||
| [worm-audit-verification-pass-1440-light](worm-audit-verification-pass-1440-light.jpg) | 1440 x 1000 | 49.7 KB | None | None | None |
|
||||
| [worm-audit-verification-pass-390-dark](worm-audit-verification-pass-390-dark.jpg) | 390 x 844 | 23.5 KB | None | None | None |
|
||||
| [worm-audit-verification-pass-390-light](worm-audit-verification-pass-390-light.jpg) | 390 x 844 | 23.4 KB | None | None | None |
|
||||
|
After Width: | Height: | Size: 41 KiB |
|
After Width: | Height: | Size: 41 KiB |
|
After Width: | Height: | Size: 22 KiB |
|
After Width: | Height: | Size: 22 KiB |
|
After Width: | Height: | Size: 84 KiB |
|
After Width: | Height: | Size: 82 KiB |
|
After Width: | Height: | Size: 85 KiB |
|
After Width: | Height: | Size: 84 KiB |
|
After Width: | Height: | Size: 32 KiB |
|
After Width: | Height: | Size: 31 KiB |
|
After Width: | Height: | Size: 30 KiB |
|
After Width: | Height: | Size: 118 KiB |
|
After Width: | Height: | Size: 118 KiB |
|
After Width: | Height: | Size: 38 KiB |
|
After Width: | Height: | Size: 37 KiB |
|
After Width: | Height: | Size: 74 KiB |
|
After Width: | Height: | Size: 75 KiB |
|
After Width: | Height: | Size: 33 KiB |
|
After Width: | Height: | Size: 33 KiB |
|
After Width: | Height: | Size: 54 KiB |
|
After Width: | Height: | Size: 54 KiB |
|
After Width: | Height: | Size: 28 KiB |
|
After Width: | Height: | Size: 28 KiB |
|
After Width: | Height: | Size: 49 KiB |
|
After Width: | Height: | Size: 50 KiB |
|
After Width: | Height: | Size: 24 KiB |
|
After Width: | Height: | Size: 23 KiB |
@@ -27,6 +27,7 @@
|
||||
<maven.compiler.target>21</maven.compiler.target>
|
||||
<jacoco.version>0.8.15</jacoco.version>
|
||||
<h2.version>2.2.224</h2.version>
|
||||
<testcontainers.version>2.0.5</testcontainers.version>
|
||||
</properties>
|
||||
|
||||
<parent>
|
||||
@@ -81,7 +82,11 @@
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
<artifactId>qqq-backend-module-rdbms</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qbits</groupId>
|
||||
<artifactId>qbit-object-storage</artifactId>
|
||||
<version>0.1.0-RC.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
@@ -99,6 +104,30 @@
|
||||
<version>${h2.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.postgresql</groupId>
|
||||
<artifactId>postgresql</artifactId>
|
||||
<version>42.7.14</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.testcontainers</groupId>
|
||||
<artifactId>testcontainers-postgresql</artifactId>
|
||||
<version>${testcontainers.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.testcontainers</groupId>
|
||||
<artifactId>testcontainers-minio</artifactId>
|
||||
<version>${testcontainers.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.testcontainers</groupId>
|
||||
<artifactId>testcontainers-junit-jupiter</artifactId>
|
||||
<version>${testcontainers.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter</artifactId>
|
||||
|
||||
@@ -21,25 +21,558 @@
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import java.security.PrivateKey;
|
||||
import java.security.PublicKey;
|
||||
import java.util.Map;
|
||||
import java.util.function.Supplier;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configuration for the WORM audit qbit.
|
||||
** Host configuration. Private signing material never enters metadata records.
|
||||
*******************************************************************************/
|
||||
public class WormAuditQBitConfig implements QBitConfig
|
||||
{
|
||||
public static final int DEFAULT_RETENTION_YEARS = 6;
|
||||
|
||||
private MetaDataCustomizerInterface<QTableMetaData> tableMetaDataCustomizer;
|
||||
private int retentionYears = DEFAULT_RETENTION_YEARS;
|
||||
/*******************************************************************************
|
||||
** Signing identity; the matching public key must be pinned in trustedPublicKeys.
|
||||
*******************************************************************************/
|
||||
public record SigningKey(String keyId, PrivateKey privateKey)
|
||||
{
|
||||
}
|
||||
|
||||
private String exportLockMode = "GOVERNANCE";
|
||||
private int exportRetentionDays = 30;
|
||||
private final WormAuditSettings settings = new WormAuditSettings(this);
|
||||
private boolean includeApp = true;
|
||||
private String appName = "worm_audit";
|
||||
private String appLabel = "WORM Audit";
|
||||
private String parentAppName;
|
||||
private String tablePathPrefix = "/app";
|
||||
private Supplier<com.kingsrook.qbits.wormaudit.export.SealedStore> sealedStoreSupplier;
|
||||
private String backendName = null;
|
||||
private int retentionYears = 6;
|
||||
private Supplier<SigningKey> signingKey = null;
|
||||
private Map<String, PublicKey> trustedPublicKeys = Map.of();
|
||||
private int chainIntervalSeconds = 30;
|
||||
private int settleHorizonSeconds = 60;
|
||||
private int batchSize = 1000;
|
||||
private int lateDetailLookbackHours = 24;
|
||||
private int checkpointEveryLinks = 10000;
|
||||
private int checkpointEveryMinutes = 60;
|
||||
private String verifyCron = "0 0 3 * * ?";
|
||||
private String exportBucketAlias = "worm_audit";
|
||||
private String exportPrefix = "worm-audit";
|
||||
private String exportCron = "0 0 2 * * ?";
|
||||
private String schedulerName = "worm_audit";
|
||||
private Map<String, String> coreAuditTableNames = Map.of();
|
||||
private String appRole = "app";
|
||||
private MetaDataCustomizerInterface<QTableMetaData> tableMetaDataCustomizer = null;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for the optional table customizer.
|
||||
** Read exportLockMode.
|
||||
*******************************************************************************/
|
||||
public String getExportLockMode()
|
||||
{
|
||||
return (exportLockMode);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure exportLockMode.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withExportLockMode(String exportLockMode)
|
||||
{
|
||||
this.exportLockMode = exportLockMode;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read exportRetentionDays.
|
||||
*******************************************************************************/
|
||||
public int getExportRetentionDays()
|
||||
{
|
||||
return (exportRetentionDays);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure exportRetentionDays.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withExportRetentionDays(int exportRetentionDays)
|
||||
{
|
||||
this.exportRetentionDays = exportRetentionDays;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read effective settings through the instance-scoped 30-second cache.
|
||||
*******************************************************************************/
|
||||
@JsonIgnore
|
||||
public WormAuditSettings.Values effectiveSettings()
|
||||
{
|
||||
return (settings.read());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Discard this instance's cached row after a successful local settings write.
|
||||
*******************************************************************************/
|
||||
public void invalidateSettings()
|
||||
{
|
||||
settings.invalidate();
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether to create the standalone app. Widgets remain available when false.
|
||||
*******************************************************************************/
|
||||
public boolean getIncludeApp()
|
||||
{
|
||||
return (includeApp);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Disable standalone app creation when the host owns the Audit app and help.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withIncludeApp(boolean includeApp)
|
||||
{
|
||||
this.includeApp = includeApp;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the app name (the host-owned destination when includeApp is false).
|
||||
*******************************************************************************/
|
||||
public String getAppName()
|
||||
{
|
||||
return (appName);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure the app name.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withAppName(String appName)
|
||||
{
|
||||
this.appName = appName;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the app label.
|
||||
*******************************************************************************/
|
||||
public String getAppLabel()
|
||||
{
|
||||
return (appLabel);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure the app label.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withAppLabel(String appLabel)
|
||||
{
|
||||
this.appLabel = appLabel;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the optional parent app.
|
||||
*******************************************************************************/
|
||||
public String getParentAppName()
|
||||
{
|
||||
return (parentAppName);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure the optional parent app.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withParentAppName(String parentAppName)
|
||||
{
|
||||
this.parentAppName = parentAppName;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the table route prefix.
|
||||
*******************************************************************************/
|
||||
public String getTablePathPrefix()
|
||||
{
|
||||
return (tablePathPrefix);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure the table route prefix.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withTablePathPrefix(String tablePathPrefix)
|
||||
{
|
||||
this.tablePathPrefix = tablePathPrefix;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the optional storage boundary (defaults to qbit-object-storage).
|
||||
*******************************************************************************/
|
||||
@JsonIgnore
|
||||
public Supplier<com.kingsrook.qbits.wormaudit.export.SealedStore> getSealedStoreSupplier()
|
||||
{
|
||||
return (sealedStoreSupplier);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure the optional storage boundary (defaults to qbit-object-storage).
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withSealedStoreSupplier(Supplier<com.kingsrook.qbits.wormaudit.export.SealedStore> sealedStoreSupplier)
|
||||
{
|
||||
this.sealedStoreSupplier = sealedStoreSupplier;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read backendName.
|
||||
*******************************************************************************/
|
||||
public String getBackendName()
|
||||
{
|
||||
return (backendName);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure backendName.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withBackendName(String backendName)
|
||||
{
|
||||
this.backendName = backendName;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read retentionYears.
|
||||
*******************************************************************************/
|
||||
public int getRetentionYears()
|
||||
{
|
||||
return (retentionYears);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure retentionYears.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withRetentionYears(int retentionYears)
|
||||
{
|
||||
this.retentionYears = retentionYears;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read signingKey.
|
||||
*******************************************************************************/
|
||||
@JsonIgnore
|
||||
public Supplier<SigningKey> getSigningKey()
|
||||
{
|
||||
return (signingKey);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure signingKey.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withSigningKey(Supplier<SigningKey> signingKey)
|
||||
{
|
||||
this.signingKey = signingKey;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read trustedPublicKeys.
|
||||
*******************************************************************************/
|
||||
@JsonIgnore
|
||||
public Map<String, PublicKey> getTrustedPublicKeys()
|
||||
{
|
||||
return (trustedPublicKeys);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure trustedPublicKeys.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withTrustedPublicKeys(Map<String, PublicKey> trustedPublicKeys)
|
||||
{
|
||||
this.trustedPublicKeys = trustedPublicKeys;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read chainIntervalSeconds.
|
||||
*******************************************************************************/
|
||||
public int getChainIntervalSeconds()
|
||||
{
|
||||
return (chainIntervalSeconds);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure chainIntervalSeconds.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withChainIntervalSeconds(int chainIntervalSeconds)
|
||||
{
|
||||
this.chainIntervalSeconds = chainIntervalSeconds;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read settleHorizonSeconds.
|
||||
*******************************************************************************/
|
||||
public int getSettleHorizonSeconds()
|
||||
{
|
||||
return (settleHorizonSeconds);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure settleHorizonSeconds.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withSettleHorizonSeconds(int settleHorizonSeconds)
|
||||
{
|
||||
this.settleHorizonSeconds = settleHorizonSeconds;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the bounded late-detail polling window in hours.
|
||||
*******************************************************************************/
|
||||
public int getLateDetailLookbackHours()
|
||||
{
|
||||
return (lateDetailLookbackHours);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure late-detail polling; older omissions are reported by verification.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withLateDetailLookbackHours(int lateDetailLookbackHours)
|
||||
{
|
||||
this.lateDetailLookbackHours = lateDetailLookbackHours;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read batchSize.
|
||||
*******************************************************************************/
|
||||
public int getBatchSize()
|
||||
{
|
||||
return (batchSize);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure batchSize.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withBatchSize(int batchSize)
|
||||
{
|
||||
this.batchSize = batchSize;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read checkpointEveryLinks.
|
||||
*******************************************************************************/
|
||||
public int getCheckpointEveryLinks()
|
||||
{
|
||||
return (checkpointEveryLinks);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure checkpointEveryLinks.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withCheckpointEveryLinks(int checkpointEveryLinks)
|
||||
{
|
||||
this.checkpointEveryLinks = checkpointEveryLinks;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read checkpointEveryMinutes.
|
||||
*******************************************************************************/
|
||||
public int getCheckpointEveryMinutes()
|
||||
{
|
||||
return (checkpointEveryMinutes);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure checkpointEveryMinutes.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withCheckpointEveryMinutes(int checkpointEveryMinutes)
|
||||
{
|
||||
this.checkpointEveryMinutes = checkpointEveryMinutes;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read verifyCron.
|
||||
*******************************************************************************/
|
||||
public String getVerifyCron()
|
||||
{
|
||||
return (verifyCron);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the Object Lock bucket alias.
|
||||
*******************************************************************************/
|
||||
public String getExportBucketAlias()
|
||||
{
|
||||
return (exportBucketAlias);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure the Object Lock bucket alias.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withExportBucketAlias(String exportBucketAlias)
|
||||
{
|
||||
this.exportBucketAlias = exportBucketAlias;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the export key prefix.
|
||||
*******************************************************************************/
|
||||
public String getExportPrefix()
|
||||
{
|
||||
return (exportPrefix);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure the export key prefix.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withExportPrefix(String exportPrefix)
|
||||
{
|
||||
this.exportPrefix = exportPrefix;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the UTC export schedule.
|
||||
*******************************************************************************/
|
||||
public String getExportCron()
|
||||
{
|
||||
return (exportCron);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure the UTC export schedule.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withExportCron(String exportCron)
|
||||
{
|
||||
this.exportCron = exportCron;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure verifyCron.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withVerifyCron(String verifyCron)
|
||||
{
|
||||
this.verifyCron = verifyCron;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read schedulerName.
|
||||
*******************************************************************************/
|
||||
public String getSchedulerName()
|
||||
{
|
||||
return (schedulerName);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure schedulerName.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withSchedulerName(String schedulerName)
|
||||
{
|
||||
this.schedulerName = schedulerName;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read coreAuditTableNames.
|
||||
*******************************************************************************/
|
||||
public Map<String, String> getCoreAuditTableNames()
|
||||
{
|
||||
return (coreAuditTableNames);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure coreAuditTableNames.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withCoreAuditTableNames(Map<String, String> coreAuditTableNames)
|
||||
{
|
||||
this.coreAuditTableNames = coreAuditTableNames;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read appRole.
|
||||
*******************************************************************************/
|
||||
public String getAppRole()
|
||||
{
|
||||
return (appRole);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure appRole.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withAppRole(String appRole)
|
||||
{
|
||||
this.appRole = appRole;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read tableMetaDataCustomizer.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public MetaDataCustomizerInterface<QTableMetaData> getTableMetaDataCustomizer()
|
||||
@@ -48,9 +581,8 @@ public class WormAuditQBitConfig implements QBitConfig
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for the optional table customizer.
|
||||
** Configure tableMetaDataCustomizer.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withTableMetaDataCustomizer(MetaDataCustomizerInterface<QTableMetaData> tableMetaDataCustomizer)
|
||||
{
|
||||
@@ -59,23 +591,27 @@ public class WormAuditQBitConfig implements QBitConfig
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for the audit retention in years (HIPAA default: 6).
|
||||
** Reject invalid runtime limits before producing metadata.
|
||||
*******************************************************************************/
|
||||
public int getRetentionYears()
|
||||
public void validate()
|
||||
{
|
||||
return (retentionYears);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for the audit retention in years.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withRetentionYears(int retentionYears)
|
||||
{
|
||||
this.retentionYears = retentionYears;
|
||||
return (this);
|
||||
if(appName == null || !appName.matches("[A-Za-z0-9_-]+") || appLabel == null || appLabel.isBlank()
|
||||
|| (parentAppName != null && !parentAppName.matches("[A-Za-z0-9_-]+"))
|
||||
|| tablePathPrefix == null || !tablePathPrefix.matches("/[A-Za-z0-9_/-]*") || tablePathPrefix.endsWith("/"))
|
||||
{
|
||||
throw (new IllegalArgumentException("App names and a local table path prefix are required"));
|
||||
}
|
||||
if(exportBucketAlias == null || exportBucketAlias.isBlank() || exportCron == null || exportCron.isBlank()
|
||||
|| exportPrefix == null || !exportPrefix.matches("[A-Za-z0-9_-]+(/[A-Za-z0-9_-]+)*"))
|
||||
{
|
||||
throw (new IllegalArgumentException("Export bucket, prefix and schedule must be configured"));
|
||||
}
|
||||
WormAuditSettings.validate(WormAuditSettings.defaults(this));
|
||||
if(backendName == null || backendName.isBlank() || retentionYears < 1 || chainIntervalSeconds < 1
|
||||
|| lateDetailLookbackHours < 1 || settleHorizonSeconds < 0 || batchSize < 1 || checkpointEveryLinks < 1 || checkpointEveryMinutes < 1)
|
||||
{
|
||||
throw (new IllegalArgumentException("backendName is required and WORM audit limits must be positive (settle horizon may be zero)"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -21,14 +21,34 @@
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.wormaudit.app.WormAuditApp;
|
||||
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
|
||||
import com.kingsrook.qbits.wormaudit.guard.AppendOnlyGuard;
|
||||
import com.kingsrook.qbits.wormaudit.model.LinkRangeWidget;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditHelp;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.ExportProcesses;
|
||||
import com.kingsrook.qbits.wormaudit.processes.WormAuditProcesses;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertDefaultMonitor;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertSeverity;
|
||||
import com.kingsrook.qqq.backend.core.alerts.QAlertTypeMetaData;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaDataProducer;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.quartz.QuartzSchedulerMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Entry point for the WORM audit qbit; no tables are produced yet.
|
||||
** Produces append-only audit metadata, verification actions and schedules.
|
||||
*******************************************************************************/
|
||||
public class WormAuditQBitProducer implements QBitMetaDataProducer<WormAuditQBitConfig>
|
||||
{
|
||||
@@ -108,13 +128,160 @@ public class WormAuditQBitProducer implements QBitMetaDataProducer<WormAuditQBit
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Register this qbit and return its metadata, without producing tables.
|
||||
** Register audit tables, verification/export actions and schedules.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public MetaDataProducerMultiOutput produce(QInstance qInstance)
|
||||
public MetaDataProducerMultiOutput produce(QInstance qInstance) throws QException
|
||||
{
|
||||
qBitConfig.validate();
|
||||
if(qInstance.getBackend(qBitConfig.getBackendName()) == null)
|
||||
{
|
||||
throw (new IllegalArgumentException("Configured audit backend is missing"));
|
||||
}
|
||||
WormAuditDdl.resolveNames(qBitConfig, qInstance);
|
||||
for(String core : AppendOnlyGuard.CORE_TABLES)
|
||||
{
|
||||
if(!qBitConfig.getBackendName().equals(qInstance.getTable(core).getBackendName()))
|
||||
{
|
||||
throw (new IllegalArgumentException("Core and WORM audit tables must share one backend"));
|
||||
}
|
||||
}
|
||||
QBitMetaData qBit = getQBitMetaData();
|
||||
qInstance.addQBit(qBit);
|
||||
return (new MetaDataProducerMultiOutput().with(qBit));
|
||||
MetaDataProducerMultiOutput output = new MetaDataProducerMultiOutput().with(qBit);
|
||||
AppendOnlyGuard guard = new AppendOnlyGuard();
|
||||
for(QTableMetaData table : WormAuditTables.tables(qBitConfig))
|
||||
{
|
||||
if(qBitConfig.getTableMetaDataCustomizer() != null)
|
||||
{
|
||||
table = qBitConfig.getTableMetaDataCustomizer().customizeMetaData(qInstance, table);
|
||||
}
|
||||
for(var field : table.getFields().values())
|
||||
{
|
||||
if(field.getInlinePossibleValueSource() != null)
|
||||
{
|
||||
output.add(field.getInlinePossibleValueSource());
|
||||
field.withPossibleValueSourceName(field.getInlinePossibleValueSource().getName()).withInlinePossibleValueSource(null);
|
||||
}
|
||||
}
|
||||
guard.customizeMetaData(qInstance, table);
|
||||
table.setSourceQBitName(qBit.getName());
|
||||
output.add(table);
|
||||
}
|
||||
for(String core : AppendOnlyGuard.CORE_TABLES)
|
||||
{
|
||||
guard.customizeMetaData(qInstance, qInstance.getTable(core));
|
||||
}
|
||||
for(String name : WormAuditTables.ALL)
|
||||
{
|
||||
QPossibleValueSource source = QPossibleValueSource.newForTable(name);
|
||||
if(WormAuditTables.CHAIN.equals(name))
|
||||
{
|
||||
source.withValueFormat("%s").withValueFields(java.util.List.of("id"));
|
||||
}
|
||||
output.add(source);
|
||||
}
|
||||
for(String name : java.util.List.of(WormAuditTables.CHECKPOINT, WormAuditTables.EXPORT, WormAuditTables.VERIFICATION))
|
||||
{
|
||||
output.add(new QWidgetMetaData().withName(name + "_links").withLabel("Chain links").withType("childRecordList").withGridColumns(12)
|
||||
.withCodeReference(new QCodeReference(LinkRangeWidget.class)).withDefaultValue("parentTable", name));
|
||||
}
|
||||
List<QTableMetaData> producedTables = output.getEach(QTableMetaData.class);
|
||||
output.addSelfToInstance(qInstance);
|
||||
output = new MetaDataProducerMultiOutput();
|
||||
var previousInstance = QContext.getQInstance();
|
||||
var previousSession = QContext.getQSession();
|
||||
var previousTransaction = QContext.getQBackendTransaction();
|
||||
try
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
// Production is still assembling the host metadata. Do not validate it //
|
||||
// mid-build. The startup settings read is system work: run it as the //
|
||||
// core system user. A plain tokenless QSession fails session validation //
|
||||
// under token-based host authentication (OAuth2, Auth0). //
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
QContext.setQInstance(qInstance);
|
||||
QContext.setQSession(new QSystemUserSession(qInstance));
|
||||
QContext.setTransaction(null);
|
||||
addProcesses(output, qBit);
|
||||
}
|
||||
finally
|
||||
{
|
||||
QContext.setQInstance(previousInstance);
|
||||
QContext.setQSession(previousSession);
|
||||
QContext.setTransaction(previousTransaction);
|
||||
}
|
||||
if(qInstance.getSchedulers() == null || !qInstance.getSchedulers().containsKey(qBitConfig.getSchedulerName()))
|
||||
{
|
||||
qInstance.addScheduler(new QuartzSchedulerMetaData().withName(qBitConfig.getSchedulerName()));
|
||||
}
|
||||
if(qBitConfig.getIncludeApp())
|
||||
{
|
||||
output.add(WormAuditApp.app(qBitConfig, producedTables));
|
||||
}
|
||||
for(QWidgetMetaData widget : WormAuditApp.widgets())
|
||||
{
|
||||
output.add(widget);
|
||||
}
|
||||
output.add(new QAlertTypeMetaData().withName("wormaudit.chain.break").withLabel("Audit chain break")
|
||||
.withSourceQBitName(qBit.getName()).withDefaultSeverity(AlertSeverity.CRITICAL)
|
||||
.withDescription("Audit chain verification found a break in content, continuity, coverage or checkpoint trust.")
|
||||
.withSubjectTableName(WormAuditTables.VERIFICATION).withVisibilityPermission("worm_audit_verification.read"));
|
||||
output.add(new QAlertTypeMetaData().withName("wormaudit.export.failed").withLabel("Sealed audit export failed")
|
||||
.withSourceQBitName(qBit.getName()).withDefaultSeverity(AlertSeverity.ERROR)
|
||||
.withDescription("The audit export could not be sealed with verified retention and object versions.")
|
||||
.withSubjectTableName(WormAuditTables.EXPORT).withVisibilityPermission("worm_audit_export.read"));
|
||||
for(String job : List.of("verify", "export"))
|
||||
{
|
||||
String name = "wormaudit." + job + ".daily";
|
||||
output.add(new QAlertTypeMetaData().withName(name).withLabel("Daily audit " + job + " overdue")
|
||||
.withSourceQBitName(qBit.getName()).withDefaultSeverity(AlertSeverity.ERROR)
|
||||
.withDescription("The daily audit " + job + " job has not sent a heartbeat within 26 hours.")
|
||||
.withVisibilityPermission(job.equals("verify") ? "worm_audit_verification.read" : "worm_audit_export.read")
|
||||
.withDefaultMonitor(AlertDefaultMonitor.heartbeat(name, name, 24 * 3600, 2 * 3600)));
|
||||
}
|
||||
output.addSelfToInstance(qInstance);
|
||||
WormAuditHelp.apply(qInstance, qBitConfig);
|
||||
denyAssistant(qInstance);
|
||||
return (new MetaDataProducerMultiOutput());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build schedules from settings available at producer startup.
|
||||
*******************************************************************************/
|
||||
private void addProcesses(MetaDataProducerMultiOutput output, QBitMetaData qBit)
|
||||
{
|
||||
for(var process : WormAuditProcesses.processes(qBitConfig))
|
||||
{
|
||||
process.setSourceQBitName(qBit.getName());
|
||||
output.add(process);
|
||||
}
|
||||
for(var process : ExportProcesses.processes(qBitConfig))
|
||||
{
|
||||
process.setSourceQBitName(qBit.getName());
|
||||
output.add(process);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** RC.5 compatibility: replace with a direct call when the qbit builds on RC.6 (qqq#1141, #1163).
|
||||
*******************************************************************************/
|
||||
private void denyAssistant(QInstance instance) throws QException
|
||||
{
|
||||
try
|
||||
{
|
||||
Class<?> type = Class.forName("com.kingsrook.qqq.backend.core.model.metadata.permissions.ClientChannelsMetaData");
|
||||
Object metadata = type.getMethod("ofOrWithNew", QInstance.class).invoke(null, instance);
|
||||
type.getMethod("withDeniedPattern", String.class, String.class).invoke(metadata, "assistant", "worm_audit_*");
|
||||
}
|
||||
catch(ClassNotFoundException expected)
|
||||
{
|
||||
// The assistant wall API is absent in RC.5.
|
||||
}
|
||||
catch(ReflectiveOperationException e)
|
||||
{
|
||||
throw (new QException("Cannot register the WORM audit assistant deny pattern", e));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import java.sql.SQLException;
|
||||
import java.time.Clock;
|
||||
import java.time.Instant;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import org.quartz.CronExpression;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Singleton settings merged over host defaults. Only absent rows/tables fall
|
||||
** back; database outages and invalid persisted settings fail closed.
|
||||
*******************************************************************************/
|
||||
public class WormAuditSettings
|
||||
{
|
||||
private final WormAuditQBitConfig config;
|
||||
private final Clock clock;
|
||||
private QInstance instance;
|
||||
private QRecord cached;
|
||||
private Instant expires = Instant.MIN;
|
||||
|
||||
/*******************************************************************************
|
||||
** An immutable snapshot for one run.
|
||||
*******************************************************************************/
|
||||
public record Values(int audit_retention_years,
|
||||
String export_lock_mode,
|
||||
int export_retention_days,
|
||||
int checkpoint_every_links,
|
||||
int checkpoint_every_minutes,
|
||||
int chain_interval_seconds,
|
||||
int settle_horizon_seconds,
|
||||
int late_detail_lookback_hours,
|
||||
String export_cron,
|
||||
String verify_cron)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Use the production clock.
|
||||
*******************************************************************************/
|
||||
public WormAuditSettings(WormAuditQBitConfig config)
|
||||
{
|
||||
this(config, Clock.systemUTC());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Inject a clock for cache boundary tests.
|
||||
*******************************************************************************/
|
||||
public WormAuditSettings(WormAuditQBitConfig config, Clock clock)
|
||||
{
|
||||
this.config = config;
|
||||
this.clock = clock;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Invalidate after a local edit; other application instances refresh within 30 seconds.
|
||||
*******************************************************************************/
|
||||
public synchronized void invalidate()
|
||||
{
|
||||
expires = Instant.MIN;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Seed the singleton only when absent; never overwrite an administrator edit.
|
||||
** The database script (changeset worm-audit:10) is the normal seed; this remains
|
||||
** for memory and H2 hosts, whose settings table cannot take generic inserts.
|
||||
**
|
||||
** @deprecated for PostgreSQL hosts, which get row 1 from the database script;
|
||||
** still the supported way to create row 1 on memory and H2 hosts. Reads already
|
||||
** fall back to the configured values while the row is absent.
|
||||
*******************************************************************************/
|
||||
@Deprecated
|
||||
public static void seed(WormAuditQBitConfig config) throws com.kingsrook.qqq.backend.core.exceptions.QException
|
||||
{
|
||||
var store = new AuditStore(null);
|
||||
if(store.get(WormAuditTables.SETTINGS, 1) == null)
|
||||
{
|
||||
store.insert(WormAuditTables.SETTINGS, java.util.List.of(defaults(config)));
|
||||
}
|
||||
config.invalidateSettings();
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read at most once per 30 seconds, isolated by QInstance identity.
|
||||
*******************************************************************************/
|
||||
public synchronized Values read()
|
||||
{
|
||||
QInstance current = QContext.getQInstance();
|
||||
if(current != instance || !clock.instant().isBefore(expires))
|
||||
{
|
||||
QRecord row = null;
|
||||
if(current != null && current.getTable(WormAuditTables.SETTINGS) != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
row = new AuditStore(null).get(WormAuditTables.SETTINGS, 1);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
if(!missingTable(e))
|
||||
{
|
||||
throw (new IllegalStateException("Cannot read WORM Audit Settings", e));
|
||||
}
|
||||
}
|
||||
}
|
||||
cached = row;
|
||||
instance = current;
|
||||
expires = row == null ? Instant.MIN : clock.instant().plusSeconds(30);
|
||||
}
|
||||
QRecord row = defaults(config);
|
||||
if(cached != null)
|
||||
{
|
||||
for(String field : row.getValues().keySet())
|
||||
{
|
||||
if(cached.getValue(field) != null)
|
||||
{
|
||||
row.setValue(field, cached.getValue(field));
|
||||
}
|
||||
}
|
||||
}
|
||||
validate(row);
|
||||
return (new Values(row.getValueInteger("audit_retention_years"),
|
||||
row.getValueString("export_lock_mode"),
|
||||
row.getValueInteger("export_retention_days"),
|
||||
row.getValueInteger("checkpoint_every_links"),
|
||||
row.getValueInteger("checkpoint_every_minutes"),
|
||||
row.getValueInteger("chain_interval_seconds"),
|
||||
row.getValueInteger("settle_horizon_seconds"),
|
||||
row.getValueInteger("late_detail_lookback_hours"),
|
||||
row.getValueString("export_cron"),
|
||||
row.getValueString("verify_cron")));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build insert-only seed values from the host defaults.
|
||||
*******************************************************************************/
|
||||
public static QRecord defaults(WormAuditQBitConfig config)
|
||||
{
|
||||
return (new QRecord().withValue("id", 1)
|
||||
.withValue("audit_retention_years", config.getRetentionYears())
|
||||
.withValue("export_lock_mode", config.getExportLockMode())
|
||||
.withValue("export_retention_days", config.getExportRetentionDays())
|
||||
.withValue("checkpoint_every_links", config.getCheckpointEveryLinks())
|
||||
.withValue("checkpoint_every_minutes", config.getCheckpointEveryMinutes())
|
||||
.withValue("chain_interval_seconds", config.getChainIntervalSeconds())
|
||||
.withValue("settle_horizon_seconds", config.getSettleHorizonSeconds())
|
||||
.withValue("late_detail_lookback_hours", config.getLateDetailLookbackHours())
|
||||
.withValue("export_cron", config.getExportCron())
|
||||
.withValue("verify_cron", config.getVerifyCron()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Validate supplied fields; partial updates leave omitted fields untouched.
|
||||
*******************************************************************************/
|
||||
public static void validate(QRecord row)
|
||||
{
|
||||
if(row.getValue("id") != null && !Integer.valueOf(1).equals(row.getValueInteger("id")))
|
||||
{
|
||||
throw (new IllegalArgumentException("WORM Audit Settings has one row only (ID 1)."));
|
||||
}
|
||||
range(row, "audit_retention_years", "Audit retention years", 1, 100);
|
||||
range(row, "export_retention_days", "Export retention days", 1, 36500);
|
||||
range(row, "checkpoint_every_links", "Checkpoint every links", 1, 2147483647);
|
||||
range(row, "checkpoint_every_minutes", "Checkpoint every minutes", 1, 2147483647);
|
||||
range(row, "chain_interval_seconds", "Chain interval seconds", 1, 2147483647);
|
||||
range(row, "settle_horizon_seconds", "Settle horizon seconds", 0, 2147483647);
|
||||
range(row, "late_detail_lookback_hours", "Late detail lookback hours", 1, 2147483647);
|
||||
if(row.getValues().containsKey("export_lock_mode") && !java.util.Arrays.asList("GOVERNANCE", "COMPLIANCE").contains(row.getValueString("export_lock_mode")))
|
||||
{
|
||||
throw (new IllegalArgumentException("Export lock mode must be GOVERNANCE or COMPLIANCE."));
|
||||
}
|
||||
for(String field : java.util.List.of("export_cron", "verify_cron"))
|
||||
{
|
||||
if(row.getValues().containsKey(field) && (row.getValueString(field) == null || !CronExpression.isValidExpression(row.getValueString(field))))
|
||||
{
|
||||
throw (new IllegalArgumentException((field.equals("export_cron") ? "Export" : "Verify") + " schedule must be a valid Quartz cron expression (UTC)."));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reject null, fractional and out-of-range numbers with a plain field label.
|
||||
*******************************************************************************/
|
||||
private static void range(QRecord row, String field, String label, int minimum, int maximum)
|
||||
{
|
||||
if(!row.getValues().containsKey(field))
|
||||
{
|
||||
return;
|
||||
}
|
||||
try
|
||||
{
|
||||
long value = Long.parseLong(String.valueOf(row.getValue(field)));
|
||||
if(value >= minimum && value <= maximum)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
catch(NumberFormatException ignored)
|
||||
{
|
||||
// Use the same clear message for invalid numbers and invalid ranges.
|
||||
}
|
||||
throw (new IllegalArgumentException(label + " must be a whole number from " + minimum + " to " + maximum + "."));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Recognize PostgreSQL/H2 missing-table errors without hiding other failures.
|
||||
*******************************************************************************/
|
||||
private static boolean missingTable(Throwable error)
|
||||
{
|
||||
for(Throwable cause = error; cause != null; cause = cause.getCause())
|
||||
{
|
||||
if(cause instanceof SQLException sql && ("42P01".equals(sql.getSQLState()) || "42S02".equals(sql.getSQLState()) || "42S04".equals(sql.getSQLState())))
|
||||
{
|
||||
return (true);
|
||||
}
|
||||
}
|
||||
return (false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.app;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSTransaction;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read-only PostgreSQL catalog probes; errors become value-free diagnostics.
|
||||
*******************************************************************************/
|
||||
public final class PostgresSetupWarnings
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private PostgresSetupWarnings()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Probe the runtime connection, never the configured migration role.
|
||||
*******************************************************************************/
|
||||
public static List<String> load(QBackendTransaction transaction, WormAuditQBitConfig config)
|
||||
{
|
||||
List<String> warnings = new ArrayList<>();
|
||||
int years = config.effectiveSettings().audit_retention_years();
|
||||
if(years != config.getRetentionYears())
|
||||
{
|
||||
warnings.add("Audit retention setting differs from the configured database purge interval (" + config.getRetentionYears() + " years). A reviewed migration is required.");
|
||||
}
|
||||
if(!(transaction instanceof RDBMSTransaction rdbms))
|
||||
{
|
||||
return (warnings);
|
||||
}
|
||||
try
|
||||
{
|
||||
var connection = rdbms.getConnection();
|
||||
if(!"PostgreSQL".equals(connection.getMetaData().getDatabaseProductName()))
|
||||
{
|
||||
return (warnings);
|
||||
}
|
||||
// The live function is authoritative on PostgreSQL, rather than the renderer defaults.
|
||||
warnings.clear();
|
||||
List<String> tables = new ArrayList<>(config.getCoreAuditTableNames().values());
|
||||
tables.addAll(WormAuditTables.ALL.stream().filter(t -> !WormAuditTables.SETTINGS.equals(t)).toList());
|
||||
for(String table : tables)
|
||||
{
|
||||
for(String trigger : List.of("worm_audit_row_guard", "worm_audit_truncate_guard"))
|
||||
{
|
||||
try(var statement = connection.prepareStatement("SELECT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid=to_regclass(?) AND tgname=? AND NOT tgisinternal AND tgenabled IN ('O','A'))"))
|
||||
{
|
||||
statement.setString(1, WormAuditDdl.identifier(table));
|
||||
statement.setString(2, trigger);
|
||||
try(var result = statement.executeQuery())
|
||||
{
|
||||
result.next();
|
||||
if(!result.getBoolean(1))
|
||||
{
|
||||
warnings.add("Missing or disabled " + trigger + " on " + table);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
try(var statement = connection.prepareStatement("SELECT EXISTS (SELECT 1 FROM pg_tables WHERE tableowner=current_user AND to_regclass(quote_ident(schemaname)||'.'||quote_ident(tablename))=to_regclass(?))"))
|
||||
{
|
||||
statement.setString(1, WormAuditDdl.identifier(table));
|
||||
try(var result = statement.executeQuery())
|
||||
{
|
||||
result.next();
|
||||
if(result.getBoolean(1))
|
||||
{
|
||||
warnings.add("Runtime role owns audit table " + table);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
try(var statement = connection.createStatement(); var result = statement.executeQuery("SELECT pg_get_functiondef(to_regprocedure('public.worm_audit_purge(timestamp with time zone)'))"))
|
||||
{
|
||||
if(result.next() && result.getString(1) != null)
|
||||
{
|
||||
var interval = java.util.regex.Pattern.compile("(?i)interval\\s+'([0-9]+) years'").matcher(result.getString(1));
|
||||
if(interval.find() && Integer.parseInt(interval.group(1)) != years)
|
||||
{
|
||||
warnings.add("Audit retention setting differs from the purge function's compiled interval (" + interval.group(1) + " years). A reviewed migration is required.");
|
||||
}
|
||||
}
|
||||
}
|
||||
try(var statement = connection.createStatement(); var result = statement.executeQuery("SELECT to_regprocedure('public.worm_audit_purge(timestamp with time zone)') IS NOT NULL"))
|
||||
{
|
||||
result.next();
|
||||
if(!result.getBoolean(1))
|
||||
{
|
||||
warnings.add("Retention purge function is missing");
|
||||
}
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
warnings.add("PostgreSQL setup checks unavailable");
|
||||
}
|
||||
return (warnings);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.app;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.VerificationRuns;
|
||||
import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.CustomPermissionChecker;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML;
|
||||
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** WORM Audit navigation and standard HTML widgets, scoped to this QInstance.
|
||||
*******************************************************************************/
|
||||
public final class WormAuditApp
|
||||
{
|
||||
public static final String HOME_WIDGET = "wormAuditHome";
|
||||
public static final String HEALTH_CARD_WIDGET = "wormAuditHealthCard";
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private WormAuditApp()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The home and optional host-dashboard summary share the app's permissions.
|
||||
*******************************************************************************/
|
||||
public static List<QWidgetMetaData> widgets()
|
||||
{
|
||||
return (List.of(
|
||||
new QWidgetMetaData().withName(HOME_WIDGET).withLabel("Overview").withType(WidgetType.HTML.getType())
|
||||
.withCodeReference(new QCodeReference(HomeRenderer.class)).withGridColumns(12).withIsCard(false).withPermissionRules(permissions()),
|
||||
new QWidgetMetaData().withName(HEALTH_CARD_WIDGET).withLabel("WORM Audit").withType(WidgetType.HTML.getType())
|
||||
.withCodeReference(new QCodeReference(HealthCardRenderer.class)).withGridColumns(12).withIsCard(true).withPermissionRules(permissions())));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Home, Verifications, Sealed Exports, Checkpoints, Audit Chain, Purges.
|
||||
*******************************************************************************/
|
||||
public static QAppMetaData app(WormAuditQBitConfig config, List<QTableMetaData> tables)
|
||||
{
|
||||
QAppMetaData app = new QAppMetaData().withName(config.getAppName()).withLabel(config.getAppLabel())
|
||||
.withIcon(new QIcon("verified_user")).withPermissionRules(permissions()).withWidgets(List.of(HOME_WIDGET));
|
||||
app.setParentAppName(config.getParentAppName());
|
||||
for(String name : WormAuditTables.NAV)
|
||||
{
|
||||
QTableMetaData table = tables.stream().filter(t -> t.getName().equals(name)).findFirst().orElseThrow();
|
||||
app.withSectionOfChildren(new QAppSection().withName(name).withLabel(WormAuditTables.navigationLabel(name)),
|
||||
WormAuditTables.SETTINGS.equals(name) ? table.clone().withLabel("Settings") : table);
|
||||
}
|
||||
return (app);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read on any WORM table grants the value-free aggregate overview.
|
||||
*******************************************************************************/
|
||||
public static QPermissionRules permissions()
|
||||
{
|
||||
return (new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION)
|
||||
.withCustomPermissionChecker(new QCodeReference(HomePermissionChecker.class)));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Honor the host's actual table permission rules, including custom checkers.
|
||||
*******************************************************************************/
|
||||
public static class HomePermissionChecker implements CustomPermissionChecker
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Reject direct widget requests unless at least one table is readable.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void checkPermissionsThrowing(AbstractActionInput input, MetaDataWithPermissionRules metadata) throws QPermissionDeniedException
|
||||
{
|
||||
for(String table : WormAuditTables.ALL)
|
||||
{
|
||||
try
|
||||
{
|
||||
PermissionsHelper.checkTablePermissionThrowing(new QueryInput(table), TablePermissionSubType.READ);
|
||||
return;
|
||||
}
|
||||
catch(QPermissionDeniedException expected)
|
||||
{
|
||||
// Another WORM table may grant access to the aggregate overview.
|
||||
}
|
||||
}
|
||||
throw (new QPermissionDeniedException("Read access to a WORM audit table is required"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Full-width home renderer.
|
||||
*******************************************************************************/
|
||||
public static class HomeRenderer extends AbstractWidgetRenderer
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Render only metadata, counts and value-free reasons.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public RenderWidgetOutput render(RenderWidgetInput input) throws QException
|
||||
{
|
||||
new HomePermissionChecker().checkPermissionsThrowing(input, null);
|
||||
return (new RenderWidgetOutput(new RawHTML(null, WormAuditHomeView.render(WormAuditHomeData.load(), VerificationRuns.config().getTablePathPrefix()))));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Optional compact host dashboard renderer.
|
||||
*******************************************************************************/
|
||||
public static class HealthCardRenderer extends AbstractWidgetRenderer
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Link the status summary to the owning app home.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public RenderWidgetOutput render(RenderWidgetInput input) throws QException
|
||||
{
|
||||
new HomePermissionChecker().checkPermissionsThrowing(input, null);
|
||||
WormAuditQBitConfig config = VerificationRuns.config();
|
||||
String path = config.getTablePathPrefix() + "/" + (config.getParentAppName() == null ? "" : config.getParentAppName() + "/") + config.getAppName();
|
||||
return (new RenderWidgetOutput(new RawHTML(config.getAppLabel(), WormAuditHomeView.renderHealthCard(WormAuditHomeData.load(), path))));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.app;
|
||||
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.export.ObjectStorageSealedStore;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedStore;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.VerificationRuns;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSTransaction;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read-only home snapshot. No audit messages, detail values or canonical content
|
||||
** enter this model. Unexported suffixes are lag; holes before the sealed head
|
||||
** are gaps. Only a later full scheduled chain PASS clears an open chain break.
|
||||
*******************************************************************************/
|
||||
public record WormAuditHomeData(long links, Instant lastLinkAt, long backlog, long settling, int settleSeconds,
|
||||
QRecord lastVerification, QRecord openBreak, QRecord lastExport, long exportLag, Long exportAgeHours,
|
||||
List<String> exportGaps, String bucketAlias, String lockCapability, String lockReason,
|
||||
String exportLockMode, int exportRetentionDays, int retentionYears, QRecord anchor, Instant oldestAudit, Instant nextPurge,
|
||||
List<QRecord> recentPurges, List<String> warnings)
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Load using the current instance's configuration and production clock.
|
||||
*******************************************************************************/
|
||||
public static WormAuditHomeData load() throws QException
|
||||
{
|
||||
return (load(VerificationRuns.config(), null, Instant.now()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The storage and time seams also serve the memory demo and deterministic tests.
|
||||
*******************************************************************************/
|
||||
public static WormAuditHomeData load(WormAuditQBitConfig config, SealedStore storage, Instant now) throws QException
|
||||
{
|
||||
var settings = config.effectiveSettings();
|
||||
List<String> warnings = new ArrayList<>();
|
||||
if("GOVERNANCE".equals(settings.export_lock_mode()))
|
||||
{
|
||||
warnings.add("Sealed exports use GOVERNANCE mode: an administrator with s3:BypassGovernanceRetention can delete them. Switch to COMPLIANCE for write-once copies.");
|
||||
}
|
||||
if(now.plus(Duration.ofDays(settings.export_retention_days())).isBefore(now.atZone(ZoneOffset.UTC).plusYears(settings.audit_retention_years()).toInstant()))
|
||||
{
|
||||
warnings.add("Export retention is shorter than audit retention.");
|
||||
}
|
||||
String capability = "UNKNOWN";
|
||||
String reason = "Object Lock capability could not be confirmed";
|
||||
try
|
||||
{
|
||||
SealedStore resolved = storage != null ? storage : config.getSealedStoreSupplier() == null
|
||||
? new ObjectStorageSealedStore(config.getExportBucketAlias(), "home") : config.getSealedStoreSupplier().get();
|
||||
CapabilityStatus status = resolved.lockCapability();
|
||||
if(status == CapabilityStatus.SUPPORTED || status == CapabilityStatus.UNSUPPORTED)
|
||||
{
|
||||
capability = status.name();
|
||||
reason = status == CapabilityStatus.SUPPORTED ? null : "Export bucket does not support Object Lock";
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
// Provider exception messages can contain credentials or object content.
|
||||
reason = "Object Lock capability check failed";
|
||||
}
|
||||
if(reason != null)
|
||||
{
|
||||
warnings.add(reason);
|
||||
}
|
||||
if(config.getSigningKey() == null)
|
||||
{
|
||||
warnings.add("No signing key configured");
|
||||
}
|
||||
if(config.getTrustedPublicKeys() == null || config.getTrustedPublicKeys().isEmpty())
|
||||
{
|
||||
warnings.add("No trusted public keys configured");
|
||||
}
|
||||
try(QBackendTransaction transaction = QBackendTransaction.openFor(new QueryInput(WormAuditTables.CHAIN)))
|
||||
{
|
||||
try
|
||||
{
|
||||
if(transaction instanceof RDBMSTransaction rdbms)
|
||||
{
|
||||
rdbms.getConnection().setTransactionIsolation(Connection.TRANSACTION_REPEATABLE_READ);
|
||||
}
|
||||
AuditStore store = new AuditStore(transaction);
|
||||
QRecord lastLink = store.last(WormAuditTables.CHAIN);
|
||||
long links = store.count(WormAuditTables.CHAIN, new QQueryFilter());
|
||||
Instant cutoff = now.minusSeconds(settings.settle_horizon_seconds());
|
||||
long backlog = store.unchainedCount(cutoff, true);
|
||||
long settling = store.unchainedCount(cutoff, false);
|
||||
QRecord verification = latest(store, WormAuditTables.VERIFICATION, new QQueryFilter(), "finished_at");
|
||||
QQueryFilter dailyFilter = new QQueryFilter(new QFilterCriteria("kind", QCriteriaOperator.EQUALS, "CHAIN"),
|
||||
new QFilterCriteria("trigger", QCriteriaOperator.EQUALS, "SCHEDULE"));
|
||||
QRecord daily = latest(store, WormAuditTables.VERIFICATION, dailyFilter, "finished_at");
|
||||
if(daily == null || daily.getValueInstant("finished_at") == null)
|
||||
{
|
||||
warnings.add("Daily verification has never run");
|
||||
}
|
||||
else if(daily.getValueInstant("finished_at").isBefore(now.minus(Duration.ofHours(26))))
|
||||
{
|
||||
warnings.add("Daily verification last ran more than 26 hours ago");
|
||||
}
|
||||
QRecord failure = latest(store, WormAuditTables.VERIFICATION, new QQueryFilter(
|
||||
new QFilterCriteria("kind", QCriteriaOperator.EQUALS, "CHAIN"), new QFilterCriteria("outcome", QCriteriaOperator.EQUALS, "FAIL")), "id");
|
||||
QRecord fullPass = latest(store, WormAuditTables.VERIFICATION, new QQueryFilter(
|
||||
new QFilterCriteria("kind", QCriteriaOperator.EQUALS, "CHAIN"), new QFilterCriteria("trigger", QCriteriaOperator.EQUALS, "SCHEDULE"),
|
||||
new QFilterCriteria("outcome", QCriteriaOperator.EQUALS, "PASS")), "id");
|
||||
QRecord openBreak = failure != null && (fullPass == null || failure.getValueLong("id") > fullPass.getValueLong("id")) ? failure : null;
|
||||
QRecord export = store.last(WormAuditTables.EXPORT);
|
||||
QRecord anchor = store.anchor();
|
||||
long anchorId = anchor.getValueLong("anchor_link_id");
|
||||
long exported = export == null ? anchorId : export.getValueLong("to_link_id");
|
||||
long lag = store.count(WormAuditTables.CHAIN, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, exported)));
|
||||
List<String> gaps = gaps(store, anchorId);
|
||||
List<QRecord> oldest = store.summary("audit", new QQueryFilter().withOrderBy(new QFilterOrderBy("timestamp", true)).withLimit(1), Set.of("timestamp"));
|
||||
Instant oldestAt = oldest.isEmpty() ? null : oldest.getFirst().getValueInstant("timestamp");
|
||||
List<QRecord> purges = store.query(WormAuditTables.PURGE, new QQueryFilter()
|
||||
.withOrderBy(new QFilterOrderBy("requested_at", false)).withOrderBy(new QFilterOrderBy("id", false)).withLimit(10));
|
||||
warnings.addAll(PostgresSetupWarnings.load(transaction, config));
|
||||
transaction.commit();
|
||||
return (new WormAuditHomeData(links, lastLink == null ? null : lastLink.getValueInstant("chained_at"), backlog, settling,
|
||||
settings.settle_horizon_seconds(), verification, openBreak, export, lag,
|
||||
export == null || export.getValueInstant("sealed_at") == null ? null : Math.max(0, Duration.between(export.getValueInstant("sealed_at"), now).toHours()),
|
||||
List.copyOf(gaps), config.getExportBucketAlias(), capability, reason, settings.export_lock_mode(), settings.export_retention_days(), settings.audit_retention_years(), anchorId == 0 ? null : anchor,
|
||||
oldestAt, oldestAt == null ? null : oldestAt.atZone(ZoneOffset.UTC).plusYears(settings.audit_retention_years()).toInstant(), List.copyOf(purges), List.copyOf(warnings)));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
transaction.rollback();
|
||||
throw (new QException("Cannot load WORM audit home summary"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read one result, using id to break equal-time ties deterministically.
|
||||
*******************************************************************************/
|
||||
private static QRecord latest(AuditStore store, String table, QQueryFilter filter, String time) throws QException
|
||||
{
|
||||
List<QRecord> rows = store.query(table, filter.withOrderBy(new QFilterOrderBy(time, false))
|
||||
.withOrderBy(new QFilterOrderBy("id", false)).withLimit(1));
|
||||
return (rows.isEmpty() ? null : rows.getFirst());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Sweep export intervals in bounded pages; never enumerate links or objects.
|
||||
** The writer appends ranges in id order. Overlapping ranges extend coverage.
|
||||
*******************************************************************************/
|
||||
private static List<String> gaps(AuditStore store, long anchor) throws QException
|
||||
{
|
||||
List<String> gaps = new ArrayList<>();
|
||||
long next = anchor + 1;
|
||||
long cursor = 0;
|
||||
while(true)
|
||||
{
|
||||
List<QRecord> page = store.page(WormAuditTables.EXPORT, cursor, 1000);
|
||||
if(page.isEmpty())
|
||||
{
|
||||
return (gaps);
|
||||
}
|
||||
for(QRecord export : page)
|
||||
{
|
||||
long from = export.getValueLong("from_link_id");
|
||||
long to = export.getValueLong("to_link_id");
|
||||
if(from > next)
|
||||
{
|
||||
gaps.add(next + "–" + (from - 1));
|
||||
}
|
||||
next = Math.max(next, to + 1);
|
||||
}
|
||||
cursor = page.getLast().getValueLong("id");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.app;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stacked, wrapping definition lists. Colors follow BackupsHomeView exactly:
|
||||
** inherited text/link colors, translucent neutral borders, muted opacity. Only
|
||||
** allowlisted summary fields are rendered; no audit content is ever inspected.
|
||||
*******************************************************************************/
|
||||
public final class WormAuditHomeView
|
||||
{
|
||||
private static final DateTimeFormatter WHEN = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm 'UTC'").withZone(ZoneOffset.UTC);
|
||||
private static final String CARD = "border:1px solid rgba(127,127,127,.3);border-radius:8px;padding:1rem;min-width:0;overflow-wrap:anywhere;";
|
||||
private static final String LINK = "color:inherit;text-decoration:underline;text-underline-offset:2px;";
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private WormAuditHomeView()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Full home with explicit empty states and no fixed-width content.
|
||||
*******************************************************************************/
|
||||
public static String render(WormAuditHomeData data, String prefix)
|
||||
{
|
||||
StringBuilder html = new StringBuilder("<div class=\"worm-audit-home\" style=\"display:grid;gap:1rem;min-width:0;max-width:100%;\">");
|
||||
open(html, "Chain status");
|
||||
row(html, "Links chained", data.links());
|
||||
row(html, "Last link", when(data.lastLinkAt(), "No links chained yet"));
|
||||
row(html, "Unchained backlog", data.backlog() + " settled; " + data.settling() + " settling (" + data.settleSeconds() + " s horizon)");
|
||||
QRecord verification = data.lastVerification();
|
||||
if(verification == null)
|
||||
{
|
||||
row(html, "Last verification", "No verification has run yet");
|
||||
}
|
||||
else
|
||||
{
|
||||
linkedRow(html, "Last verification", recordLink(prefix, WormAuditTables.VERIFICATION, verification.getValueLong("id"), "Verification #" + verification.getValueLong("id")));
|
||||
row(html, "Verified at", when(verification.getValueInstant("finished_at"), "Not recorded"));
|
||||
row(html, "Result", verification.getValueString("outcome") + "; " + verification.getValueLong("links_checked") + " links checked");
|
||||
}
|
||||
QRecord broken = data.openBreak();
|
||||
if(broken == null)
|
||||
{
|
||||
row(html, "Open chain breaks", verification == null ? "Unknown until verification runs" : "None reported");
|
||||
}
|
||||
else
|
||||
{
|
||||
linkedRow(html, "Open chain breaks", recordLink(prefix, WormAuditTables.VERIFICATION, broken.getValueLong("id"), broken.getValueLong("break_count") + " reported"));
|
||||
linkedRow(html, "First break link", recordLink(prefix, WormAuditTables.CHAIN, broken.getValueLong("first_break_link_id"), "Link #" + broken.getValueLong("first_break_link_id")));
|
||||
row(html, "First break audit", broken.getValueLong("first_break_audit_id"));
|
||||
row(html, "Reason", broken.getValueString("first_break_reason"));
|
||||
}
|
||||
close(html);
|
||||
open(html, "Export status");
|
||||
row(html, "Export lock mode", data.exportLockMode());
|
||||
row(html, "Export retention", data.exportRetentionDays() + " days");
|
||||
QRecord export = data.lastExport();
|
||||
if(export == null)
|
||||
{
|
||||
row(html, "Last sealed export", "No sealed exports yet");
|
||||
}
|
||||
else
|
||||
{
|
||||
linkedRow(html, "Last sealed export", recordLink(prefix, WormAuditTables.EXPORT, export.getValueLong("id"), "Export #" + export.getValueLong("id")));
|
||||
row(html, "Link range", export.getValueLong("from_link_id") + "–" + export.getValueLong("to_link_id"));
|
||||
row(html, "Sealed at", when(export.getValueInstant("sealed_at"), "Not recorded"));
|
||||
row(html, "Retain until", when(export.getValueInstant("retain_until"), "Not recorded"));
|
||||
row(html, "Last export lock mode", export.getValueString("lock_mode"));
|
||||
}
|
||||
row(html, "Export lag", data.exportLag() + " links; " + (data.exportAgeHours() == null ? "no previous export time" : data.exportAgeHours() + " h since last export"));
|
||||
row(html, "Gaps in sealed ranges", data.exportGaps().isEmpty() ? (export == null ? "No sealed ranges to check" : "None") : String.join(", ", data.exportGaps()));
|
||||
row(html, "Bucket alias", data.bucketAlias());
|
||||
row(html, "Object Lock capability", data.lockCapability() + (data.lockReason() == null ? "" : ": " + data.lockReason()));
|
||||
close(html);
|
||||
open(html, "Retention");
|
||||
row(html, "Configured retention", data.retentionYears() + " years");
|
||||
QRecord anchor = data.anchor();
|
||||
row(html, "Purge anchor", anchor == null ? "None" : "Link #" + anchor.getValueLong("anchor_link_id") + "; hash " + shortHash(anchor.getValueString("anchor_hash")));
|
||||
row(html, "Oldest live audit", when(data.oldestAudit(), "No live audits"));
|
||||
row(html, "Next eligible purge", when(data.nextPurge(), "No live audits to purge"));
|
||||
close(html);
|
||||
open(html, "Recent purges");
|
||||
if(data.recentPurges().isEmpty())
|
||||
{
|
||||
row(html, "History", "No purge requests yet");
|
||||
}
|
||||
for(QRecord purge : data.recentPurges())
|
||||
{
|
||||
linkedRow(html, "Purge", recordLink(prefix, WormAuditTables.PURGE, purge.getValueLong("id"), "Purge #" + purge.getValueLong("id")));
|
||||
row(html, "Requested at", when(purge.getValueInstant("requested_at"), "Not recorded"));
|
||||
row(html, "Requested by", purge.getValueString("requested_by"));
|
||||
row(html, "Outcome", purge.getValueString("outcome"));
|
||||
row(html, "Cutoff", when(purge.getValueInstant("cutoff"), "Not recorded"));
|
||||
row(html, "Deleted", purge.getValueLong("audits_deleted") + " audits; " + purge.getValueLong("details_deleted") + " details; " + purge.getValueLong("links_deleted") + " links");
|
||||
row(html, "Reason", purge.getValueString("reason"));
|
||||
}
|
||||
close(html);
|
||||
if(!data.warnings().isEmpty())
|
||||
{
|
||||
open(html, "Setup warnings");
|
||||
for(String warning : data.warnings())
|
||||
{
|
||||
row(html, "Warning", warning);
|
||||
}
|
||||
close(html);
|
||||
}
|
||||
return (html.append("</div>").toString());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Honest compact status, owned by and linked to the full home.
|
||||
*******************************************************************************/
|
||||
public static String renderHealthCard(WormAuditHomeData data, String homePath)
|
||||
{
|
||||
String status = data.openBreak() != null ? "Chain breaks reported" : data.lastVerification() == null ? "Not verified yet"
|
||||
: data.warnings().isEmpty() ? "No open chain breaks reported" : "Setup needs attention";
|
||||
return ("<div style=\"" + CARD + "\"><strong>" + esc(status) + "</strong><p>" + data.links() + " links; " + data.backlog()
|
||||
+ " settled audits waiting; " + data.exportLag() + " links awaiting export.</p><a style=\"" + LINK + "\" href=\"" + esc(homePath) + "\">Open WORM Audit</a></div>");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Section and definition list, with fluid wrapping label/value rows.
|
||||
*******************************************************************************/
|
||||
private static void open(StringBuilder html, String title)
|
||||
{
|
||||
html.append("<section style=\"").append(CARD).append("\"><h3 style=\"margin:0 0 .75rem;font-size:1rem;font-weight:600;\">")
|
||||
.append(esc(title)).append("</h3><dl style=\"margin:0;font-size:.9rem;\">");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Close a section.
|
||||
*******************************************************************************/
|
||||
private static void close(StringBuilder html)
|
||||
{
|
||||
html.append("</dl></section>");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Escape every value, including reasons and runtime identities.
|
||||
*******************************************************************************/
|
||||
private static void row(StringBuilder html, String label, Object value)
|
||||
{
|
||||
linkedRow(html, label, esc(value == null ? "Not recorded" : value.toString()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Only internally built and escaped links may supply HTML here.
|
||||
*******************************************************************************/
|
||||
private static void linkedRow(StringBuilder html, String label, String value)
|
||||
{
|
||||
html.append("<div style=\"display:flex;flex-wrap:wrap;gap:.25rem .75rem;margin:.4rem 0;min-width:0;\"><dt style=\"opacity:.75;flex:0 1 12rem;\">")
|
||||
.append(esc(label)).append("</dt><dd style=\"margin:0;flex:1 1 14rem;min-width:0;overflow-wrap:anywhere;\">").append(value).append("</dd></div>");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Standard QQQ record route; identifiers are numeric, never content.
|
||||
*******************************************************************************/
|
||||
private static String recordLink(String prefix, String table, Long id, String text)
|
||||
{
|
||||
return (id == null ? "Not recorded" : "<a style=\"" + LINK + "\" href=\"" + esc(prefix + "/" + table + "/" + id) + "\">" + esc(text) + "</a>");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** UTC time with an explicit empty state.
|
||||
*******************************************************************************/
|
||||
private static String when(Instant time, String empty)
|
||||
{
|
||||
return (time == null ? empty : WHEN.format(time));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Keep hashes compact even on a narrow screen.
|
||||
*******************************************************************************/
|
||||
private static String shortHash(String hash)
|
||||
{
|
||||
return (hash == null ? "Not recorded" : hash.substring(0, Math.min(12, hash.length())) + "…");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** HTML text and quoted attribute escaping.
|
||||
*******************************************************************************/
|
||||
private static String esc(String text)
|
||||
{
|
||||
return (text == null ? "" : text.replace("&", "&").replace("<", "<").replace(">", ">").replace("\"", """).replace("'", "'"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.canonical;
|
||||
|
||||
|
||||
import java.time.ZoneOffset;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Comparator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** v1 audit content, exactly as stored. Subsets use the same document schema for
|
||||
** LATE_DETAIL links; no canonical content is persisted by this qbit.
|
||||
*******************************************************************************/
|
||||
public final class AuditCanonical
|
||||
{
|
||||
private static final DateTimeFormatter TIMESTAMP = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'").withZone(ZoneOffset.UTC);
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private AuditCanonical()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve names outside this function, without looking up application data.
|
||||
*******************************************************************************/
|
||||
public static byte[] bytes(QRecord audit, String table, String user, List<QRecord> details)
|
||||
{
|
||||
Map<String, Object> header = new LinkedHashMap<>();
|
||||
header.put("clientChannel", audit.getValueString("clientChannel"));
|
||||
header.put("id", audit.getValueLong("id"));
|
||||
header.put("message", audit.getValueString("message"));
|
||||
header.put("recordId", audit.getValueString("recordId"));
|
||||
header.put("table", table);
|
||||
header.put("timestamp", TIMESTAMP.format(audit.getValueInstant("timestamp")));
|
||||
header.put("user", user);
|
||||
List<Map<String, Object>> rows = details.stream().sorted(Comparator.comparing(r -> r.getValueLong("id"))).map(detail ->
|
||||
{
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
row.put("id", detail.getValueLong("id"));
|
||||
for(String field : List.of("fieldName", "message", "newValue", "oldValue"))
|
||||
{
|
||||
row.put(field, detail.getValueString(field));
|
||||
}
|
||||
return (row);
|
||||
}).toList();
|
||||
return (Jcs.bytes(Map.of("audit", header, "details", rows, "v", 1)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.canonical;
|
||||
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** RFC 8785 serialization of the audit schema: strings, integer longs, null,
|
||||
** objects and arrays. Java String ordering is UTF-16 code-unit ordering. Reject
|
||||
** malformed surrogate pairs instead of silently replacing them during UTF-8 encoding.
|
||||
*******************************************************************************/
|
||||
public final class Jcs
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private Jcs()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Encode a document with no insignificant whitespace.
|
||||
*******************************************************************************/
|
||||
public static byte[] bytes(Object value)
|
||||
{
|
||||
return (string(value).getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Serialize only the value types supported by the v1 schema.
|
||||
*******************************************************************************/
|
||||
public static String string(Object value)
|
||||
{
|
||||
if(value == null)
|
||||
{
|
||||
return ("null");
|
||||
}
|
||||
if(value instanceof String text)
|
||||
{
|
||||
return (quote(text));
|
||||
}
|
||||
if(value instanceof Long || value instanceof Integer)
|
||||
{
|
||||
return (value.toString());
|
||||
}
|
||||
if(value instanceof List<?> list)
|
||||
{
|
||||
return ("[" + String.join(",", list.stream().map(Jcs::string).toList()) + "]");
|
||||
}
|
||||
if(value instanceof Map<?, ?> map)
|
||||
{
|
||||
TreeMap<String, Object> sorted = new TreeMap<>();
|
||||
map.forEach((key, item) -> sorted.put((String) key, item));
|
||||
return ("{" + String.join(",", sorted.entrySet().stream().map(e -> quote(e.getKey()) + ":" + string(e.getValue())).toList()) + "}");
|
||||
}
|
||||
throw (new IllegalArgumentException("Unsupported canonical value type"));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** JSON escaping mandated by JCS, including lowercase control escapes.
|
||||
*******************************************************************************/
|
||||
private static String quote(String text)
|
||||
{
|
||||
StringBuilder out = new StringBuilder("\"");
|
||||
for(int i = 0; i < text.length(); i++)
|
||||
{
|
||||
char c = text.charAt(i);
|
||||
switch(c)
|
||||
{
|
||||
case '"' -> out.append("\\\"");
|
||||
case '\\' -> out.append("\\\\");
|
||||
case '\b' -> out.append("\\b");
|
||||
case '\f' -> out.append("\\f");
|
||||
case '\n' -> out.append("\\n");
|
||||
case '\r' -> out.append("\\r");
|
||||
case '\t' -> out.append("\\t");
|
||||
default ->
|
||||
{
|
||||
if(c < 32)
|
||||
{
|
||||
out.append(String.format(java.util.Locale.ROOT, "\\u%04x", (int) c));
|
||||
}
|
||||
else if(Character.isHighSurrogate(c))
|
||||
{
|
||||
if(i + 1 >= text.length() || !Character.isLowSurrogate(text.charAt(i + 1)))
|
||||
{
|
||||
throw (new IllegalArgumentException("Unpaired surrogate in canonical string"));
|
||||
}
|
||||
out.append(c).append(text.charAt(++i));
|
||||
}
|
||||
else if(Character.isLowSurrogate(c))
|
||||
{
|
||||
throw (new IllegalArgumentException("Unpaired surrogate in canonical string"));
|
||||
}
|
||||
else
|
||||
{
|
||||
out.append(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return (out.append('"').toString());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,638 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.chain;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.Timestamp;
|
||||
import java.sql.Types;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import com.kingsrook.qbits.wormaudit.canonical.AuditCanonical;
|
||||
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
|
||||
import com.kingsrook.qbits.wormaudit.guard.AppendOnlyGuard;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.VerificationResult;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSTransaction;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** QQQ storage shared by chainer, verifier and future export code. All reads use
|
||||
** the caller's transaction; only insert operations exist here. Bypass applies
|
||||
** to internal catalog work after process permission checks, never user requests.
|
||||
*******************************************************************************/
|
||||
public class AuditStore
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Every (detail id, link id) pair the chain's own detail lists cover.
|
||||
*******************************************************************************/
|
||||
private static final String CHAIN_PAIRS = "SELECT x.id::bigint AS detail_id, c.id AS chain_id FROM worm_audit_chain c"
|
||||
+ " CROSS JOIN LATERAL unnest(string_to_array(c.detail_ids, ',')) AS x(id) WHERE c.detail_ids <> ''";
|
||||
|
||||
/*******************************************************************************
|
||||
** Chain coverage counted per detail id, aggregated in one pass.
|
||||
*******************************************************************************/
|
||||
private static final String CHAIN_COVERAGE = "SELECT detail_id AS id, count(*) AS n FROM (" + CHAIN_PAIRS + ") pairs GROUP BY detail_id";
|
||||
|
||||
private final QBackendTransaction transaction;
|
||||
private final Map<String, String> names = new LinkedHashMap<>(16, 0.75F, true);
|
||||
private Boolean timestampWithZone;
|
||||
private Boolean postgres;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Bind operations to one snapshot/transaction, or the memory backend.
|
||||
*******************************************************************************/
|
||||
public AuditStore(QBackendTransaction transaction)
|
||||
{
|
||||
this.transaction = transaction;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read in deterministic id order, explicitly fetching TEXT fields.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> query(String table, QQueryFilter filter) throws QException
|
||||
{
|
||||
QueryInput input = new QueryInput(table).withFilter(filter).withTransaction(transaction).withShouldFetchHeavyFields(true);
|
||||
input.withPermissionBypass("WORM audit internal read");
|
||||
return (new QueryAction().execute(input).getRecords());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Project only summary fields; home reads never fetch audit content.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> summary(String table, QQueryFilter filter, Set<String> fields) throws QException
|
||||
{
|
||||
QueryInput input = new QueryInput(table).withFilter(filter).withTransaction(transaction)
|
||||
.withFieldNamesToInclude(fields).withShouldFetchHeavyFields(true);
|
||||
input.withPermissionBypass("WORM audit home summary after widget permission check");
|
||||
return (new QueryAction().execute(input).getRecords());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Aggregate counts avoid loading the live chain for the home.
|
||||
*******************************************************************************/
|
||||
public long count(String table, QQueryFilter filter) throws QException
|
||||
{
|
||||
CountInput input = new CountInput(table).withFilter(filter).withTransaction(transaction);
|
||||
input.withPermissionBypass("WORM audit home count after widget permission check");
|
||||
return (new CountAction().execute(input).getCount());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Count the same AUDIT anti-join as the writer, split at the settle horizon.
|
||||
*******************************************************************************/
|
||||
public long unchainedCount(Instant cutoff, boolean settled) throws QException
|
||||
{
|
||||
if(isRdbms())
|
||||
{
|
||||
String sql = "SELECT count(*) FROM " + physical("audit") + " a WHERE a." + column("audit", "timestamp")
|
||||
+ (settled ? " < ?" : " >= ?") + unchainedPredicate();
|
||||
return (sqlIds(sql, cutoff).getFirst());
|
||||
}
|
||||
Set<Long> covered = summary(WormAuditTables.CHAIN, new QQueryFilter(new QFilterCriteria("kind", QCriteriaOperator.EQUALS, "AUDIT")), Set.of("audit_id"))
|
||||
.stream().map(r -> r.getValueLong("audit_id")).collect(Collectors.toSet());
|
||||
return (summary("audit", new QQueryFilter(), Set.of("id", "timestamp")).stream()
|
||||
.filter(r -> r.getValueInstant("timestamp") != null && r.getValueInstant("timestamp").isBefore(cutoff) == settled
|
||||
&& !covered.contains(r.getValueLong("id"))).count());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Shared SQL coverage predicate for both writer selection and home counts.
|
||||
*******************************************************************************/
|
||||
private String unchainedPredicate()
|
||||
{
|
||||
return (" AND NOT EXISTS (SELECT 1 FROM worm_audit_chain c WHERE c.audit_id=a." + column("audit", "id") + " AND c.kind='AUDIT')");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** All records ordered by id.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> all(String table) throws QException
|
||||
{
|
||||
return (query(table, new QQueryFilter().withOrderBy(new QFilterOrderBy("id", true))));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** An equality query.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> where(String table, String field, Serializable value) throws QException
|
||||
{
|
||||
// Core lookup keys are INTEGER while audit and chain keys are LONG.
|
||||
// Memory equality is type-sensitive; normalize to the declared field type.
|
||||
if(value instanceof Number number && QContext.getQInstance().getTable(table).getField(field).getType()
|
||||
== com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType.INTEGER)
|
||||
{
|
||||
value = Math.toIntExact(number.longValue());
|
||||
}
|
||||
return (query(table, new QQueryFilter(new QFilterCriteria(field, QCriteriaOperator.EQUALS, value)).withOrderBy(new QFilterOrderBy("id", true))));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Return a row or null if a tamper removed it.
|
||||
*******************************************************************************/
|
||||
public QRecord get(String table, long id) throws QException
|
||||
{
|
||||
return (where(table, "id", id).stream().findFirst().orElse(null));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Insert a batch, propagating failures so a transaction can roll back.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> insert(String table, List<QRecord> rows) throws QException
|
||||
{
|
||||
if(rows.isEmpty())
|
||||
{
|
||||
return (rows);
|
||||
}
|
||||
try
|
||||
{
|
||||
if(WormAuditTables.ALL.contains(table) && QContext.getQInstance().getBackendForTable(table)
|
||||
instanceof com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData)
|
||||
{
|
||||
// RC.5 binds Instant as a UTC LocalDateTime, which shifts timestamptz
|
||||
// in non-UTC sessions. Its Timestamp path preserves the actual instant.
|
||||
rows.forEach(row -> row.getValues().replaceAll((field, value) -> value instanceof Instant instant ? Timestamp.from(instant) : value));
|
||||
}
|
||||
InsertInput input = new InsertInput(table).withRecords(rows).withTransaction(transaction).withOmitDmlAudit(true);
|
||||
input.withPermissionBypass("WORM audit internal append");
|
||||
List<QRecord> inserted = new InsertAction().execute(input).getRecords();
|
||||
for(QRecord row : inserted)
|
||||
{
|
||||
if(row.getErrors() != null && !row.getErrors().isEmpty())
|
||||
{
|
||||
// Never echo a rejected record value into logs or alerts.
|
||||
throw (new QException("WORM audit append failed for " + table));
|
||||
}
|
||||
}
|
||||
return (inserted);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw (AppendOnlyGuard.map(e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Parse explicit coverage; duplicates are retained so verification sees them.
|
||||
*******************************************************************************/
|
||||
public static List<Long> detailIds(QRecord link)
|
||||
{
|
||||
String ids = link.getValueString("detail_ids");
|
||||
return (ids == null || ids.isEmpty() ? List.of() : java.util.Arrays.stream(ids.split(",")).map(Long::valueOf).toList());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve a lookup name without touching application tables.
|
||||
*******************************************************************************/
|
||||
public String name(String table, Long id) throws QException
|
||||
{
|
||||
if(id == null)
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
String key = table + ":" + id;
|
||||
if(names.containsKey(key))
|
||||
{
|
||||
return (names.get(key));
|
||||
}
|
||||
QRecord row = get(table, id);
|
||||
if(row == null)
|
||||
{
|
||||
throw (new QException("Missing " + table + " " + id));
|
||||
}
|
||||
if(names.size() >= 2000)
|
||||
{
|
||||
names.remove(names.keySet().iterator().next());
|
||||
}
|
||||
names.put(key, row.getValueString("name"));
|
||||
return (row.getValueString("name"));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Canonical bytes for the exact detail ids recorded on a link.
|
||||
*******************************************************************************/
|
||||
public byte[] canonical(QRecord audit, List<Long> ids) throws QException
|
||||
{
|
||||
return (canonical(audit, ids, index(in("auditDetail", "id", ids))));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Settled unchained headers. RDBMS uses an anti-join under the writer lock;
|
||||
** memory uses QQQ queries and the same ordering and cutoff semantics.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> unchained(Instant cutoff, int limit) throws QException
|
||||
{
|
||||
return (unchained(cutoff, limit, 0L));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Settled unchained headers above a swept audit id floor (RDBMS only); every
|
||||
** settled header at or below the floor is already known to be chained.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> unchained(Instant cutoff, int limit, long floor) throws QException
|
||||
{
|
||||
if(isRdbms())
|
||||
{
|
||||
String sql = "SELECT a." + column("audit", "id") + " FROM " + physical("audit") + " a WHERE a." + column("audit", "id") + " > ? AND a."
|
||||
+ column("audit", "timestamp") + " < ?" + unchainedPredicate() + " ORDER BY a." + column("audit", "timestamp") + ", a." + column("audit", "id")
|
||||
+ " LIMIT " + limit;
|
||||
Map<Long, QRecord> rows = index(in("audit", "id", sqlIds(sql, floor, cutoff)));
|
||||
return (rows.values().stream().sorted(Comparator.comparing((QRecord r) -> r.getValueInstant("timestamp"))
|
||||
.thenComparing(r -> r.getValueLong("id"))).toList());
|
||||
}
|
||||
var covered = all(WormAuditTables.CHAIN).stream().filter(r -> "AUDIT".equals(r.getValueString("kind"))).map(r -> r.getValueLong("audit_id")).collect(java.util.stream.Collectors.toSet());
|
||||
return (all("audit").stream().filter(r -> r.getValueInstant("timestamp").isBefore(cutoff) && !covered.contains(r.getValueLong("id")))
|
||||
.sorted(Comparator.comparing((QRecord r) -> r.getValueInstant("timestamp")).thenComparing(r -> r.getValueLong("id"))).limit(limit).toList());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Latest successful retention anchor, or genesis.
|
||||
*******************************************************************************/
|
||||
public QRecord anchor() throws QException
|
||||
{
|
||||
List<QRecord> rows = query(WormAuditTables.PURGE, new QQueryFilter(new QFilterCriteria("outcome", QCriteriaOperator.EQUALS, "PURGED"))
|
||||
.withOrderBy(new QFilterOrderBy("id", false)).withLimit(1));
|
||||
return (rows.isEmpty() ? new QRecord().withValue("anchor_link_id", 0L).withValue("anchor_hash", Hashes.GENESIS) : rows.getFirst());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether SQL anti-joins are available in this snapshot.
|
||||
*******************************************************************************/
|
||||
public boolean isRdbms()
|
||||
{
|
||||
return (transaction instanceof RDBMSTransaction);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Only the in-memory backend may use an unbounded completeness index.
|
||||
*******************************************************************************/
|
||||
public boolean isMemory()
|
||||
{
|
||||
return ("memory".equals(QContext.getQInstance().getBackendForTable(WormAuditTables.CHAIN).getBackendType()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fetch one keyset page, never using an offset or a whole-chain read.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> page(String table, long after, int limit) throws QException
|
||||
{
|
||||
return (query(table, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, after))
|
||||
.withOrderBy(new QFilterOrderBy("id", true)).withLimit(limit)));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fetch just the tail row.
|
||||
*******************************************************************************/
|
||||
public QRecord last(String table) throws QException
|
||||
{
|
||||
List<QRecord> rows = query(table, new QQueryFilter().withOrderBy(new QFilterOrderBy("id", false)).withLimit(1));
|
||||
return (rows.isEmpty() ? null : rows.getFirst());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One batched IN query, with memory-backend integer key normalization.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> in(String table, String field, Collection<Long> ids) throws QException
|
||||
{
|
||||
if(ids.isEmpty())
|
||||
{
|
||||
return (List.of());
|
||||
}
|
||||
boolean integer = QContext.getQInstance().getTable(table).getField(field).getType()
|
||||
== com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType.INTEGER;
|
||||
List<Serializable> values = ids.stream().distinct().map(id -> integer ? (Serializable) Math.toIntExact(id) : id).toList();
|
||||
return (query(table, new QQueryFilter(new QFilterCriteria(field, QCriteriaOperator.IN, values)).withOrderBy(new QFilterOrderBy("id", true))));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Index one bounded batch by primary key.
|
||||
*******************************************************************************/
|
||||
public static Map<Long, QRecord> index(List<QRecord> rows)
|
||||
{
|
||||
return (rows.stream().collect(Collectors.toMap(r -> r.getValueLong("id"), r -> r)));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Canonicalize already fetched content; a missing detail never triggers a query.
|
||||
*******************************************************************************/
|
||||
public byte[] canonical(QRecord audit, List<Long> ids, Map<Long, QRecord> details) throws QException
|
||||
{
|
||||
List<QRecord> rows = new ArrayList<>();
|
||||
for(long id : ids)
|
||||
{
|
||||
QRecord detail = details.get(id);
|
||||
if(detail == null || !audit.getValueLong("id").equals(detail.getValueLong("auditId")))
|
||||
{
|
||||
throw (new QException("audit_detail " + id + " is missing or belongs to another audit"));
|
||||
}
|
||||
rows.add(detail);
|
||||
}
|
||||
return (AuditCanonical.bytes(audit, name("auditTable", audit.getValueLong("auditTableId")), name("auditUser", audit.getValueLong("auditUserId")), rows));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Poll uncovered details in a bounded time window; memory retains its simple fallback.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> lateDetails(Instant cutoff, int hours, int limit) throws QException
|
||||
{
|
||||
return (lateDetails(cutoff, hours, limit, 0L));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Uncovered details above a swept detail id floor (RDBMS only). Coverage is a
|
||||
** primary-key probe of the normalized coverage table, never an array scan.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> lateDetails(Instant cutoff, int hours, int limit, long floor) throws QException
|
||||
{
|
||||
if(isRdbms())
|
||||
{
|
||||
String detailId = "d." + column("auditDetail", "id");
|
||||
String sql = "SELECT " + detailId + " FROM " + physical("auditDetail") + " d JOIN " + physical("audit")
|
||||
+ " a ON a." + column("audit", "id") + "=d." + column("auditDetail", "auditId")
|
||||
+ " WHERE " + detailId + " > ? AND a." + column("audit", "timestamp") + " < ? AND a." + column("audit", "timestamp") + " >= ?"
|
||||
+ " AND EXISTS (SELECT 1 FROM worm_audit_chain h WHERE h.kind='AUDIT' AND h.audit_id=a." + column("audit", "id") + ")"
|
||||
+ " AND NOT EXISTS (SELECT 1 FROM " + WormAuditTables.CHAIN_DETAIL + " x WHERE x.detail_id=" + detailId + ") ORDER BY " + detailId + " LIMIT " + limit;
|
||||
return (in("auditDetail", "id", sqlIds(sql, floor, cutoff, cutoff.minusSeconds(hours * 3600L))));
|
||||
}
|
||||
List<QRecord> links = all(WormAuditTables.CHAIN);
|
||||
Set<Long> headers = links.stream().filter(r -> "AUDIT".equals(r.getValueString("kind"))).map(r -> r.getValueLong("audit_id")).collect(Collectors.toSet());
|
||||
Set<Long> covered = new HashSet<>();
|
||||
links.forEach(link -> covered.addAll(detailIds(link)));
|
||||
Set<Long> settled = all("audit").stream().filter(a -> headers.contains(a.getValueLong("id")) && a.getValueInstant("timestamp").isBefore(cutoff))
|
||||
.map(a -> a.getValueLong("id")).collect(Collectors.toSet());
|
||||
return (all("auditDetail").stream().filter(d -> settled.contains(d.getValueLong("auditId")) && !covered.contains(d.getValueLong("id"))).toList());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** SQL completeness counts all offenders but returns only the first 100 of each type.
|
||||
*******************************************************************************/
|
||||
public void completeness(Instant cutoff, VerificationResult result) throws QException
|
||||
{
|
||||
String auditId = "a." + column("audit", "id");
|
||||
String time = "a." + column("audit", "timestamp");
|
||||
String headers = "SELECT " + auditId + " AS id, (SELECT count(*) FROM worm_audit_chain c WHERE c.kind='AUDIT' AND c.audit_id=" + auditId
|
||||
+ ") AS coverage FROM " + physical("audit") + " a WHERE " + time + " < ? OR " + time + " IS NULL";
|
||||
String detailId = "d." + column("auditDetail", "id");
|
||||
String scope = " FROM " + physical("auditDetail") + " d LEFT JOIN " + physical("audit") + " a ON " + auditId + "=d."
|
||||
+ column("auditDetail", "auditId");
|
||||
String details;
|
||||
if(isPostgres())
|
||||
{
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
// Count coverage from the chain itself (the hashed source of truth), //
|
||||
// aggregating every link's ids once rather than probing per detail. //
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
details = "SELECT " + detailId + " AS id, coalesce(cv.n, 0) AS coverage" + scope + " LEFT JOIN (" + CHAIN_COVERAGE
|
||||
+ ") cv ON cv.id=" + detailId + " WHERE " + time + " < ? OR " + time + " IS NULL";
|
||||
}
|
||||
else
|
||||
{
|
||||
details = "SELECT " + detailId + " AS id, (SELECT count(*) FROM worm_audit_chain c WHERE POSITION(',' || CAST(" + detailId
|
||||
+ " AS varchar) || ',' IN ',' || c.detail_ids || ',') > 0) AS coverage" + scope + " WHERE " + time + " < ? OR " + time + " IS NULL";
|
||||
}
|
||||
reportCoverage(headers, "audit", cutoff, result);
|
||||
reportCoverage(details, "audit_detail", cutoff, result);
|
||||
coverageTable(result);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The coverage table must hold exactly the chain's own detail lists. A
|
||||
** mismatch would let the writer skip or re-chain a detail, so it is a break.
|
||||
*******************************************************************************/
|
||||
private void coverageTable(VerificationResult result) throws QException
|
||||
{
|
||||
String table = "SELECT detail_id, chain_id FROM " + WormAuditTables.CHAIN_DETAIL;
|
||||
String sql = isPostgres()
|
||||
? "SELECT detail_id, count(*) OVER() FROM ((" + CHAIN_PAIRS + " EXCEPT " + table + ") UNION ALL (" + table + " EXCEPT " + CHAIN_PAIRS
|
||||
+ ")) mismatch ORDER BY detail_id LIMIT 100"
|
||||
: "SELECT x.detail_id, count(*) OVER() FROM " + WormAuditTables.CHAIN_DETAIL + " x WHERE NOT EXISTS (SELECT 1 FROM worm_audit_chain c"
|
||||
+ " WHERE c.id=x.chain_id AND POSITION(',' || CAST(x.detail_id AS varchar) || ',' IN ',' || c.detail_ids || ',') > 0) ORDER BY x.detail_id LIMIT 100";
|
||||
try(PreparedStatement statement = ((RDBMSTransaction) transaction).getConnection().prepareStatement(sql); ResultSet rows = statement.executeQuery())
|
||||
{
|
||||
long total = 0;
|
||||
int reported = 0;
|
||||
while(rows.next())
|
||||
{
|
||||
total = rows.getLong(2);
|
||||
result.add(null, null, "audit_detail " + com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier.number(rows.getLong(1))
|
||||
+ " has a detail coverage row that disagrees with the chain");
|
||||
reported++;
|
||||
}
|
||||
result.omitted(total - reported);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw (AppendOnlyGuard.map(e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Highest id in a core audit table, or 0 when it is empty (RDBMS only).
|
||||
*******************************************************************************/
|
||||
public long maxId(String table) throws QException
|
||||
{
|
||||
return (sqlIds("SELECT coalesce(max(" + column(table, "id") + "), 0) FROM " + physical(table)).getFirst());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** PostgreSQL maintains detail coverage in its chain trigger; other backends
|
||||
** have no trigger, so the writer appends coverage rows itself.
|
||||
*******************************************************************************/
|
||||
public boolean isPostgres() throws QException
|
||||
{
|
||||
if(postgres == null)
|
||||
{
|
||||
try
|
||||
{
|
||||
postgres = isRdbms() && "PostgreSQL".equals(((RDBMSTransaction) transaction).getConnection().getMetaData().getDatabaseProductName());
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw (AppendOnlyGuard.map(e));
|
||||
}
|
||||
}
|
||||
return (postgres);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Keep exact offender ids and total counts without materializing the result set.
|
||||
*******************************************************************************/
|
||||
private void reportCoverage(String source, String table, Instant cutoff, VerificationResult result) throws QException
|
||||
{
|
||||
String sql = "SELECT id, coverage, count(*) OVER() AS total FROM (" + source + ") offenders WHERE coverage <> 1 ORDER BY id LIMIT 100";
|
||||
try(PreparedStatement statement = ((RDBMSTransaction) transaction).getConnection().prepareStatement(sql))
|
||||
{
|
||||
bindCutoff(statement, 1, cutoff);
|
||||
long total = 0;
|
||||
int reported = 0;
|
||||
try(ResultSet rows = statement.executeQuery())
|
||||
{
|
||||
while(rows.next())
|
||||
{
|
||||
long id = rows.getLong(1);
|
||||
total = rows.getLong(3);
|
||||
result.add(null, table.equals("audit") ? id : null, table + " " + com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier.number(id)
|
||||
+ (rows.getLong(2) == 0 ? " is not covered by the chain" : " is covered twice or more by the chain"));
|
||||
reported++;
|
||||
}
|
||||
}
|
||||
result.omitted(total - reported);
|
||||
if(total > reported)
|
||||
{
|
||||
result.warn(table + ": " + total + " coverage offenders; first 100 ids reported");
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw (AppendOnlyGuard.map(e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read ids from an anti-join, binding instants by column type and ids as longs.
|
||||
*******************************************************************************/
|
||||
private List<Long> sqlIds(String sql, Object... parameters) throws QException
|
||||
{
|
||||
try(PreparedStatement statement = ((RDBMSTransaction) transaction).getConnection().prepareStatement(sql))
|
||||
{
|
||||
for(int i = 0; i < parameters.length; i++)
|
||||
{
|
||||
if(parameters[i] instanceof Instant instant)
|
||||
{
|
||||
bindCutoff(statement, i + 1, instant);
|
||||
}
|
||||
else
|
||||
{
|
||||
statement.setLong(i + 1, (Long) parameters[i]);
|
||||
}
|
||||
}
|
||||
List<Long> ids = new ArrayList<>();
|
||||
try(ResultSet rows = statement.executeQuery())
|
||||
{
|
||||
while(rows.next())
|
||||
{
|
||||
ids.add(rows.getLong(1));
|
||||
}
|
||||
}
|
||||
return (ids);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw (AppendOnlyGuard.map(e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** UTC local timestamps for without-tz columns; offset timestamps for timestamptz.
|
||||
*******************************************************************************/
|
||||
private void bindCutoff(PreparedStatement statement, int index, Instant cutoff) throws Exception
|
||||
{
|
||||
if(timestampWithZone == null)
|
||||
{
|
||||
try(var probe = ((RDBMSTransaction) transaction).getConnection().createStatement();
|
||||
var rows = probe.executeQuery("SELECT " + column("audit", "timestamp") + " FROM " + physical("audit") + " WHERE 1=0"))
|
||||
{
|
||||
var metadata = rows.getMetaData();
|
||||
timestampWithZone = metadata.getColumnType(1) == Types.TIMESTAMP_WITH_TIMEZONE
|
||||
|| "timestamptz".equalsIgnoreCase(metadata.getColumnTypeName(1));
|
||||
}
|
||||
}
|
||||
statement.setObject(index, timestampWithZone ? cutoff.atOffset(ZoneOffset.UTC) : LocalDateTime.ofInstant(cutoff, ZoneOffset.UTC));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve and quote physical core table names.
|
||||
*******************************************************************************/
|
||||
private String physical(String table)
|
||||
{
|
||||
return (WormAuditDdl.identifier(((RDBMSTableBackendDetails) QContext.getQInstance().getTable(table).getBackendDetails()).getTableName()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve and quote physical field names for runtime queries.
|
||||
*******************************************************************************/
|
||||
private String column(String table, String field)
|
||||
{
|
||||
String name = QContext.getQInstance().getTable(table).getField(field).getBackendName();
|
||||
return (WormAuditDdl.identifier(name == null ? field : name));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.chain;
|
||||
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HexFormat;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** SHA-256 primitives. Link predecessors are the 64 lowercase ASCII hex bytes;
|
||||
** Merkle leaves are the decoded 32-byte link hashes, with odd nodes promoted.
|
||||
*******************************************************************************/
|
||||
public final class Hashes
|
||||
{
|
||||
public static final String GENESIS = "0".repeat(64);
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private Hashes()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** SHA-256 digest of concatenated byte arrays.
|
||||
*******************************************************************************/
|
||||
public static byte[] digest(byte[]... parts)
|
||||
{
|
||||
try
|
||||
{
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
for(byte[] part : parts)
|
||||
{
|
||||
digest.update(part);
|
||||
}
|
||||
return (digest.digest());
|
||||
}
|
||||
catch(NoSuchAlgorithmException e)
|
||||
{
|
||||
throw (new IllegalStateException("JDK lacks SHA-256", e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Link hash over predecessor, NUL delimiter and canonical bytes.
|
||||
*******************************************************************************/
|
||||
public static String link(String previous, byte[] canonical)
|
||||
{
|
||||
return (HexFormat.of().formatHex(digest(previous.getBytes(StandardCharsets.US_ASCII), new byte[] { 0 }, canonical)));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Binary Merkle root; an empty range cannot be checkpointed.
|
||||
*******************************************************************************/
|
||||
public static String merkle(List<String> hashes)
|
||||
{
|
||||
if(hashes.isEmpty())
|
||||
{
|
||||
throw (new IllegalArgumentException("A Merkle range must contain links"));
|
||||
}
|
||||
List<byte[]> level = hashes.stream().map(HexFormat.of()::parseHex).toList();
|
||||
while(level.size() > 1)
|
||||
{
|
||||
List<byte[]> next = new ArrayList<>();
|
||||
for(int i = 0; i < level.size(); i += 2)
|
||||
{
|
||||
next.add(i + 1 == level.size() ? level.get(i) : digest(level.get(i), level.get(i + 1)));
|
||||
}
|
||||
level = next;
|
||||
}
|
||||
return (HexFormat.of().formatHex(level.getFirst()));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.chain;
|
||||
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.Collections;
|
||||
import java.util.Deque;
|
||||
import java.util.Map;
|
||||
import java.util.WeakHashMap;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Per-JVM id floors for the chain writer's two sweeps, so a tick reads only
|
||||
** audit and detail rows newer than the last complete sweep instead of the whole
|
||||
** settle window or late-detail lookback.
|
||||
**
|
||||
** Each tick records the highest audit and audit_detail ids it can see. After a
|
||||
** sweep that returned fewer rows than its limit, every row allocated before
|
||||
** (settle horizon + MARGIN) ago is chained, on the same assumption the settle
|
||||
** horizon already makes: a row whose id was allocated that long ago has
|
||||
** committed. The floor moves to the highest id observed at or before that time.
|
||||
** A row whose transaction commits later than that is still chained: once an
|
||||
** hour (and after every restart) a tick ignores the floors and sweeps
|
||||
** everything, exactly as before floors existed. Floors live in memory only; a
|
||||
** second replica keeps its own. The daily verification checks full coverage
|
||||
** independently of all of this.
|
||||
*******************************************************************************/
|
||||
public final class SweepFloors
|
||||
{
|
||||
public static final Duration MARGIN = Duration.ofMinutes(5);
|
||||
public static final Duration FULL_SWEEP_EVERY = Duration.ofHours(1);
|
||||
|
||||
private static final Map<WormAuditQBitConfig, SweepFloors> BY_CONFIG = Collections.synchronizedMap(new WeakHashMap<>());
|
||||
|
||||
private final Deque<Observation> observations = new ArrayDeque<>();
|
||||
private long audit;
|
||||
private long detail;
|
||||
private Instant lastFullSweep;
|
||||
|
||||
/*******************************************************************************
|
||||
** The highest ids visible at one tick.
|
||||
*******************************************************************************/
|
||||
public record Observation(Instant at, long audit, long detail)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Floors for one installed qbit (config identity), created on first use.
|
||||
*******************************************************************************/
|
||||
public static SweepFloors of(WormAuditQBitConfig config)
|
||||
{
|
||||
return (BY_CONFIG.computeIfAbsent(config, c -> new SweepFloors()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Audit ids at or below this are known chained.
|
||||
*******************************************************************************/
|
||||
public synchronized long audit()
|
||||
{
|
||||
return (audit);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Detail ids at or below this are known covered (or outside the lookback).
|
||||
*******************************************************************************/
|
||||
public synchronized long detail()
|
||||
{
|
||||
return (detail);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether this tick must ignore the floors: first tick, an hour since the last
|
||||
** complete full sweep, or a clock that moved backwards.
|
||||
*******************************************************************************/
|
||||
public synchronized boolean fullSweepDue(Instant now)
|
||||
{
|
||||
return (lastFullSweep == null || now.isBefore(lastFullSweep) || !now.isBefore(lastFullSweep.plus(FULL_SWEEP_EVERY)));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Record the highest visible ids. Ids below a floor mean rows were removed
|
||||
** wholesale (a purge of everything, or a reset test database): start over.
|
||||
*******************************************************************************/
|
||||
public synchronized Observation observe(Instant now, long maxAudit, long maxDetail)
|
||||
{
|
||||
if(maxAudit < audit || maxDetail < detail)
|
||||
{
|
||||
observations.clear();
|
||||
audit = 0;
|
||||
detail = 0;
|
||||
}
|
||||
Observation observation = new Observation(now, maxAudit, maxDetail);
|
||||
observations.addLast(observation);
|
||||
return (observation);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** After a tick, raise each completely swept floor to the newest observation
|
||||
** made at or before the horizon, then drop observations no longer needed. A
|
||||
** full sweep (fullSweepAt not null) counts only when both sweeps completed.
|
||||
*******************************************************************************/
|
||||
public synchronized void swept(Observation current, Instant horizon, boolean auditsComplete, boolean detailsComplete, Instant fullSweepAt)
|
||||
{
|
||||
if(fullSweepAt != null && auditsComplete && detailsComplete)
|
||||
{
|
||||
lastFullSweep = fullSweepAt;
|
||||
}
|
||||
Observation settled = null;
|
||||
for(Observation observation : observations)
|
||||
{
|
||||
if(!observation.at().isAfter(horizon) && (settled == null || !observation.at().isBefore(settled.at())))
|
||||
{
|
||||
settled = observation;
|
||||
}
|
||||
}
|
||||
if(settled == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
if(auditsComplete)
|
||||
{
|
||||
audit = Math.max(audit, Math.min(settled.audit(), current.audit()));
|
||||
}
|
||||
if(detailsComplete)
|
||||
{
|
||||
detail = Math.max(detail, Math.min(settled.detail(), current.detail()));
|
||||
}
|
||||
Observation keep = settled;
|
||||
observations.removeIf(observation -> observation != keep && !observation.at().isAfter(horizon));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.chain;
|
||||
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.Statement;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import java.util.stream.Collectors;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditSettings;
|
||||
import com.kingsrook.qbits.wormaudit.checkpoint.Checkpoints;
|
||||
import com.kingsrook.qbits.wormaudit.guard.AppendOnlyGuard;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSTransaction;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Single asynchronous writer. PostgreSQL uses a transaction-scoped advisory
|
||||
** lock shared with the retention function. Other backends use a JVM-wide lock:
|
||||
** those deployments must run their chainer in one JVM only.
|
||||
*******************************************************************************/
|
||||
public class WormAuditChainer
|
||||
{
|
||||
public static final long ADVISORY_LOCK = 86753090421L;
|
||||
public static final ReentrantLock JVM_LOCK = new ReentrantLock();
|
||||
private final WormAuditQBitConfig config;
|
||||
|
||||
/*******************************************************************************
|
||||
** Counts describe committed work; a contended writer tick is skipped.
|
||||
*******************************************************************************/
|
||||
public record Counts(int audits, int lateDetails, int links, int checkpoints, boolean skipped)
|
||||
{
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Bind host configuration.
|
||||
*******************************************************************************/
|
||||
public WormAuditChainer(WormAuditQBitConfig config)
|
||||
{
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Run a writer tick under one transaction; link ids cannot gap on rollback.
|
||||
*******************************************************************************/
|
||||
public Counts chainOnce() throws QException
|
||||
{
|
||||
return (chainOnce(Instant.now()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Clock-injected tick for deterministic settling and checkpoint tests.
|
||||
*******************************************************************************/
|
||||
public Counts chainOnce(Instant now) throws QException
|
||||
{
|
||||
config.validate();
|
||||
JVM_LOCK.lock();
|
||||
try(QBackendTransaction transaction = QBackendTransaction.openFor(new QueryInput(WormAuditTables.CHAIN)))
|
||||
{
|
||||
try
|
||||
{
|
||||
if(transaction instanceof RDBMSTransaction rdbms && "PostgreSQL".equals(rdbms.getConnection().getMetaData().getDatabaseProductName()))
|
||||
{
|
||||
try(Statement statement = rdbms.getConnection().createStatement(); ResultSet result = statement.executeQuery("SELECT pg_try_advisory_xact_lock(" + ADVISORY_LOCK + ")"))
|
||||
{
|
||||
result.next();
|
||||
if(!result.getBoolean(1))
|
||||
{
|
||||
return (new Counts(0, 0, 0, 0, true));
|
||||
}
|
||||
}
|
||||
}
|
||||
Counts counts = append(new AuditStore(transaction), now, config.effectiveSettings());
|
||||
transaction.commit();
|
||||
return (counts);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
transaction.rollback();
|
||||
throw (AppendOnlyGuard.map(e));
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
JVM_LOCK.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Append settled headers and uncovered detail chunks, then checkpoint.
|
||||
*******************************************************************************/
|
||||
private Counts append(AuditStore store, Instant now, WormAuditSettings.Values settings) throws Exception
|
||||
{
|
||||
QRecord tail = store.last(WormAuditTables.CHAIN);
|
||||
QRecord anchor = store.anchor();
|
||||
long nextId = tail == null ? anchor.getValueLong("anchor_link_id") + 1 : tail.getValueLong("id") + 1;
|
||||
String previous = tail == null ? anchor.getValueString("anchor_hash") : tail.getValueString("link_hash");
|
||||
List<QRecord> appended = new ArrayList<>();
|
||||
Instant cutoff = now.minusSeconds(settings.settle_horizon_seconds());
|
||||
SweepFloors floors = SweepFloors.of(config);
|
||||
SweepFloors.Observation observed = store.isRdbms() ? floors.observe(now, store.maxId("audit"), store.maxId("auditDetail")) : null;
|
||||
boolean full = floors.fullSweepDue(now);
|
||||
List<QRecord> audits = store.unchained(cutoff, config.getBatchSize(), full ? 0L : floors.audit());
|
||||
Map<Long, QRecord> details = AuditStore.index(store.in("auditDetail", "auditId", audits.stream().map(r -> r.getValueLong("id")).toList()));
|
||||
Map<Long, List<QRecord>> grouped = details.values().stream().collect(Collectors.groupingBy(r -> r.getValueLong("auditId")));
|
||||
int lateDetails = 0;
|
||||
for(QRecord audit : audits)
|
||||
{
|
||||
List<Long> ids = grouped.getOrDefault(audit.getValueLong("id"), List.of()).stream().map(r -> r.getValueLong("id")).sorted().toList();
|
||||
for(int start = 0; start == 0 || start < ids.size(); start += 250)
|
||||
{
|
||||
List<Long> chunk = ids.subList(start, Math.min(start + 250, ids.size()));
|
||||
QRecord link = link(store, nextId++, start == 0 ? "AUDIT" : "LATE_DETAIL", audit, chunk, details, previous, now);
|
||||
appended.add(link);
|
||||
previous = link.getValueString("link_hash");
|
||||
if(start > 0)
|
||||
{
|
||||
lateDetails += chunk.size();
|
||||
}
|
||||
}
|
||||
}
|
||||
insertLinks(store, appended);
|
||||
int linkCount = appended.size();
|
||||
appended.clear();
|
||||
details = AuditStore.index(store.lateDetails(cutoff, settings.late_detail_lookback_hours(), config.getBatchSize(), full ? 0L : floors.detail()));
|
||||
grouped = details.values().stream().collect(Collectors.groupingBy(r -> r.getValueLong("auditId"), TreeMap::new, Collectors.toList()));
|
||||
Map<Long, QRecord> headers = AuditStore.index(store.in("audit", "id", grouped.keySet()));
|
||||
for(var entry : grouped.entrySet())
|
||||
{
|
||||
List<Long> ids = entry.getValue().stream().map(r -> r.getValueLong("id")).sorted().toList();
|
||||
for(int start = 0; start < ids.size(); start += 250)
|
||||
{
|
||||
List<Long> chunk = ids.subList(start, Math.min(start + 250, ids.size()));
|
||||
QRecord link = link(store, nextId++, "LATE_DETAIL", headers.get(entry.getKey()), chunk, details, previous, now);
|
||||
appended.add(link);
|
||||
previous = link.getValueString("link_hash");
|
||||
lateDetails += chunk.size();
|
||||
}
|
||||
}
|
||||
insertLinks(store, appended);
|
||||
if(observed != null)
|
||||
{
|
||||
floors.swept(observed, now.minusSeconds(settings.settle_horizon_seconds()).minus(SweepFloors.MARGIN),
|
||||
audits.size() < config.getBatchSize(), details.size() < config.getBatchSize(), full ? now : null);
|
||||
}
|
||||
return (new Counts(audits.size(), lateDetails, linkCount + appended.size(), checkpoint(store, now, settings), false));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Persist bounded insert batches inside the writer transaction. PostgreSQL's
|
||||
** chain trigger writes the detail coverage rows; elsewhere the writer does,
|
||||
** in the same transaction, so coverage never diverges from the chain.
|
||||
*******************************************************************************/
|
||||
private void insertLinks(AuditStore store, List<QRecord> links) throws QException
|
||||
{
|
||||
for(int start = 0; start < links.size(); start += config.getBatchSize())
|
||||
{
|
||||
store.insert(WormAuditTables.CHAIN, links.subList(start, Math.min(start + config.getBatchSize(), links.size())));
|
||||
}
|
||||
if(!store.isPostgres())
|
||||
{
|
||||
List<QRecord> coverage = new ArrayList<>();
|
||||
for(QRecord link : links)
|
||||
{
|
||||
AuditStore.detailIds(link).forEach(id -> coverage.add(new QRecord().withValue("detail_id", id).withValue("chain_id", link.getValueLong("id"))));
|
||||
}
|
||||
for(int start = 0; start < coverage.size(); start += config.getBatchSize())
|
||||
{
|
||||
store.insert(WormAuditTables.CHAIN_DETAIL, coverage.subList(start, Math.min(start + config.getBatchSize(), coverage.size())));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build one immutable link from live stored content.
|
||||
*******************************************************************************/
|
||||
private QRecord link(AuditStore store, long id, String kind, QRecord audit, List<Long> details, Map<Long, QRecord> rows, String previous, Instant now) throws QException
|
||||
{
|
||||
return (new QRecord().withValue("id", id).withValue("kind", kind).withValue("audit_id", audit.getValueLong("id"))
|
||||
.withValue("detail_ids", details.stream().map(Object::toString).collect(Collectors.joining(",")))
|
||||
.withValue("prev_hash", previous).withValue("link_hash", Hashes.link(previous, store.canonical(audit, details, rows))).withValue("chained_at", now));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Checkpoint the uncheckpointed suffix by count or elapsed time.
|
||||
*******************************************************************************/
|
||||
private int checkpoint(AuditStore store, Instant now, WormAuditSettings.Values settings) throws Exception
|
||||
{
|
||||
if(config.getSigningKey() == null)
|
||||
{
|
||||
return (0);
|
||||
}
|
||||
QRecord latest = store.last(WormAuditTables.CHECKPOINT);
|
||||
long lastId = latest == null ? store.anchor().getValueLong("anchor_link_id") : latest.getValueLong("to_link_id");
|
||||
int count = 0;
|
||||
while(true)
|
||||
{
|
||||
List<QRecord> pending = store.page(WormAuditTables.CHAIN, lastId, settings.checkpoint_every_links());
|
||||
if(pending.isEmpty())
|
||||
{
|
||||
return (count);
|
||||
}
|
||||
Instant since = latest == null ? pending.getFirst().getValueInstant("chained_at") : latest.getValueInstant("created_at");
|
||||
if(pending.size() < settings.checkpoint_every_links() && now.isBefore(since.plusSeconds(settings.checkpoint_every_minutes() * 60L)))
|
||||
{
|
||||
return (count);
|
||||
}
|
||||
latest = store.insert(WormAuditTables.CHECKPOINT, List.of(Checkpoints.sign(pending, config, now))).getFirst();
|
||||
lastId = latest.getValueLong("to_link_id");
|
||||
count++;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.checkpoint;
|
||||
|
||||
|
||||
import java.security.GeneralSecurityException;
|
||||
import java.security.PublicKey;
|
||||
import java.security.Signature;
|
||||
import java.time.Instant;
|
||||
import java.util.Base64;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.canonical.Jcs;
|
||||
import com.kingsrook.qbits.wormaudit.chain.Hashes;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Ed25519 checkpoint envelopes. Trust comes exclusively from host-pinned keys,
|
||||
** never the public_key supplied by an untrusted checkpoint row.
|
||||
*******************************************************************************/
|
||||
public final class Checkpoints
|
||||
{
|
||||
private static final List<String> SIGNED_FIELDS = List.of("from_link_id", "to_link_id", "link_count", "head_hash", "merkle_root", "algorithm", "key_id");
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private Checkpoints()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The exact signed payload; public_key and signature are envelope fields.
|
||||
*******************************************************************************/
|
||||
public static byte[] payload(QRecord checkpoint)
|
||||
{
|
||||
Map<String, Object> fields = new LinkedHashMap<>();
|
||||
for(String field : SIGNED_FIELDS)
|
||||
{
|
||||
fields.put(field, field.endsWith("_id") && !field.equals("key_id") || field.equals("link_count")
|
||||
? checkpoint.getValueLong(field) : checkpoint.getValueString(field));
|
||||
}
|
||||
return (Jcs.bytes(fields));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Create a checkpoint only when the configured private key matches its pin.
|
||||
*******************************************************************************/
|
||||
public static QRecord sign(List<QRecord> links, WormAuditQBitConfig config, Instant now) throws GeneralSecurityException
|
||||
{
|
||||
WormAuditQBitConfig.SigningKey key = config.getSigningKey().get();
|
||||
PublicKey publicKey = config.getTrustedPublicKeys().get(key.keyId());
|
||||
if(publicKey == null)
|
||||
{
|
||||
throw (new GeneralSecurityException("Signing key id is not pinned"));
|
||||
}
|
||||
QRecord record = new QRecord().withValue("from_link_id", links.getFirst().getValueLong("id"))
|
||||
.withValue("to_link_id", links.getLast().getValueLong("id")).withValue("link_count", (long) links.size())
|
||||
.withValue("head_hash", links.getLast().getValueString("link_hash"))
|
||||
.withValue("merkle_root", Hashes.merkle(links.stream().map(r -> r.getValueString("link_hash")).toList()))
|
||||
.withValue("algorithm", "Ed25519").withValue("key_id", key.keyId())
|
||||
.withValue("public_key", Base64.getEncoder().encodeToString(publicKey.getEncoded())).withValue("created_at", now);
|
||||
Signature signature = Signature.getInstance("Ed25519");
|
||||
signature.initSign(key.privateKey());
|
||||
signature.update(payload(record));
|
||||
record.setValue("signature", Base64.getEncoder().encodeToString(signature.sign()));
|
||||
if(!verify(record, publicKey))
|
||||
{
|
||||
throw (new GeneralSecurityException("Signing private key does not match its pin"));
|
||||
}
|
||||
return (record);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify with a pinned key, rejecting substituted algorithms and public keys.
|
||||
*******************************************************************************/
|
||||
public static boolean verify(QRecord record, PublicKey pin)
|
||||
{
|
||||
try
|
||||
{
|
||||
if(pin == null || !"Ed25519".equals(record.getValueString("algorithm"))
|
||||
|| !Base64.getEncoder().encodeToString(pin.getEncoded()).equals(record.getValueString("public_key")))
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
Signature signature = Signature.getInstance("Ed25519");
|
||||
signature.initVerify(pin);
|
||||
signature.update(payload(record));
|
||||
return (signature.verify(Base64.getDecoder().decode(record.getValueString("signature"))));
|
||||
}
|
||||
catch(GeneralSecurityException | IllegalArgumentException e)
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.ddl;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Render a reviewed migration for the host's physical audit table names.
|
||||
** Core physical columns must use snake_case: audit.id/timestamp,
|
||||
** audit_detail.id/audit_id, audit_table.id/name/label/create_date/modify_date,
|
||||
** and audit_user.id/name/create_date/modify_date. Only table names are rendered;
|
||||
** QInstance field backend-name overrides do not change this DDL contract.
|
||||
** Identifiers are restricted and quoted; configuration is never executable SQL.
|
||||
*******************************************************************************/
|
||||
public final class WormAuditDdl
|
||||
{
|
||||
public static final Map<String, String> DEFAULT_NAMES = Map.of("audit", "audit", "auditDetail", "audit_detail", "auditTable", "audit_table", "auditUser", "audit_user");
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private WormAuditDdl()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A safe SQL identifier, optionally schema qualified.
|
||||
*******************************************************************************/
|
||||
public static String identifier(String value)
|
||||
{
|
||||
if(value == null || !value.matches("[A-Za-z_][A-Za-z0-9_]*(\\.[A-Za-z_][A-Za-z0-9_]*)?"))
|
||||
{
|
||||
throw (new IllegalArgumentException("Invalid SQL identifier"));
|
||||
}
|
||||
return ("\"" + value.replace(".", "\".\"") + "\"");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve backend names while preserving explicit DDL overrides.
|
||||
*******************************************************************************/
|
||||
public static void resolveNames(WormAuditQBitConfig config, QInstance instance)
|
||||
{
|
||||
Map<String, String> names = new LinkedHashMap<>();
|
||||
for(String name : DEFAULT_NAMES.keySet())
|
||||
{
|
||||
QTableMetaData table = instance.getTable(name);
|
||||
if(table == null)
|
||||
{
|
||||
throw (new IllegalArgumentException("Core audit table is required: " + name));
|
||||
}
|
||||
names.put(name, table.getBackendDetails() instanceof RDBMSTableBackendDetails details ? details.getTableName() : DEFAULT_NAMES.get(name));
|
||||
}
|
||||
names.putAll(config.getCoreAuditTableNames());
|
||||
config.withCoreAuditTableNames(Map.copyOf(names));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Render the template. Retention is compiled into the function, never a GUC.
|
||||
*******************************************************************************/
|
||||
public static String postgres(WormAuditQBitConfig config)
|
||||
{
|
||||
if(config.getRetentionYears() < 1 || config.getAppRole().contains("."))
|
||||
{
|
||||
throw (new IllegalArgumentException("Invalid retention or app role"));
|
||||
}
|
||||
try(var stream = WormAuditDdl.class.getResourceAsStream("/com/kingsrook/qbits/wormaudit/ddl/postgres/worm-audit.sql"))
|
||||
{
|
||||
String sql = new String(java.util.Objects.requireNonNull(stream).readAllBytes(), StandardCharsets.UTF_8);
|
||||
Map<String, String> names = new LinkedHashMap<>(DEFAULT_NAMES);
|
||||
names.putAll(config.getCoreAuditTableNames());
|
||||
for(var entry : DEFAULT_NAMES.entrySet())
|
||||
{
|
||||
sql = sql.replace("${" + entry.getValue() + "}", identifier(names.get(entry.getKey())));
|
||||
}
|
||||
var defaults = com.kingsrook.qbits.wormaudit.WormAuditSettings.defaults(config);
|
||||
com.kingsrook.qbits.wormaudit.WormAuditSettings.validate(defaults);
|
||||
String values = java.util.List.of("id", "audit_retention_years", "export_lock_mode", "export_retention_days", "checkpoint_every_links",
|
||||
"checkpoint_every_minutes", "chain_interval_seconds", "settle_horizon_seconds", "late_detail_lookback_hours", "export_cron", "verify_cron")
|
||||
.stream().map(field -> defaults.getValue(field) instanceof Number ? defaults.getValue(field).toString()
|
||||
: "'" + defaults.getValueString(field).replace("'", "''") + "'").collect(java.util.stream.Collectors.joining(", "));
|
||||
sql = sql.replace("${settings_values}", values);
|
||||
return (sql.replace("${app_role}", identifier(config.getAppRole())).replace("${retention}", config.getRetentionYears() + " years"));
|
||||
}
|
||||
catch(IOException e)
|
||||
{
|
||||
throw (new IllegalStateException("Cannot read WORM audit DDL", e));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.export;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.Serializable;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.security.MessageDigest;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HexFormat;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.fasterxml.jackson.core.JsonParser;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.kingsrook.qbits.objectstorage.api.StoredObject;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Version-one export encoding and temporary files. JSON errors never echo input.
|
||||
*******************************************************************************/
|
||||
public class ExportFiles implements AutoCloseable
|
||||
{
|
||||
public static final List<String> NAMES = List.of("links.jsonl", "checkpoints.jsonl", "manifest.json");
|
||||
private static final ObjectMapper JSON = new ObjectMapper().enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION)
|
||||
.enable(DeserializationFeature.USE_LONG_FOR_INTS).enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
|
||||
private final Path directory;
|
||||
|
||||
/*******************************************************************************
|
||||
** Create a private temporary directory, removed on all normal failure paths.
|
||||
*******************************************************************************/
|
||||
public ExportFiles() throws IOException
|
||||
{
|
||||
directory = Files.createTempDirectory("worm-audit-");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve only one of the fixed filenames, never a manifest-supplied path.
|
||||
*******************************************************************************/
|
||||
public Path file(String name)
|
||||
{
|
||||
if(!NAMES.contains(name))
|
||||
{
|
||||
throw (new IllegalArgumentException("Unknown export file"));
|
||||
}
|
||||
return (directory.resolve(name));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Parse the JCS schema with strict duplicate and trailing-token rejection.
|
||||
*******************************************************************************/
|
||||
public static Map<String, Object> parse(byte[] bytes) throws IOException
|
||||
{
|
||||
try
|
||||
{
|
||||
return (JSON.readValue(bytes, new TypeReference<Map<String, Object>>() {}));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw (new IOException("Invalid sealed JSON"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Wire representation of a chain row, excluding database-only timestamps.
|
||||
*******************************************************************************/
|
||||
public static Map<String, Object> link(QRecord row)
|
||||
{
|
||||
return (Map.of("id", row.getValueLong("id"), "kind", row.getValueString("kind"),
|
||||
"auditId", row.getValueLong("audit_id"), "detailIds", AuditStore.detailIds(row),
|
||||
"prevHash", row.getValueString("prev_hash"), "linkHash", row.getValueString("link_hash")));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Preserve every checkpoint field, representing the timestamp as UTC text.
|
||||
*******************************************************************************/
|
||||
public static Map<String, Object> checkpoint(QRecord row)
|
||||
{
|
||||
Map<String, Object> fields = new LinkedHashMap<>();
|
||||
for(String key : List.of("id", "from_link_id", "to_link_id", "link_count"))
|
||||
{
|
||||
fields.put(key, row.getValueLong(key));
|
||||
}
|
||||
for(String key : List.of("head_hash", "merkle_root", "algorithm", "key_id", "public_key", "signature"))
|
||||
{
|
||||
fields.put(key, row.getValueString(key));
|
||||
}
|
||||
fields.put("created_at", row.getValueInstant("created_at").toString());
|
||||
return (fields);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Convert a checkpoint envelope back to the phase (a) signature input.
|
||||
*******************************************************************************/
|
||||
public static QRecord record(Map<String, Object> fields)
|
||||
{
|
||||
QRecord row = new QRecord();
|
||||
fields.forEach((key, value) -> row.setValue(key, (Serializable) value));
|
||||
return (row);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Compute a digest without materializing a file.
|
||||
*******************************************************************************/
|
||||
public static String sha256(Path path) throws Exception
|
||||
{
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
try(InputStream input = Files.newInputStream(path))
|
||||
{
|
||||
byte[] buffer = new byte[65536];
|
||||
int read;
|
||||
while((read = input.read(buffer)) != -1)
|
||||
{
|
||||
digest.update(buffer, 0, read);
|
||||
}
|
||||
}
|
||||
return (HexFormat.of().formatHex(digest.digest()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A lock echo is required even if a provider advertises the capability.
|
||||
*******************************************************************************/
|
||||
public static boolean locked(StoredObject object, String mode, java.time.Instant until)
|
||||
{
|
||||
return (object != null && java.util.Objects.equals(mode, object.getLockMode()) && object.getLockRetainUntil() != null
|
||||
&& !object.getLockRetainUntil().isBefore(until));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A null version belongs to unversioned storage and cannot be sealed.
|
||||
*******************************************************************************/
|
||||
public static boolean validVersion(String versionId)
|
||||
{
|
||||
return (versionId != null && !versionId.isBlank() && !"null".equals(versionId));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fetch covered detail rows in bounded IN batches.
|
||||
*******************************************************************************/
|
||||
public static Map<Long, QRecord> details(AuditStore store, List<QRecord> links) throws Exception
|
||||
{
|
||||
List<Long> ids = new ArrayList<>();
|
||||
for(QRecord link : links)
|
||||
{
|
||||
List<Long> covered = AuditStore.detailIds(link);
|
||||
if(covered.size() > 250)
|
||||
{
|
||||
throw (new IOException("Invalid detail coverage"));
|
||||
}
|
||||
ids.addAll(covered);
|
||||
}
|
||||
Map<Long, QRecord> rows = new LinkedHashMap<>();
|
||||
for(int start = 0; start < ids.size(); start += 1000)
|
||||
{
|
||||
rows.putAll(AuditStore.index(store.in("auditDetail", "id", ids.subList(start, Math.min(start + 1000, ids.size())))));
|
||||
}
|
||||
return (rows);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Remove only the three owned files and their directory.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void close() throws IOException
|
||||
{
|
||||
IOException failure = null;
|
||||
for(String name : NAMES)
|
||||
{
|
||||
try
|
||||
{
|
||||
Files.deleteIfExists(file(name));
|
||||
}
|
||||
catch(IOException e)
|
||||
{
|
||||
failure = new IOException("Cannot remove temporary export files");
|
||||
}
|
||||
}
|
||||
Files.deleteIfExists(directory);
|
||||
if(failure != null)
|
||||
{
|
||||
throw (failure);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.export;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Instant;
|
||||
import com.kingsrook.qbits.objectstorage.api.Capability;
|
||||
import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
|
||||
import com.kingsrook.qbits.objectstorage.api.ListRequest;
|
||||
import com.kingsrook.qbits.objectstorage.api.ObjectStorage;
|
||||
import com.kingsrook.qbits.objectstorage.api.ObjectSummary;
|
||||
import com.kingsrook.qbits.objectstorage.api.Page;
|
||||
import com.kingsrook.qbits.objectstorage.api.PutRequest;
|
||||
import com.kingsrook.qbits.objectstorage.api.StorageBucket;
|
||||
import com.kingsrook.qbits.objectstorage.api.StorageKeys;
|
||||
import com.kingsrook.qbits.objectstorage.api.StoredObject;
|
||||
import com.kingsrook.qbits.objectstorage.config.BucketRole;
|
||||
import com.kingsrook.qbits.objectstorage.internal.ObjectStorageRuntime;
|
||||
import com.kingsrook.qbits.objectstorage.internal.SecretRefs;
|
||||
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Thin system-caller adapter to qbit-object-storage.
|
||||
*******************************************************************************/
|
||||
public class ObjectStorageSealedStore implements SealedStore
|
||||
{
|
||||
private final StorageBucket bucket;
|
||||
|
||||
/*******************************************************************************
|
||||
** Bind the recorded alias, so old exports survive configuration changes.
|
||||
*******************************************************************************/
|
||||
public ObjectStorageSealedStore(String alias, String runId)
|
||||
{
|
||||
bucket = ObjectStorage.asSystem("worm_audit", runId).bucket(alias);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Refuse unknown, partial, or unsupported lock capabilities.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean supportsLock()
|
||||
{
|
||||
return (bucket.config().getRole() == BucketRole.BACKUP
|
||||
&& bucket.capability(Capability.OBJECT_LOCK) == CapabilityStatus.SUPPORTED);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Report the provider capability without collapsing UNKNOWN into UNSUPPORTED.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public CapabilityStatus lockCapability()
|
||||
{
|
||||
return (bucket.capability(Capability.OBJECT_LOCK));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stream a bounded upload and close the source even when the provider fails.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String put(String key, Path file, com.kingsrook.qbits.objectstorage.api.LockMode mode, Instant retainUntil) throws Exception
|
||||
{
|
||||
try(InputStream input = Files.newInputStream(file))
|
||||
{
|
||||
return (bucket.put(PutRequest.of(key, input).withContentLength(Files.size(file))
|
||||
.withObjectLock(mode, retainUntil)).getVersionId());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read retention metadata.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public StoredObject head(String key)
|
||||
{
|
||||
return (bucket.head(key).orElse(null));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** RC.2 has no version GET facade. Reuse its configured system connection,
|
||||
** key normalization and limiter; closing the stream closes the S3 response.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public InputStream getVersion(String key, String versionId) throws IOException
|
||||
{
|
||||
if(!ExportFiles.validVersion(versionId))
|
||||
{
|
||||
throw (new IOException("Missing sealed object version"));
|
||||
}
|
||||
var connection = ObjectStorageRuntime.connection(bucket.config().getConnection());
|
||||
try(var ignored = connection.getRateLimiter().acquire())
|
||||
{
|
||||
var input = connection.client().getObject(GetObjectRequest.builder()
|
||||
.bucket(SecretRefs.resolveSetting(bucket.config().getBucketName())).key(StorageKeys.normalize(key)).versionId(versionId).build());
|
||||
if(!versionId.equals(input.response().versionId()))
|
||||
{
|
||||
input.close();
|
||||
throw (new IOException("Storage returned a different object version"));
|
||||
}
|
||||
return (input);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Forward bounded listing.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public Page<ObjectSummary> list(ListRequest request)
|
||||
{
|
||||
return (bucket.list(request));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,424 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.export;
|
||||
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.sql.Connection;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.canonical.Jcs;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.chain.Hashes;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.checkpoint.Checkpoints;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.VerificationRuns;
|
||||
import com.kingsrook.qbits.wormaudit.verify.VerificationResult;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSTransaction;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify the sealed evidence before comparing it to a repeatable live snapshot.
|
||||
** No exported content or provider/parser exception text enters diagnostics.
|
||||
*******************************************************************************/
|
||||
public class SealedExportVerifier
|
||||
{
|
||||
private final WormAuditQBitConfig config;
|
||||
private final SealedStore suppliedStorage;
|
||||
|
||||
/*******************************************************************************
|
||||
** Findings plus the number verified without live rows after a retention purge.
|
||||
*******************************************************************************/
|
||||
public record Report(VerificationResult result, long purged)
|
||||
{
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Production entry point.
|
||||
*******************************************************************************/
|
||||
public SealedExportVerifier(WormAuditQBitConfig config)
|
||||
{
|
||||
this(config, null);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Testable storage boundary.
|
||||
*******************************************************************************/
|
||||
public SealedExportVerifier(WormAuditQBitConfig config, SealedStore storage)
|
||||
{
|
||||
this.config = config;
|
||||
this.suppliedStorage = storage;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Persist exactly one verification row for this selected export, even on failure.
|
||||
*******************************************************************************/
|
||||
public Report verify(QRecord export, String trigger) throws QException
|
||||
{
|
||||
config.effectiveSettings();
|
||||
VerificationResult failure = new VerificationResult();
|
||||
failure.scope(0, export.getValueLong("from_link_id"), export.getValueLong("to_link_id"));
|
||||
Report report;
|
||||
WormAuditChainer.JVM_LOCK.lock();
|
||||
try(QBackendTransaction transaction = QBackendTransaction.openFor(new QueryInput(WormAuditTables.CHAIN));
|
||||
ExportFiles files = new ExportFiles())
|
||||
{
|
||||
try
|
||||
{
|
||||
if(transaction instanceof RDBMSTransaction rdbms)
|
||||
{
|
||||
rdbms.getConnection().setTransactionIsolation(Connection.TRANSACTION_REPEATABLE_READ);
|
||||
}
|
||||
SealedStore storage = suppliedStorage == null
|
||||
? config.getSealedStoreSupplier() == null || !config.getExportBucketAlias().equals(export.getValueString("bucket_alias"))
|
||||
? new ObjectStorageSealedStore(export.getValueString("bucket_alias"), "verify-" + export.getValueLong("id"))
|
||||
: config.getSealedStoreSupplier().get() : suppliedStorage;
|
||||
Map<String, Object> versions = ExportFiles.parse(export.getValueString("object_versions").getBytes(StandardCharsets.UTF_8));
|
||||
for(String name : ExportFiles.NAMES)
|
||||
{
|
||||
String key = export.getValueString("object_prefix") + name;
|
||||
if(!(versions.get(name) instanceof String version) || !ExportFiles.validVersion(version))
|
||||
{
|
||||
failure.add(null, null, "object " + name + " has no recorded sealed version id");
|
||||
throw (new IOException("Missing sealed object version"));
|
||||
}
|
||||
var head = storage.head(key);
|
||||
if(head == null)
|
||||
{
|
||||
failure.add(null, null, "object " + key + " is missing" + ("GOVERNANCE".equals(export.getValueString("lock_mode"))
|
||||
? "; possible GOVERNANCE bypass removal by an administrator" : ""));
|
||||
}
|
||||
if(head != null && !version.equals(head.getVersionId()))
|
||||
{
|
||||
failure.add(null, null, "object " + name + " was overwritten after sealing (current version " + head.getVersionId()
|
||||
+ ", sealed version " + version + " is still retained under Object Lock)");
|
||||
}
|
||||
if(!ExportFiles.locked(head, export.getValueString("lock_mode"), export.getValueInstant("retain_until")))
|
||||
{
|
||||
failure.add(null, null, "object " + name + " lacks the recorded " + export.getValueString("lock_mode") + " retention");
|
||||
}
|
||||
}
|
||||
for(String name : ExportFiles.NAMES)
|
||||
{
|
||||
String key = export.getValueString("object_prefix") + name;
|
||||
try(InputStream input = storage.getVersion(key, (String) versions.get(name)))
|
||||
{
|
||||
Files.copy(input, files.file(name));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
failure.add(null, null, "object " + key + " sealed version cannot be read" + ("GOVERNANCE".equals(export.getValueString("lock_mode"))
|
||||
? "; if removed, possible GOVERNANCE bypass removal by an administrator" : ""));
|
||||
throw (new IOException("Cannot read sealed object version"));
|
||||
}
|
||||
}
|
||||
report = inspect(files, export, new AuditStore(transaction));
|
||||
for(var item : failure.getBreaks())
|
||||
{
|
||||
report.result().add(item.linkId(), item.auditId(), item.reason());
|
||||
}
|
||||
transaction.commit();
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
transaction.rollback();
|
||||
throw (e);
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
failure.add(null, null, "Sealed export cannot be read or has invalid structure");
|
||||
report = new Report(failure, 0);
|
||||
}
|
||||
finally
|
||||
{
|
||||
WormAuditChainer.JVM_LOCK.unlock();
|
||||
}
|
||||
VerificationRuns.save(report.result(), trigger, export.getValueLong("id"));
|
||||
return (report);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Inspect local files, also used by the exporter before committing any objects.
|
||||
*******************************************************************************/
|
||||
public Report inspect(ExportFiles files, QRecord export, AuditStore store) throws Exception
|
||||
{
|
||||
VerificationResult result = new VerificationResult();
|
||||
long from = export.getValueLong("from_link_id");
|
||||
long to = export.getValueLong("to_link_id");
|
||||
result.scope(0, from, to);
|
||||
if(!Objects.equals(ExportFiles.sha256(files.file("manifest.json")), export.getValueString("manifest_sha256")))
|
||||
{
|
||||
result.add(null, null, "object manifest.json hash does not match the export row");
|
||||
return (new Report(result, 0));
|
||||
}
|
||||
if(Files.size(files.file("manifest.json")) > 65536)
|
||||
{
|
||||
result.add(null, null, "Manifest exceeds the format size limit");
|
||||
return (new Report(result, 0));
|
||||
}
|
||||
Map<String, Object> manifest = ExportFiles.parse(Files.readAllBytes(files.file("manifest.json")));
|
||||
if(!"worm-audit-export/1".equals(manifest.get("format")) || from < 1 || to < from
|
||||
|| !Objects.equals(manifest.get("fromLinkId"), from) || !Objects.equals(manifest.get("toLinkId"), to)
|
||||
|| !Objects.equals(manifest.get("linkCount"), to - from + 1)
|
||||
|| !Objects.equals(manifest.get("createdAt"), export.getValueInstant("sealed_at").toString())
|
||||
|| !java.util.Set.of("GOVERNANCE", "COMPLIANCE").contains(export.getValueString("lock_mode"))
|
||||
|| !Objects.equals(export.getValueString("manifest_key"), export.getValueString("object_prefix") + "manifest.json"))
|
||||
{
|
||||
result.add(null, null, "Manifest metadata does not match the export row");
|
||||
}
|
||||
QQueryFilter previousFilter = new QQueryFilter().withOrderBy(new QFilterOrderBy("id", false)).withLimit(1);
|
||||
if(export.getValueLong("id") != null)
|
||||
{
|
||||
previousFilter.withCriteria(new QFilterCriteria("id", QCriteriaOperator.LESS_THAN, export.getValueLong("id")));
|
||||
}
|
||||
List<QRecord> previousRows = store.query(WormAuditTables.EXPORT, previousFilter);
|
||||
QRecord previous = previousRows.isEmpty() ? null : previousRows.getFirst();
|
||||
String previousHash = previous == null ? null : previous.getValueString("manifest_sha256");
|
||||
if(!Objects.equals(manifest.get("prevManifestSha256"), previousHash)
|
||||
|| !Objects.equals(export.getValueString("prev_manifest_sha256"), previousHash)
|
||||
|| (previous != null && previous.getValueLong("to_link_id") != from - 1))
|
||||
{
|
||||
result.add(null, null, "prevManifestSha256 does not match the previous export row");
|
||||
}
|
||||
Map<?, ?> hashes = (Map<?, ?>) manifest.get("files");
|
||||
for(String name : List.of("links.jsonl", "checkpoints.jsonl"))
|
||||
{
|
||||
if(!Objects.equals(ExportFiles.sha256(files.file(name)), hashes.get(name)))
|
||||
{
|
||||
result.add(null, null, "object " + name + " hash does not match the manifest");
|
||||
}
|
||||
}
|
||||
if(!result.passed())
|
||||
{
|
||||
return (new Report(result, 0));
|
||||
}
|
||||
return (checkLines(files, export, store, manifest, result));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Consume links alongside their checkpoint envelopes. Only one checkpoint's
|
||||
** hashes and one content page are retained, never the whole export.
|
||||
*******************************************************************************/
|
||||
private Report checkLines(ExportFiles files, QRecord export, AuditStore store, Map<String, Object> manifest, VerificationResult result) throws Exception
|
||||
{
|
||||
long from = export.getValueLong("from_link_id");
|
||||
long to = export.getValueLong("to_link_id");
|
||||
long cursor = from - 1;
|
||||
long checkpointCount = 0;
|
||||
long checked = 0;
|
||||
long purged = 0;
|
||||
long anchor = store.anchor().getValueLong("anchor_link_id");
|
||||
String previousHash = from == 1 ? Hashes.GENESIS : null;
|
||||
if(cursor == anchor)
|
||||
{
|
||||
previousHash = store.anchor().getValueString("anchor_hash");
|
||||
}
|
||||
long lastCheckpoint = 0;
|
||||
try(BufferedReader links = Files.newBufferedReader(files.file("links.jsonl"), StandardCharsets.UTF_8);
|
||||
BufferedReader checkpoints = Files.newBufferedReader(files.file("checkpoints.jsonl"), StandardCharsets.UTF_8))
|
||||
{
|
||||
String line;
|
||||
while((line = checkpoints.readLine()) != null)
|
||||
{
|
||||
QRecord checkpoint = ExportFiles.record(ExportFiles.parse(line.getBytes(StandardCharsets.UTF_8)));
|
||||
long checkpointId = checkpoint.getValueLong("id");
|
||||
String label = "checkpoint " + WormAuditVerifier.number(checkpointId) + ": ";
|
||||
long end = checkpoint.getValueLong("to_link_id");
|
||||
long start = checkpoint.getValueLong("from_link_id");
|
||||
long count = checkpoint.getValueLong("link_count");
|
||||
if(start != cursor + 1 || end < start || end > to || count != end - start + 1
|
||||
|| count > Integer.MAX_VALUE || checkpointId <= lastCheckpoint)
|
||||
{
|
||||
result.add(null, null, label + "invalid or incomplete checkpoint range");
|
||||
break;
|
||||
}
|
||||
if(!Checkpoints.verify(checkpoint, config.getTrustedPublicKeys().get(checkpoint.getValueString("key_id"))))
|
||||
{
|
||||
result.add(null, null, label + "signature does not verify");
|
||||
}
|
||||
if(checkpointCount == 0 && !Objects.equals(export.getValueLong("from_checkpoint_id"), checkpointId))
|
||||
{
|
||||
result.add(null, null, label + "first checkpoint does not match the export row");
|
||||
}
|
||||
List<String> hashes = new ArrayList<>();
|
||||
while(cursor < end)
|
||||
{
|
||||
List<Map<String, Object>> page = new ArrayList<>();
|
||||
for(int i = 0, size = (int) Math.min(1000, end - cursor); i < size; i++)
|
||||
{
|
||||
String linkLine = links.readLine();
|
||||
if(linkLine == null)
|
||||
{
|
||||
result.add(cursor + 1, null, "Sealed links are missing from the checkpoint range");
|
||||
result.scope(checked, from, to);
|
||||
return (new Report(result, purged));
|
||||
}
|
||||
page.add(ExportFiles.parse(linkLine.getBytes(StandardCharsets.UTF_8)));
|
||||
}
|
||||
compareLive(page, store, export, anchor, result);
|
||||
for(Map<String, Object> document : page)
|
||||
{
|
||||
Map<?, ?> link = (Map<?, ?>) document.get("link");
|
||||
long id = (Long) link.get("id");
|
||||
long auditId = (Long) link.get("auditId");
|
||||
String prefix = "link " + WormAuditVerifier.number(id) + " (audit " + WormAuditVerifier.number(auditId) + "): ";
|
||||
String prev = (String) link.get("prevHash");
|
||||
String hash = (String) link.get("linkHash");
|
||||
if(id != cursor + 1 || (previousHash != null && !Objects.equals(prev, previousHash)))
|
||||
{
|
||||
result.add(id, auditId, prefix + "sealed chain continuity differs");
|
||||
}
|
||||
Map<?, ?> content = (Map<?, ?>) document.get("content");
|
||||
Map<?, ?> audit = (Map<?, ?>) content.get("audit");
|
||||
List<?> detailIds = (List<?>) link.get("detailIds");
|
||||
List<?> details = (List<?>) content.get("details");
|
||||
List<?> contentIds = details.stream().map(d -> ((Map<?, ?>) d).get("id")).toList();
|
||||
if(!Objects.equals(audit.get("id"), auditId) || !Objects.equals(content.get("v"), 1L)
|
||||
|| !List.of("AUDIT", "LATE_DETAIL").contains(link.get("kind"))
|
||||
|| detailIds.size() > 250 || !detailIds.equals(contentIds)
|
||||
|| detailIds.stream().distinct().count() != detailIds.size()
|
||||
|| ("LATE_DETAIL".equals(link.get("kind")) && detailIds.isEmpty()))
|
||||
{
|
||||
result.add(id, auditId, prefix + "sealed content identifiers or coverage differ");
|
||||
}
|
||||
if(!Hashes.link(prev, Jcs.bytes(content)).equals(hash))
|
||||
{
|
||||
result.add(id, auditId, prefix + "sealed content differs from the chained hash");
|
||||
}
|
||||
hashes.add(hash);
|
||||
previousHash = hash;
|
||||
cursor++;
|
||||
checked++;
|
||||
if(id <= anchor)
|
||||
{
|
||||
purged++;
|
||||
}
|
||||
}
|
||||
}
|
||||
if(!Objects.equals(Hashes.merkle(hashes), checkpoint.getValueString("merkle_root"))
|
||||
|| !Objects.equals(previousHash, checkpoint.getValueString("head_hash")))
|
||||
{
|
||||
result.add(null, null, label + "Merkle root or head hash differs");
|
||||
}
|
||||
lastCheckpoint = checkpointId;
|
||||
checkpointCount++;
|
||||
}
|
||||
if(cursor != to || links.readLine() != null || !Objects.equals(manifest.get("checkpointCount"), checkpointCount)
|
||||
|| !Objects.equals(export.getValueLong("to_checkpoint_id"), lastCheckpoint)
|
||||
|| !Objects.equals(manifest.get("headHash"), previousHash))
|
||||
{
|
||||
result.add(null, null, "Sealed range, counts or head hash differ from the manifest");
|
||||
}
|
||||
}
|
||||
result.scope(checked, from, to);
|
||||
return (new Report(result, purged));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Compare a bounded page by exported ids, so missing live rows cannot vanish
|
||||
** from the verification scope. Purged ids are checked from signed copies only.
|
||||
*******************************************************************************/
|
||||
private void compareLive(List<Map<String, Object>> page, AuditStore store, QRecord export, long anchor, VerificationResult result) throws Exception
|
||||
{
|
||||
List<QRecord> sealed = page.stream().map(document ->
|
||||
{
|
||||
Map<?, ?> link = (Map<?, ?>) document.get("link");
|
||||
List<?> ids = (List<?>) link.get("detailIds");
|
||||
return (new QRecord().withValue("id", (Long) link.get("id")).withValue("audit_id", (Long) link.get("auditId"))
|
||||
.withValue("kind", (String) link.get("kind")).withValue("prev_hash", (String) link.get("prevHash"))
|
||||
.withValue("link_hash", (String) link.get("linkHash"))
|
||||
.withValue("detail_ids", ids.stream().map(Object::toString).collect(java.util.stream.Collectors.joining(","))));
|
||||
}).toList();
|
||||
List<QRecord> liveScope = sealed.stream().filter(r -> r.getValueLong("id") > anchor).toList();
|
||||
Map<Long, QRecord> live = AuditStore.index(store.in(WormAuditTables.CHAIN, "id", liveScope.stream().map(r -> r.getValueLong("id")).toList()));
|
||||
Map<Long, QRecord> audits = AuditStore.index(store.in("audit", "id", liveScope.stream().map(r -> r.getValueLong("audit_id")).toList()));
|
||||
Map<Long, QRecord> details = ExportFiles.details(store, liveScope);
|
||||
for(int i = 0; i < sealed.size(); i++)
|
||||
{
|
||||
QRecord link = sealed.get(i);
|
||||
long id = link.getValueLong("id");
|
||||
long auditId = link.getValueLong("audit_id");
|
||||
if(id <= anchor)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
String prefix = "link " + WormAuditVerifier.number(id) + " (audit " + WormAuditVerifier.number(auditId) + "): ";
|
||||
QRecord current = live.get(id);
|
||||
if(current == null)
|
||||
{
|
||||
result.add(id, auditId, "link " + WormAuditVerifier.number(id) + " is missing from the live chain");
|
||||
}
|
||||
else
|
||||
{
|
||||
for(String field : List.of("kind", "audit_id", "detail_ids", "prev_hash", "link_hash"))
|
||||
{
|
||||
if(!Objects.equals(link.getValue(field), current.getValue(field)))
|
||||
{
|
||||
result.add(id, auditId, prefix + "live chain " + field + " differs from the sealed copy");
|
||||
}
|
||||
}
|
||||
}
|
||||
try
|
||||
{
|
||||
QRecord audit = audits.get(auditId);
|
||||
if(audit == null)
|
||||
{
|
||||
result.add(id, auditId, prefix + "live audit header is missing");
|
||||
}
|
||||
else if(!Arrays.equals(store.canonical(audit, AuditStore.detailIds(link), details), Jcs.bytes(page.get(i).get("content"))))
|
||||
{
|
||||
result.add(id, auditId, prefix + "live content differs from the sealed copy (sealed "
|
||||
+ export.getValueInstant("sealed_at").atZone(ZoneOffset.UTC).toLocalDate() + ")");
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
result.add(id, auditId, prefix + "live content is missing or invalid");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,392 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.export;
|
||||
|
||||
|
||||
import java.io.BufferedWriter;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.Statement;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitProducer;
|
||||
import com.kingsrook.qbits.wormaudit.canonical.Jcs;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.chain.Hashes;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertSignal;
|
||||
import com.kingsrook.qqq.backend.core.alerts.QAlerts;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSTransaction;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Seal the checkpointed suffix under the chainer/purger lock. Only successful
|
||||
** uploads with verified lock echoes produce an export row.
|
||||
*******************************************************************************/
|
||||
public class SealedExporter
|
||||
{
|
||||
private final WormAuditQBitConfig config;
|
||||
private final SealedStore suppliedStorage;
|
||||
|
||||
/*******************************************************************************
|
||||
** A review scope; execution always recomputes it under the writer lock.
|
||||
*******************************************************************************/
|
||||
public record Scope(long from, long to, long checkpoints, QRecord previous)
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Number of links, including the empty checkpoint suffix.
|
||||
*******************************************************************************/
|
||||
public long count()
|
||||
{
|
||||
return (Math.max(0, to - from + 1));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Value-free Review text.
|
||||
*******************************************************************************/
|
||||
public String review()
|
||||
{
|
||||
return (count() == 0 ? "Nothing to seal: no new checkpoint" : WormAuditVerifier.number(count())
|
||||
+ " chain links will be sealed (links " + WormAuditVerifier.number(from) + "–" + WormAuditVerifier.number(to)
|
||||
+ ", " + WormAuditVerifier.number(checkpoints) + " checkpoints)");
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** A completed, empty, or failed run suitable for a Result line.
|
||||
*******************************************************************************/
|
||||
public record Result(QRecord row, String message, boolean failed)
|
||||
{
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Production storage is resolved lazily, so an empty run needs no bucket.
|
||||
*******************************************************************************/
|
||||
public SealedExporter(WormAuditQBitConfig config)
|
||||
{
|
||||
this(config, null);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Inject the tiny storage boundary for deterministic tests.
|
||||
*******************************************************************************/
|
||||
public SealedExporter(WormAuditQBitConfig config, SealedStore storage)
|
||||
{
|
||||
this.config = config;
|
||||
this.suppliedStorage = storage;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Select the contiguous suffix, never silently skipping an unexported purge.
|
||||
*******************************************************************************/
|
||||
public static Scope scope(AuditStore store) throws QException
|
||||
{
|
||||
QRecord previous = store.last(WormAuditTables.EXPORT);
|
||||
long anchor = store.anchor().getValueLong("anchor_link_id");
|
||||
long after = previous == null ? anchor : previous.getValueLong("to_link_id");
|
||||
if(after < anchor)
|
||||
{
|
||||
throw (new QException("Unexported links were purged"));
|
||||
}
|
||||
QRecord checkpoint = store.last(WormAuditTables.CHECKPOINT);
|
||||
long to = checkpoint == null ? after : Math.max(after, checkpoint.getValueLong("to_link_id"));
|
||||
long count = 0;
|
||||
long cursor = 0;
|
||||
while(true)
|
||||
{
|
||||
List<QRecord> page = checkpoints(store, after + 1, to, cursor);
|
||||
if(page.isEmpty())
|
||||
{
|
||||
break;
|
||||
}
|
||||
count += page.size();
|
||||
cursor = page.getLast().getValueLong("id");
|
||||
}
|
||||
return (new Scope(after + 1, to, count, previous));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Select checkpoint envelopes wholly contained in the export range.
|
||||
*******************************************************************************/
|
||||
private static List<QRecord> checkpoints(AuditStore store, long from, long to, long after) throws QException
|
||||
{
|
||||
return (store.query(WormAuditTables.CHECKPOINT, new QQueryFilter(
|
||||
new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, after),
|
||||
new QFilterCriteria("from_link_id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, from),
|
||||
new QFilterCriteria("to_link_id", QCriteriaOperator.LESS_THAN_OR_EQUALS, to))
|
||||
.withOrderBy(new QFilterOrderBy("id", true)).withLimit(1000)));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Execute with alerts on user and scheduled failures; schedules always beat.
|
||||
*******************************************************************************/
|
||||
public Result run(String trigger, Instant now)
|
||||
{
|
||||
try
|
||||
{
|
||||
config.validate();
|
||||
Result result = lockedRun(now);
|
||||
QAlerts.resolve("wormaudit.export.failed", "export", "Sealed audit export completed");
|
||||
return (result);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
// All exposed failures originate here or use a fixed, value-free code.
|
||||
String reason = e instanceof ExportFailure ? e.getMessage() : "Sealed export failed: storage or database operation failed";
|
||||
QAlerts.raise(new AlertSignal("wormaudit.export.failed").withDedupKey("export").withTitle(reason));
|
||||
return (new Result(null, reason, true));
|
||||
}
|
||||
finally
|
||||
{
|
||||
if("SCHEDULE".equals(trigger))
|
||||
{
|
||||
QAlerts.heartbeat("wormaudit.export.daily");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Serialize with writers in this JVM and PostgreSQL writers on other hosts.
|
||||
*******************************************************************************/
|
||||
private Result lockedRun(Instant now) throws Exception
|
||||
{
|
||||
WormAuditChainer.JVM_LOCK.lock();
|
||||
try(QBackendTransaction transaction = QBackendTransaction.openFor(new QueryInput(WormAuditTables.CHAIN)))
|
||||
{
|
||||
try
|
||||
{
|
||||
if(transaction instanceof RDBMSTransaction rdbms && "PostgreSQL".equals(rdbms.getConnection().getMetaData().getDatabaseProductName()))
|
||||
{
|
||||
try(Statement statement = rdbms.getConnection().createStatement();
|
||||
ResultSet result = statement.executeQuery("SELECT pg_advisory_xact_lock(" + WormAuditChainer.ADVISORY_LOCK + ")"))
|
||||
{
|
||||
result.next();
|
||||
}
|
||||
}
|
||||
AuditStore store = new AuditStore(transaction);
|
||||
Scope scope = scope(store);
|
||||
Result result;
|
||||
if(scope.count() == 0)
|
||||
{
|
||||
result = new Result(null, scope.review(), false);
|
||||
}
|
||||
else
|
||||
{
|
||||
SealedStore storage = suppliedStorage == null
|
||||
? config.getSealedStoreSupplier() == null ? new ObjectStorageSealedStore(config.getExportBucketAlias(), java.util.UUID.randomUUID().toString())
|
||||
: config.getSealedStoreSupplier().get() : suppliedStorage;
|
||||
result = seal(store, storage, scope, now.truncatedTo(ChronoUnit.MILLIS));
|
||||
}
|
||||
transaction.commit();
|
||||
return (result);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
transaction.rollback();
|
||||
throw (e);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
WormAuditChainer.JVM_LOCK.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build locally before uploading. Partial remote uploads have no database row.
|
||||
*******************************************************************************/
|
||||
private Result seal(AuditStore store, SealedStore storage, Scope scope, Instant now) throws Exception
|
||||
{
|
||||
if(!storage.supportsLock())
|
||||
{
|
||||
throw (new ExportFailure("Export refused: bucket does not support Object Lock"));
|
||||
}
|
||||
var settings = config.effectiveSettings();
|
||||
var mode = com.kingsrook.qbits.objectstorage.api.LockMode.valueOf(settings.export_lock_mode());
|
||||
Instant until = now.plus(settings.export_retention_days(), ChronoUnit.DAYS);
|
||||
String prefix = config.getExportPrefix() + "/" + DateTimeFormatter.ofPattern("yyyy/MM/dd").withZone(ZoneOffset.UTC).format(now)
|
||||
+ "/" + scope.from() + "-" + scope.to() + "/";
|
||||
QRecord row;
|
||||
try(ExportFiles files = new ExportFiles())
|
||||
{
|
||||
String head = writeLinks(files, store, scope);
|
||||
long[] checkpointIds = writeCheckpoints(files, store, scope);
|
||||
Map<String, Object> manifest = new LinkedHashMap<>();
|
||||
manifest.put("format", "worm-audit-export/1");
|
||||
manifest.put("fromLinkId", scope.from());
|
||||
manifest.put("toLinkId", scope.to());
|
||||
manifest.put("linkCount", scope.count());
|
||||
manifest.put("checkpointCount", scope.checkpoints());
|
||||
manifest.put("files", Map.of("links.jsonl", ExportFiles.sha256(files.file("links.jsonl")),
|
||||
"checkpoints.jsonl", ExportFiles.sha256(files.file("checkpoints.jsonl"))));
|
||||
manifest.put("headHash", head);
|
||||
String previous = scope.previous() == null ? null : scope.previous().getValueString("manifest_sha256");
|
||||
manifest.put("prevManifestSha256", previous);
|
||||
manifest.put("qbitVersion", WormAuditQBitProducer.VERSION);
|
||||
manifest.put("createdAt", now.toString());
|
||||
Files.write(files.file("manifest.json"), Jcs.bytes(manifest));
|
||||
row = new QRecord().withValue("from_link_id", scope.from()).withValue("to_link_id", scope.to())
|
||||
.withValue("from_checkpoint_id", checkpointIds[0]).withValue("to_checkpoint_id", checkpointIds[1])
|
||||
.withValue("bucket_alias", config.getExportBucketAlias()).withValue("object_prefix", prefix)
|
||||
.withValue("manifest_key", prefix + "manifest.json").withValue("manifest_sha256", ExportFiles.sha256(files.file("manifest.json")))
|
||||
.withValue("prev_manifest_sha256", previous).withValue("object_count", 3L)
|
||||
.withValue("lock_mode", mode.name()).withValue("retain_until", until).withValue("sealed_at", now);
|
||||
if(!new SealedExportVerifier(config, storage).inspect(files, row, store).result().passed())
|
||||
{
|
||||
throw (new ExportFailure("Export refused: source does not match the signed chain"));
|
||||
}
|
||||
long bytes = 0;
|
||||
Map<String, String> versions = new LinkedHashMap<>();
|
||||
for(String name : ExportFiles.NAMES)
|
||||
{
|
||||
String version = storage.put(prefix + name, files.file(name), mode, until);
|
||||
if(!ExportFiles.validVersion(version))
|
||||
{
|
||||
throw (new ExportFailure("Export refused: object " + name + " returned no version id"));
|
||||
}
|
||||
var headObject = storage.head(prefix + name);
|
||||
if(!ExportFiles.locked(headObject, mode.name(), until))
|
||||
{
|
||||
throw (new ExportFailure("Export refused: object " + name + " did not echo " + mode.name() + " retention"));
|
||||
}
|
||||
if(!version.equals(headObject.getVersionId()))
|
||||
{
|
||||
throw (new ExportFailure("Export refused: object " + name + " version changed after upload"));
|
||||
}
|
||||
versions.put(name, version);
|
||||
bytes += Files.size(files.file(name));
|
||||
}
|
||||
row.withValue("byte_count", bytes).withValue("object_versions", Jcs.string(versions));
|
||||
}
|
||||
row = store.insert(WormAuditTables.EXPORT, List.of(row)).getFirst();
|
||||
return (new Result(row, WormAuditVerifier.number(scope.count()) + " links sealed in export " + row.getValueLong("id")
|
||||
+ " (retained until " + until + ")", false));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stream canonical lines in pages of 1000 with batched content lookups.
|
||||
*******************************************************************************/
|
||||
private String writeLinks(ExportFiles files, AuditStore store, Scope scope) throws Exception
|
||||
{
|
||||
long cursor = scope.from() - 1;
|
||||
String head = null;
|
||||
try(BufferedWriter writer = Files.newBufferedWriter(files.file("links.jsonl"), StandardCharsets.UTF_8))
|
||||
{
|
||||
while(cursor < scope.to())
|
||||
{
|
||||
List<QRecord> links = store.page(WormAuditTables.CHAIN, cursor, (int) Math.min(1000, scope.to() - cursor));
|
||||
if(links.isEmpty())
|
||||
{
|
||||
throw (new ExportFailure("Export refused: chain range is incomplete"));
|
||||
}
|
||||
Map<Long, QRecord> audits = AuditStore.index(store.in("audit", "id", links.stream().map(r -> r.getValueLong("audit_id")).toList()));
|
||||
Map<Long, QRecord> details = ExportFiles.details(store, links);
|
||||
for(QRecord link : links)
|
||||
{
|
||||
if(link.getValueLong("id") != cursor + 1 || link.getValueLong("id") > scope.to())
|
||||
{
|
||||
throw (new ExportFailure("Export refused: chain range is incomplete"));
|
||||
}
|
||||
byte[] content = store.canonical(audits.get(link.getValueLong("audit_id")), AuditStore.detailIds(link), details);
|
||||
if(!Hashes.link(link.getValueString("prev_hash"), content).equals(link.getValueString("link_hash")))
|
||||
{
|
||||
throw (new ExportFailure("Export refused: content differs at link " + link.getValueLong("id")));
|
||||
}
|
||||
writer.write(Jcs.string(Map.of("link", ExportFiles.link(link), "content", ExportFiles.parse(content))));
|
||||
writer.write('\n');
|
||||
head = link.getValueString("link_hash");
|
||||
cursor = link.getValueLong("id");
|
||||
}
|
||||
}
|
||||
}
|
||||
return (head);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stream complete signed checkpoint envelopes in keyset pages.
|
||||
*******************************************************************************/
|
||||
private long[] writeCheckpoints(ExportFiles files, AuditStore store, Scope scope) throws Exception
|
||||
{
|
||||
long first = 0;
|
||||
long cursor = 0;
|
||||
try(BufferedWriter writer = Files.newBufferedWriter(files.file("checkpoints.jsonl"), StandardCharsets.UTF_8))
|
||||
{
|
||||
while(true)
|
||||
{
|
||||
List<QRecord> page = checkpoints(store, scope.from(), scope.to(), cursor);
|
||||
if(page.isEmpty())
|
||||
{
|
||||
break;
|
||||
}
|
||||
for(QRecord row : page)
|
||||
{
|
||||
first = first == 0 ? row.getValueLong("id") : first;
|
||||
cursor = row.getValueLong("id");
|
||||
writer.write(Jcs.string(ExportFiles.checkpoint(row)));
|
||||
writer.write('\n');
|
||||
}
|
||||
}
|
||||
}
|
||||
return (new long[] { first, cursor });
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Only fixed messages constructed here may reach an alert or Result screen.
|
||||
*******************************************************************************/
|
||||
private static class ExportFailure extends IOException
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Carry a value-free exporter reason without a provider exception cause.
|
||||
*******************************************************************************/
|
||||
ExportFailure(String reason)
|
||||
{
|
||||
super(reason);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.export;
|
||||
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Instant;
|
||||
import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
|
||||
import com.kingsrook.qbits.objectstorage.api.ListRequest;
|
||||
import com.kingsrook.qbits.objectstorage.api.ObjectSummary;
|
||||
import com.kingsrook.qbits.objectstorage.api.Page;
|
||||
import com.kingsrook.qbits.objectstorage.api.StoredObject;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Minimal storage boundary. The application never deletes sealed objects.
|
||||
*******************************************************************************/
|
||||
public interface SealedStore
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Only positively confirmed Object Lock support is acceptable.
|
||||
*******************************************************************************/
|
||||
boolean supportsLock();
|
||||
|
||||
/*******************************************************************************
|
||||
** Preserve unknown provider capabilities for read-only setup diagnostics.
|
||||
*******************************************************************************/
|
||||
default CapabilityStatus lockCapability()
|
||||
{
|
||||
return (supportsLock() ? CapabilityStatus.SUPPORTED : CapabilityStatus.UNSUPPORTED);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Upload a file with the selected Object Lock mode and retention.
|
||||
*******************************************************************************/
|
||||
String put(String key, Path file, com.kingsrook.qbits.objectstorage.api.LockMode mode, Instant retainUntil) throws Exception;
|
||||
|
||||
/*******************************************************************************
|
||||
** Read current object metadata, or null for an absent object.
|
||||
*******************************************************************************/
|
||||
StoredObject head(String key) throws Exception;
|
||||
|
||||
/*******************************************************************************
|
||||
** Open the exact sealed version; the caller owns and closes the stream.
|
||||
*******************************************************************************/
|
||||
InputStream getVersion(String key, String versionId) throws Exception;
|
||||
|
||||
/*******************************************************************************
|
||||
** List one bounded page.
|
||||
*******************************************************************************/
|
||||
Page<ObjectSummary> list(ListRequest request) throws Exception;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.guard;
|
||||
|
||||
|
||||
import java.sql.SQLException;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Metadata defense and a value-free mapping of the database write-once error.
|
||||
*******************************************************************************/
|
||||
public class AppendOnlyGuard implements MetaDataCustomizerInterface<QTableMetaData>
|
||||
{
|
||||
public static final List<String> CORE_TABLES = List.of("audit", "auditDetail", "auditTable", "auditUser");
|
||||
public static final String MESSAGE = "Audit records are write-once and cannot be changed or deleted.";
|
||||
|
||||
/*******************************************************************************
|
||||
** Protect core and sidecar audit metadata after host customization.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QTableMetaData customizeMetaData(QInstance instance, QTableMetaData table)
|
||||
{
|
||||
if(com.kingsrook.qbits.wormaudit.model.WormAuditTables.SETTINGS.equals(table.getName()))
|
||||
{
|
||||
table.withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_DELETE);
|
||||
table.withAuditRules(new com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules()
|
||||
.withAuditLevel(com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel.FIELD));
|
||||
return (table);
|
||||
}
|
||||
if(CORE_TABLES.contains(table.getName()) || table.getName().startsWith("worm_audit_"))
|
||||
{
|
||||
table.withoutCapabilities(Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
|
||||
}
|
||||
if(table.getName().startsWith("worm_audit_"))
|
||||
{
|
||||
table.withoutCapabilities(Capability.TABLE_INSERT);
|
||||
}
|
||||
return (table);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Preserve unrelated failures, map only the documented SQLSTATE and prefix.
|
||||
*******************************************************************************/
|
||||
public static QException map(Exception error)
|
||||
{
|
||||
for(Throwable cause = error; cause != null; cause = cause.getCause())
|
||||
{
|
||||
if(cause instanceof SQLException sql && "P0001".equals(sql.getSQLState())
|
||||
&& sql.getMessage() != null && sql.getMessage().startsWith("worm_audit:"))
|
||||
{
|
||||
return (new QUserFacingException(MESSAGE));
|
||||
}
|
||||
// The PostgreSQL driver prepends ERROR: to server error messages.
|
||||
if(cause instanceof SQLException sql && "P0001".equals(sql.getSQLState())
|
||||
&& sql.getMessage() != null && sql.getMessage().startsWith("ERROR: worm_audit:"))
|
||||
{
|
||||
return (new QUserFacingException(MESSAGE));
|
||||
}
|
||||
}
|
||||
return (error instanceof QException q ? q : new QException("WORM audit operation failed", error));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.model;
|
||||
|
||||
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChildRecordListData;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A standard child-record list for a checkpoint/export/verification link range.
|
||||
** This is a record-screen relation, not an app-home widget. Reads retain user
|
||||
** permissions and use a bounded page with a filtered View All link.
|
||||
*******************************************************************************/
|
||||
public class LinkRangeWidget extends AbstractWidgetRenderer
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Render the inclusive range stored on the parent record.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public RenderWidgetOutput render(RenderWidgetInput input) throws QException
|
||||
{
|
||||
String parent = (String) input.getWidgetMetaData().getDefaultValues().get("parentTable");
|
||||
GetInput get = new GetInput().withPrimaryKey(input.getQueryParams().get("id")).withInputSource(QInputSource.USER);
|
||||
get.setTableName(parent);
|
||||
get.withPermissionEnforcement(PermissionEnforcement.ENFORCE);
|
||||
QRecord record = new GetAction().execute(get).getRecord();
|
||||
if(record == null)
|
||||
{
|
||||
throw (new QNotFoundException("Audit range record was not found"));
|
||||
}
|
||||
Long from = record.getValueLong("from_link_id");
|
||||
Long to = record.getValueLong("to_link_id");
|
||||
QQueryFilter filter = new QQueryFilter()
|
||||
.withCriteria(new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, from == null ? 1L : from))
|
||||
.withCriteria(new QFilterCriteria("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, to == null ? 0L : to))
|
||||
.withOrderBy(new QFilterOrderBy("id", true));
|
||||
CountInput count = new CountInput(WormAuditTables.CHAIN).withFilter(filter).withInputSource(QInputSource.USER);
|
||||
count.withPermissionEnforcement(PermissionEnforcement.ENFORCE);
|
||||
int total = new CountAction().execute(count).getCount();
|
||||
String path = QContext.getQInstance().getTablePath(WormAuditTables.CHAIN);
|
||||
String viewAll = path + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), StandardCharsets.UTF_8);
|
||||
QueryInput query = new QueryInput(WormAuditTables.CHAIN).withFilter(filter.clone().withLimit(100)).withInputSource(QInputSource.USER);
|
||||
query.withShouldTranslatePossibleValues(true);
|
||||
query.withPermissionEnforcement(PermissionEnforcement.ENFORCE);
|
||||
return (new RenderWidgetOutput(new ChildRecordListData("Chain links", new QueryAction().execute(query),
|
||||
QContext.getQInstance().getTable(WormAuditTables.CHAIN), path, viewAll, total)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.model;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditSettings;
|
||||
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Validate the editable singleton, including internal insert-only seeds.
|
||||
*******************************************************************************/
|
||||
public class SettingsCustomizer implements TableCustomizerInterface
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Only the fixed singleton may be inserted by the internal seed path.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> preInsert(InsertInput input, List<QRecord> records, boolean preview) throws QException
|
||||
{
|
||||
if(records.size() != 1 || !Integer.valueOf(1).equals(records.getFirst().getValueInteger("id")))
|
||||
{
|
||||
throw (new QUserFacingException("WORM Audit Settings has one row only (ID 1)."));
|
||||
}
|
||||
if(new com.kingsrook.qbits.wormaudit.chain.AuditStore(input.getTransaction()).get(WormAuditTables.SETTINGS, 1) != null)
|
||||
{
|
||||
throw (new QUserFacingException("WORM Audit Settings already exists. Edit record 1 instead of adding another record."));
|
||||
}
|
||||
return (validate(records));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Validate partial edits without replacing the existing settings.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> preUpdate(UpdateInput input, List<QRecord> records, boolean preview, Optional<List<QRecord>> oldRecords) throws QException
|
||||
{
|
||||
PermissionsHelper.checkTablePermissionThrowing(input, TablePermissionSubType.EDIT);
|
||||
return (validate(records));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reject deletion even for direct core calls where capability checks are off.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> preDelete(DeleteInput input, List<QRecord> records, boolean preview) throws QException
|
||||
{
|
||||
throw (new QUserFacingException("WORM Audit Settings cannot be deleted."));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Refresh local settings after persistence; remote JVMs retain the bounded cache.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> postUpdate(UpdateInput input, List<QRecord> records, Optional<List<QRecord>> oldRecords) throws QException
|
||||
{
|
||||
com.kingsrook.qbits.wormaudit.processes.VerificationRuns.config().invalidateSettings();
|
||||
return (records);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stamp the editor and time on every valid write; core writes FIELD audits.
|
||||
*******************************************************************************/
|
||||
private List<QRecord> validate(List<QRecord> records) throws QException
|
||||
{
|
||||
for(QRecord row : records)
|
||||
{
|
||||
try
|
||||
{
|
||||
WormAuditSettings.validate(row);
|
||||
}
|
||||
catch(IllegalArgumentException e)
|
||||
{
|
||||
throw (new QUserFacingException(e.getMessage()));
|
||||
}
|
||||
row.setValue("modify_date", Instant.now());
|
||||
row.setValue("modified_by", QContext.getQSession() == null || QContext.getQSession().getUser() == null
|
||||
? "system" : QContext.getQSession().getUser().getIdReference());
|
||||
}
|
||||
return (records);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.model;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.CustomPermissionChecker;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reading uses the full table name; editing additionally needs settings access.
|
||||
*******************************************************************************/
|
||||
public class SettingsPermissions implements CustomPermissionChecker
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Enforce table permission and the separate settings permission.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void checkPermissionsThrowing(AbstractActionInput input, MetaDataWithPermissionRules metadata) throws QPermissionDeniedException
|
||||
{
|
||||
var session = QContext.getQSession();
|
||||
boolean edit = input instanceof UpdateInput;
|
||||
if(input instanceof InsertInput || input instanceof DeleteInput || session == null
|
||||
|| !session.hasPermission(WormAuditTables.SETTINGS + (edit ? ".edit" : ".read"))
|
||||
|| (edit && !session.hasPermission("worm_audit.settings")))
|
||||
{
|
||||
throw (new QPermissionDeniedException(edit ? "Editing WORM Audit Settings requires worm_audit_settings.edit and worm_audit.settings."
|
||||
: "WORM Audit Settings requires read access; adding or deleting settings is not allowed."));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.model;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Applies help/qbit-worm-audit.yaml (the QQQ help-bundle format) to the WORM
|
||||
** Audit app and its worm_audit_* tables: what each table is for and how its
|
||||
** rows get created, shown on the table screen and when it is empty.
|
||||
** The bundle's app entry is keyed to the default app name; a host that renames
|
||||
** the app gets the same text.
|
||||
** STAND-IN(core=QRun-IO/qqq#1198, removeBy=4.1.0-RC.7): replace with
|
||||
** HelpBundles.register once the pinned QQQ release includes it; the bundle
|
||||
** file stays the same.
|
||||
*******************************************************************************/
|
||||
public final class WormAuditHelp
|
||||
{
|
||||
public static final String RESOURCE = "/help/qbit-worm-audit.yaml";
|
||||
public static final String SLOT = "help";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private WormAuditHelp()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The bundle's entries by key, validated (fails startup if malformed).
|
||||
*******************************************************************************/
|
||||
public static Map<String, JsonNode> entries() throws QException
|
||||
{
|
||||
try(InputStream stream = WormAuditHelp.class.getResourceAsStream(RESOURCE))
|
||||
{
|
||||
if(stream == null)
|
||||
{
|
||||
throw (new IOException("missing resource"));
|
||||
}
|
||||
JsonNode bundle = new ObjectMapper(new YAMLFactory()).readTree(stream);
|
||||
if(!"qbit-worm-audit".equals(bundle.path("bundle").asText()) || !"QBIT".equals(bundle.path("layer").asText()))
|
||||
{
|
||||
throw (new IOException("invalid bundle header"));
|
||||
}
|
||||
Map<String, JsonNode> entries = new LinkedHashMap<>();
|
||||
for(JsonNode entry : bundle.path("entries"))
|
||||
{
|
||||
String key = entry.path("key").asText();
|
||||
String summary = entry.path("summary").asText();
|
||||
if(key.isBlank() || summary.isBlank() || summary.length() > 120 || entry.path("content").asText().isBlank() || entries.put(key, entry) != null)
|
||||
{
|
||||
throw (new IOException("invalid or duplicate entry: " + key));
|
||||
}
|
||||
}
|
||||
return (entries);
|
||||
}
|
||||
catch(IOException | RuntimeException e)
|
||||
{
|
||||
throw (new QException("Could not load " + RESOURCE, e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Apply the help to the app and every worm_audit_* table. A host's own help on
|
||||
** a table or the app is kept (the bundle only fills the slot).
|
||||
*******************************************************************************/
|
||||
public static void apply(QInstance qInstance, WormAuditQBitConfig config) throws QException
|
||||
{
|
||||
for(Map.Entry<String, JsonNode> entry : entries().entrySet())
|
||||
{
|
||||
String key = entry.getKey();
|
||||
if(key.startsWith("table:"))
|
||||
{
|
||||
QTableMetaData table = qInstance.getTable(key.substring("table:".length(), key.indexOf(';')));
|
||||
if(table != null && (table.getHelpContent() == null || !table.getHelpContent().containsKey(SLOT)))
|
||||
{
|
||||
table.withHelpContent(SLOT, content(entry.getValue()));
|
||||
}
|
||||
}
|
||||
else if(config.getIncludeApp() && key.startsWith("app:"))
|
||||
{
|
||||
QAppMetaData app = qInstance.getApp(config.getAppName());
|
||||
if(app != null && (app.getHelpContent() == null || !app.getHelpContent().containsKey(SLOT)))
|
||||
{
|
||||
app.withHelpContent(SLOT, content(entry.getValue()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Summary as the first paragraph, then the content, as markdown.
|
||||
*******************************************************************************/
|
||||
static QHelpContent content(JsonNode entry)
|
||||
{
|
||||
return (new QHelpContent().withContentAsMarkdown(entry.path("summary").asText().strip() + "\n\n" + entry.path("content").asText().strip()));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,318 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.model;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditSettings;
|
||||
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Append-only sidecar tables. Snake-case fields map directly to DDL columns.
|
||||
** Labels also supply the standard QQQ app navigation wording.
|
||||
*******************************************************************************/
|
||||
public final class WormAuditTables
|
||||
{
|
||||
public static final String CHAIN = "worm_audit_chain";
|
||||
public static final String CHAIN_DETAIL = "worm_audit_chain_detail";
|
||||
public static final String CHECKPOINT = "worm_audit_checkpoint";
|
||||
public static final String EXPORT = "worm_audit_export";
|
||||
public static final String VERIFICATION = "worm_audit_verification";
|
||||
public static final String PURGE = "worm_audit_purge";
|
||||
public static final String SETTINGS = "worm_audit_settings";
|
||||
public static final List<String> ALL = List.of(CHAIN, CHAIN_DETAIL, CHECKPOINT, EXPORT, VERIFICATION, PURGE, SETTINGS);
|
||||
public static final List<String> NAV = List.of(VERIFICATION, EXPORT, CHECKPOINT, CHAIN, CHAIN_DETAIL, PURGE, SETTINGS);
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private WormAuditTables()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Construct full record screens, with no recursive audit writes.
|
||||
*******************************************************************************/
|
||||
public static List<QTableMetaData> tables(WormAuditQBitConfig config)
|
||||
{
|
||||
List<QTableMetaData> tables = new ArrayList<>();
|
||||
tables.add(table(config, CHAIN, "Audit Chain Link", "id:L kind:S audit_id:L detail_ids:T prev_hash:S link_hash:S chained_at:D"));
|
||||
// Keyed by the covered audit_detail id: one row per detail, written with its link.
|
||||
tables.add(table(config, CHAIN_DETAIL, "Detail Coverage", "detail_id:L chain_id:L").withPrimaryKeyField("detail_id")
|
||||
.withRecordLabelFormatAndFields("Detail #%s (link %s)", "detail_id", "chain_id"));
|
||||
tables.add(table(config, CHECKPOINT, "Checkpoint", "id:L from_link_id:L to_link_id:L link_count:L head_hash:S merkle_root:S algorithm:S key_id:S public_key:T signature:T created_at:D"));
|
||||
tables.add(table(config, EXPORT, "Sealed Export", "id:L from_link_id:L to_link_id:L from_checkpoint_id:L to_checkpoint_id:L bucket_alias:S object_prefix:T object_versions:T manifest_key:T manifest_sha256:S prev_manifest_sha256:S object_count:L byte_count:L lock_mode:S retain_until:D sealed_at:D"));
|
||||
tables.add(table(config, VERIFICATION, "Verification", "id:L kind:S trigger:S started_at:D finished_at:D from_link_id:L to_link_id:L links_checked:L outcome:S break_count:L first_break_link_id:L first_break_audit_id:L first_break_reason:T export_id:L run_by:S"));
|
||||
tables.add(table(config, PURGE, "Purge", "id:L requested_at:D requested_by:S cutoff:D retention_interval:S outcome:S reason:T audits_deleted:L details_deleted:L links_deleted:L anchor_link_id:L anchor_hash:S"));
|
||||
tables.add(settings(config));
|
||||
return (tables);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Editable, FIELD-audited singleton with standard QQQ record sections.
|
||||
*******************************************************************************/
|
||||
public static QTableMetaData settings(WormAuditQBitConfig config)
|
||||
{
|
||||
QTableMetaData table = new QTableMetaData().withName(SETTINGS).withLabel("WORM Audit Settings")
|
||||
.withBackendName(config.getBackendName()).withBackendDetails(new RDBMSTableBackendDetails().withTableName(SETTINGS))
|
||||
.withPrimaryKeyField("id").withRecordLabelFields("id").withRecordLabelFormat("WORM Audit Settings")
|
||||
.withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD))
|
||||
.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_INSERT_EDIT_DELETE_PERMISSIONS)
|
||||
.withPermissionBaseName(SETTINGS).withCustomPermissionChecker(new QCodeReference(SettingsPermissions.class)))
|
||||
.withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_DELETE)
|
||||
.withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(SettingsCustomizer.class))
|
||||
.withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, new QCodeReference(SettingsCustomizer.class))
|
||||
.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, new QCodeReference(SettingsCustomizer.class))
|
||||
.withCustomizer(TableCustomizers.PRE_DELETE_RECORD, new QCodeReference(SettingsCustomizer.class))
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withLabel("ID").withBackendName("id").withIsEditable(false).withDefaultValue(1));
|
||||
var defaults = WormAuditSettings.defaults(config);
|
||||
table.withField(new QFieldMetaData("audit_retention_years", QFieldType.INTEGER).withBackendName("audit_retention_years").withLabel("Audit retention years")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("audit_retention_years")));
|
||||
table.withField(new QFieldMetaData("export_lock_mode", QFieldType.STRING).withBackendName("export_lock_mode").withLabel("Export lock mode")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("export_lock_mode")));
|
||||
table.withField(new QFieldMetaData("export_retention_days", QFieldType.INTEGER).withBackendName("export_retention_days").withLabel("Export retention days")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("export_retention_days")));
|
||||
table.withField(new QFieldMetaData("checkpoint_every_links", QFieldType.INTEGER).withBackendName("checkpoint_every_links").withLabel("Checkpoint every links")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("checkpoint_every_links")));
|
||||
table.withField(new QFieldMetaData("checkpoint_every_minutes", QFieldType.INTEGER).withBackendName("checkpoint_every_minutes").withLabel("Checkpoint every minutes")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("checkpoint_every_minutes")));
|
||||
table.withField(new QFieldMetaData("chain_interval_seconds", QFieldType.INTEGER).withBackendName("chain_interval_seconds").withLabel("Chain interval seconds")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("chain_interval_seconds")));
|
||||
table.withField(new QFieldMetaData("settle_horizon_seconds", QFieldType.INTEGER).withBackendName("settle_horizon_seconds").withLabel("Settle horizon seconds")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("settle_horizon_seconds")));
|
||||
table.withField(new QFieldMetaData("late_detail_lookback_hours", QFieldType.INTEGER).withBackendName("late_detail_lookback_hours").withLabel("Late detail lookback hours")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("late_detail_lookback_hours")));
|
||||
table.withField(new QFieldMetaData("export_cron", QFieldType.STRING).withBackendName("export_cron").withLabel("Export schedule (UTC)")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("export_cron")));
|
||||
table.withField(new QFieldMetaData("verify_cron", QFieldType.STRING).withBackendName("verify_cron").withLabel("Verify schedule (UTC)")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("verify_cron")));
|
||||
table.getField("export_lock_mode").withInlinePossibleValueSource(new QPossibleValueSource().withName(SETTINGS + "_lock_mode")
|
||||
.withType(QPossibleValueSourceType.ENUM).withIdType(QFieldType.STRING).withEnumValues(List.of(
|
||||
new QPossibleValue<>("GOVERNANCE", "GOVERNANCE"), new QPossibleValue<>("COMPLIANCE", "COMPLIANCE"))));
|
||||
for(String field : List.of("export_cron", "verify_cron", "chain_interval_seconds"))
|
||||
{
|
||||
table.getField(field).withHelpContent(new QHelpContent("Takes effect after restart. Cron schedules use UTC and Quartz syntax."));
|
||||
}
|
||||
table.getField("audit_retention_years").withHelpContent(new QHelpContent("Used for the next eligible purge date. The database purge interval changes only through a reviewed migration."));
|
||||
table.withField(new QFieldMetaData("notes", QFieldType.TEXT).withLabel("Notes").withBackendName("notes"));
|
||||
table.withField(new QFieldMetaData("modify_date", QFieldType.DATE_TIME).withLabel("Modified").withBackendName("modify_date").withIsEditable(false));
|
||||
table.withField(new QFieldMetaData("modified_by", QFieldType.STRING).withLabel("Modified by").withBackendName("modified_by").withIsEditable(false));
|
||||
table.withSection(new QFieldSection("retention", "Retention", new QIcon("lock"), Tier.T1,
|
||||
List.of("id", "audit_retention_years", "export_lock_mode", "export_retention_days")).withGridColumns(12));
|
||||
table.withSection(new QFieldSection("cadence", "Chain and checkpoints", new QIcon("link"), Tier.T2,
|
||||
List.of("checkpoint_every_links", "checkpoint_every_minutes", "settle_horizon_seconds", "late_detail_lookback_hours")).withGridColumns(12));
|
||||
table.withSection(new QFieldSection("schedules", "Schedules (after restart)", new QIcon("schedule"), Tier.T2,
|
||||
List.of("chain_interval_seconds", "export_cron", "verify_cron")).withGridColumns(12));
|
||||
table.withSection(new QFieldSection("changes", "Changes", new QIcon("edit_note"), Tier.T2,
|
||||
List.of("notes", "modify_date", "modified_by")).withGridColumns(12));
|
||||
return (table);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Common metadata for a process-written table.
|
||||
*******************************************************************************/
|
||||
private static QTableMetaData table(WormAuditQBitConfig config, String name, String label, String fields)
|
||||
{
|
||||
QTableMetaData table = new QTableMetaData().withName(name).withLabel(navigationLabel(name))
|
||||
.withBackendName(config.getBackendName()).withBackendDetails(new RDBMSTableBackendDetails().withTableName(name))
|
||||
.withPrimaryKeyField("id").withRecordLabelFields("id").withRecordLabelFormat(label + " #%s")
|
||||
.withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.NONE))
|
||||
.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_INSERT_EDIT_DELETE_PERMISSIONS).withPermissionBaseName(name));
|
||||
for(String spec : fields.split(" "))
|
||||
{
|
||||
String[] parts = spec.split(":");
|
||||
QFieldType type = switch(parts[1])
|
||||
{
|
||||
case "L" -> QFieldType.LONG;
|
||||
case "D" -> QFieldType.DATE_TIME;
|
||||
case "T" -> QFieldType.TEXT;
|
||||
default -> QFieldType.STRING;
|
||||
};
|
||||
table.withField(new QFieldMetaData(parts[0], type).withBackendName(parts[0]).withLabel(fieldLabel(parts[0])));
|
||||
}
|
||||
for(String reference : List.of("from_link_id", "to_link_id", "first_break_link_id", "anchor_link_id", "chain_id"))
|
||||
{
|
||||
if(table.getFields().containsKey(reference))
|
||||
{
|
||||
table.getField(reference).withPossibleValueSourceName(CHAIN);
|
||||
}
|
||||
}
|
||||
for(String reference : List.of("from_checkpoint_id", "to_checkpoint_id"))
|
||||
{
|
||||
if(table.getFields().containsKey(reference))
|
||||
{
|
||||
table.getField(reference).withPossibleValueSourceName(CHECKPOINT);
|
||||
}
|
||||
}
|
||||
if(table.getFields().containsKey("export_id"))
|
||||
{
|
||||
table.getField("export_id").withPossibleValueSourceName(EXPORT);
|
||||
}
|
||||
switch(name)
|
||||
{
|
||||
case CHAIN -> table.withRecordLabelFormat("Link #%s");
|
||||
case EXPORT -> table.withRecordLabelFormatAndFields("Export #%s (links %s–%s)", "id", "from_link_id", "to_link_id");
|
||||
case VERIFICATION, PURGE -> table.withRecordLabelFormatAndFields(label + " #%s (%s)", "id", "outcome");
|
||||
default ->
|
||||
{
|
||||
}
|
||||
}
|
||||
if(table.getFields().containsKey("kind"))
|
||||
{
|
||||
table.getField("kind").withInlinePossibleValueSource(new QPossibleValueSource().withName(name + "_kind")
|
||||
.withType(QPossibleValueSourceType.ENUM).withIdType(QFieldType.STRING).withEnumValues(List.of(new QPossibleValue<>("AUDIT", "Audit"),
|
||||
new QPossibleValue<>("LATE_DETAIL", "Late detail"), new QPossibleValue<>("CHAIN", "Chain"),
|
||||
new QPossibleValue<>("SEALED_EXPORT", "Sealed export"))));
|
||||
}
|
||||
if(table.getFields().containsKey("outcome"))
|
||||
{
|
||||
table.getField("outcome").withInlinePossibleValueSource(new QPossibleValueSource().withName(name + "_outcome")
|
||||
.withType(QPossibleValueSourceType.ENUM).withIdType(QFieldType.STRING).withEnumValues(List.of(new QPossibleValue<>("PASS", "Pass"),
|
||||
new QPossibleValue<>("FAIL", "Fail"), new QPossibleValue<>("PURGED", "Purged"), new QPossibleValue<>("REFUSED", "Refused"))));
|
||||
}
|
||||
if(VERIFICATION.equals(name))
|
||||
{
|
||||
table.withSection(new QFieldSection("result", "Result", new QIcon("fact_check"), Tier.T2,
|
||||
List.of("id", "outcome", "break_count", "first_break_link_id", "first_break_audit_id", "first_break_reason")).withGridColumns(12));
|
||||
table.withSection(new QFieldSection("scope", "Scope", new QIcon("link"), Tier.T2,
|
||||
List.of("kind", "trigger", "from_link_id", "to_link_id", "links_checked", "export_id")).withGridColumns(12));
|
||||
table.withSection(new QFieldSection("run", "Run", new QIcon("schedule"), Tier.T2,
|
||||
List.of("started_at", "finished_at", "run_by")).withGridColumns(12));
|
||||
}
|
||||
else
|
||||
{
|
||||
table.withSection(new QFieldSection("record", label, new QIcon("fact_check"), Tier.T1,
|
||||
new ArrayList<>(table.getFields().keySet())).withGridColumns(12));
|
||||
}
|
||||
if(List.of(CHECKPOINT, EXPORT, VERIFICATION).contains(name))
|
||||
{
|
||||
table.withSection(new QFieldSection("links", new QIcon("link"), Tier.T2).withLabel("Chain links")
|
||||
.withWidgetName(name + "_links").withGridColumns(12));
|
||||
}
|
||||
return (table);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Core RC.5 shares the table label with its navigation tree node.
|
||||
*******************************************************************************/
|
||||
public static String navigationLabel(String name)
|
||||
{
|
||||
return (switch(name)
|
||||
{
|
||||
case SETTINGS -> "Settings";
|
||||
case CHAIN -> "Audit Chain";
|
||||
case CHAIN_DETAIL -> "Detail Coverage";
|
||||
case CHECKPOINT -> "Checkpoints";
|
||||
case EXPORT -> "Sealed Exports";
|
||||
case VERIFICATION -> "Verifications";
|
||||
default -> "Purges";
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Plain record labels independent of physical column names.
|
||||
*******************************************************************************/
|
||||
private static String fieldLabel(String name)
|
||||
{
|
||||
return (switch(name)
|
||||
{
|
||||
case "id" -> "ID";
|
||||
case "kind" -> "Kind";
|
||||
case "audit_id" -> "Audit";
|
||||
case "detail_ids" -> "Detail ids";
|
||||
case "detail_id" -> "Audit detail";
|
||||
case "chain_id" -> "Link";
|
||||
case "prev_hash" -> "Previous hash";
|
||||
case "link_hash" -> "Link hash";
|
||||
case "chained_at" -> "Chained";
|
||||
case "from_link_id" -> "From link";
|
||||
case "to_link_id" -> "To link";
|
||||
case "link_count" -> "Links";
|
||||
case "head_hash" -> "Head hash";
|
||||
case "merkle_root" -> "Merkle root";
|
||||
case "algorithm" -> "Algorithm";
|
||||
case "key_id" -> "Key ID";
|
||||
case "public_key" -> "Public key";
|
||||
case "signature" -> "Signature";
|
||||
case "created_at" -> "Created";
|
||||
case "from_checkpoint_id" -> "From checkpoint";
|
||||
case "to_checkpoint_id" -> "To checkpoint";
|
||||
case "bucket_alias" -> "Bucket alias";
|
||||
case "object_prefix" -> "Object prefix";
|
||||
case "object_versions" -> "Object versions";
|
||||
case "manifest_key" -> "Manifest key";
|
||||
case "manifest_sha256" -> "Manifest SHA-256";
|
||||
case "prev_manifest_sha256" -> "Previous manifest SHA-256";
|
||||
case "object_count" -> "Objects";
|
||||
case "byte_count" -> "Bytes";
|
||||
case "lock_mode" -> "Lock mode";
|
||||
case "retain_until" -> "Retain until";
|
||||
case "sealed_at" -> "Sealed";
|
||||
case "trigger" -> "Trigger";
|
||||
case "started_at" -> "Started";
|
||||
case "finished_at" -> "Finished";
|
||||
case "links_checked" -> "Links checked";
|
||||
case "outcome" -> "Outcome";
|
||||
case "break_count" -> "Breaks";
|
||||
case "first_break_link_id" -> "First break link";
|
||||
case "first_break_audit_id" -> "First break audit";
|
||||
case "first_break_reason" -> "First break";
|
||||
case "export_id" -> "Sealed export";
|
||||
case "run_by" -> "Run by";
|
||||
case "requested_at" -> "Requested";
|
||||
case "requested_by" -> "Requested by";
|
||||
case "cutoff" -> "Cutoff";
|
||||
case "retention_interval" -> "Retention interval";
|
||||
case "reason" -> "Reason";
|
||||
case "audits_deleted" -> "Audits deleted";
|
||||
case "details_deleted" -> "Details deleted";
|
||||
case "links_deleted" -> "Links deleted";
|
||||
case "anchor_link_id" -> "Anchor link";
|
||||
case "anchor_hash" -> "Anchor hash";
|
||||
default -> throw new IllegalArgumentException("Missing field label: " + name);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,377 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.processes;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedExportVerifier;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedExporter;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedStore;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.UseOtherPermissionNameCustomPermissionChecker;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunInput;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunOutput;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Standard Review/Result actions for sealing and verifying exports.
|
||||
*******************************************************************************/
|
||||
public final class ExportProcesses
|
||||
{
|
||||
public static final String EXPORT = "worm_audit_export_now";
|
||||
public static final String VERIFY = "worm_audit_verify_export";
|
||||
public static final String DAILY = "worm_audit_export_daily";
|
||||
public static final String PERMISSION = "worm_audit.export";
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private ExportProcesses()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Export is a table action; verification requires one or more selected exports.
|
||||
*******************************************************************************/
|
||||
public static List<QProcessMetaData> processes(WormAuditQBitConfig config)
|
||||
{
|
||||
var settings = config.effectiveSettings();
|
||||
Map<String, Serializable> values = new HashMap<>();
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, WormAuditTables.EXPORT);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, WormAuditTables.EXPORT);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION, true);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true);
|
||||
QProcessMetaData export = StreamedETLWithFrontendProcess.defineProcessMetaData(ExportExtract.class, ExportReview.class, ExportLoad.class, new HashMap<>(values))
|
||||
.withName(EXPORT).withLabel("Export now").withTableName(WormAuditTables.EXPORT).withPermissionRules(permission(PERMISSION));
|
||||
QProcessMetaData verify = StreamedETLWithFrontendProcess.defineProcessMetaData(VerifyExtract.class, VerifyReview.class, VerifyLoad.class, new HashMap<>(values))
|
||||
.withName(VERIFY).withLabel("Verify sealed export").withTableName(WormAuditTables.EXPORT).withMinInputRecords(1)
|
||||
.withPermissionRules(permission(WormAuditProcesses.PERMISSION));
|
||||
QProcessMetaData daily = new QProcessMetaData().withName(DAILY).withLabel("Daily sealed audit export")
|
||||
.withStep(new QBackendStepMetaData().withName("export").withCode(new QCodeReference(Daily.class)))
|
||||
.withPermissionRules(permission(PERMISSION)).withSchedule(new QScheduleMetaData().withSchedulerName(config.getSchedulerName())
|
||||
.withCronExpression(settings.export_cron()).withCronTimeZoneId("UTC"));
|
||||
return (List.of(export, verify, daily));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reuse the phase (a) explicit permission-name checker.
|
||||
*******************************************************************************/
|
||||
private static QPermissionRules permission(String name)
|
||||
{
|
||||
return (new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION)
|
||||
.withCustomPermissionChecker(UseOtherPermissionNameCustomPermissionChecker.build(name)));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Plain text summaries must not acquire the framework's default zero count.
|
||||
*******************************************************************************/
|
||||
private static ProcessSummaryLine text(Status status, String message)
|
||||
{
|
||||
ProcessSummaryLine line = new ProcessSummaryLine(status, message);
|
||||
line.setCount(null);
|
||||
return (line);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One synthetic work item allows the first export on an empty export table.
|
||||
*******************************************************************************/
|
||||
public static class ExportExtract extends AbstractExtractStep
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Count the single export operation, not previous export rows.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public Integer doCount(RunBackendStepInput input)
|
||||
{
|
||||
return (1);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Feed one work item through the standard stream.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
getRecordPipe().addRecord(new QRecord().withValue("id", 0L));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Review is read-only; the writer reselects the suffix after confirmation.
|
||||
*******************************************************************************/
|
||||
public static class ExportReview extends AbstractTransformStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private String message;
|
||||
private boolean failed;
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the planned range once without uploading or inserting.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
try
|
||||
{
|
||||
message = SealedExporter.scope(new AuditStore(null)).review();
|
||||
failed = false;
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
message = "Cannot select a contiguous export range";
|
||||
failed = true;
|
||||
}
|
||||
output.getRecords().addAll(input.getRecords());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Show the range, including the explicit no-checkpoint case.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
return (new ArrayList<>(List.of(text(failed ? Status.ERROR : Status.INFO, message == null ? "Nothing to seal: no new checkpoint" : message))));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Seal once after the stream is complete.
|
||||
*******************************************************************************/
|
||||
public static class ExportLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final SealedStore storage;
|
||||
private SealedExporter.Result result;
|
||||
|
||||
/*******************************************************************************
|
||||
** Framework constructor.
|
||||
*******************************************************************************/
|
||||
public ExportLoad()
|
||||
{
|
||||
this(null);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Storage-injected action for unit tests.
|
||||
*******************************************************************************/
|
||||
public ExportLoad(SealedStore storage)
|
||||
{
|
||||
this.storage = storage;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Forward the work item without retaining content.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output)
|
||||
{
|
||||
output.getRecords().addAll(input.getRecords());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The same execution path as the daily schedule.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void postRun(BackendStepPostRunInput input, BackendStepPostRunOutput output)
|
||||
{
|
||||
result = new SealedExporter(VerificationRuns.config(), storage).run("USER", Instant.now());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Result text is value-free even when the provider fails.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
return (result == null ? new ArrayList<>() : new ArrayList<>(List.of(text(result.failed() ? Status.ERROR : Status.OK, result.message()))));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Respect core's selected-record/filter interpretation; no implicit all-export run.
|
||||
*******************************************************************************/
|
||||
public static class VerifyExtract extends ExtractViaQueryStep
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Deterministic selection with full text fields from the export table.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QQueryFilter getQueryFilter(RunBackendStepInput input) throws QException
|
||||
{
|
||||
input.addValue(FIELD_SOURCE_TABLE, WormAuditTables.EXPORT);
|
||||
QQueryFilter filter = super.getQueryFilter(input).clone();
|
||||
filter.setOrderBys(List.of(new QFilterOrderBy("id", true)));
|
||||
return (filter);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Review selected exports without reading their content.
|
||||
*******************************************************************************/
|
||||
public static class VerifyReview extends AbstractTransformStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private long count;
|
||||
|
||||
/*******************************************************************************
|
||||
** Count and forward the selected export records.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output)
|
||||
{
|
||||
count += input.getRecords().size();
|
||||
output.getRecords().addAll(input.getRecords());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One Review summary.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
return (new ArrayList<>(List.of(text(Status.INFO, WormAuditVerifier.number(count) + " sealed exports will be verified"))));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify each selected export and persist its independent result.
|
||||
*******************************************************************************/
|
||||
public static class VerifyLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final SealedStore storage;
|
||||
private final ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
|
||||
/*******************************************************************************
|
||||
** Framework constructor.
|
||||
*******************************************************************************/
|
||||
public VerifyLoad()
|
||||
{
|
||||
this(null);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Storage-injected action for unit tests.
|
||||
*******************************************************************************/
|
||||
public VerifyLoad(SealedStore storage)
|
||||
{
|
||||
this.storage = storage;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Process the current page without retaining exported content.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
for(QRecord selected : input.getRecords())
|
||||
{
|
||||
QRecord row = new AuditStore(null).get(WormAuditTables.EXPORT, selected.getValueLong("id"));
|
||||
if(row == null)
|
||||
{
|
||||
lines.add(text(Status.ERROR, "Selected export " + selected.getValueLong("id") + " is missing"));
|
||||
continue;
|
||||
}
|
||||
var report = new SealedExportVerifier(VerificationRuns.config(), storage).verify(row, "USER");
|
||||
if(report.result().passed())
|
||||
{
|
||||
lines.add(text(Status.OK, WormAuditVerifier.number(report.result().getLinksChecked()) + " links verified in export " + row.getValueLong("id")));
|
||||
}
|
||||
report.result().getBreaks().forEach(b -> lines.add(text(Status.ERROR, b.reason())));
|
||||
if(report.purged() > 0 && report.result().passed())
|
||||
{
|
||||
lines.add(text(Status.INFO, WormAuditVerifier.number(report.purged()) + " links verified from the sealed copy only (purged from the live database)"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exact per-link differences and purged-copy information.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Daily export sends heartbeat even when sealing fails.
|
||||
*******************************************************************************/
|
||||
public static class Daily implements BackendStep
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Run the exporter with the installed host configuration.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output)
|
||||
{
|
||||
var result = new SealedExporter(VerificationRuns.config()).run("SCHEDULE", Instant.now());
|
||||
output.addValue("outcome", result.failed() ? "FAIL" : "PASS");
|
||||
output.addValue("result", result.message());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.processes;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.VerificationResult;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertRecordRef;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertSignal;
|
||||
import com.kingsrook.qqq.backend.core.alerts.QAlerts;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verification run persistence and the core alerts SPI. Both UI and schedule
|
||||
** write exactly one immutable result row; only schedules change alert state.
|
||||
*******************************************************************************/
|
||||
public final class VerificationRuns
|
||||
{
|
||||
/** How many break reasons the alert summary spells out; the title carries the total. */
|
||||
private static final int ALERT_SUMMARY_REASONS = 3;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private VerificationRuns()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve this instance's configuration, never a global cross-instance cache.
|
||||
*******************************************************************************/
|
||||
public static WormAuditQBitConfig config()
|
||||
{
|
||||
return (QContext.getQInstance().getQBits().values().stream().map(q -> q.getConfig()).filter(WormAuditQBitConfig.class::isInstance)
|
||||
.map(WormAuditQBitConfig.class::cast).findFirst().orElseThrow(() -> new IllegalStateException("WORM audit qbit is not installed")));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify and persist, sending a heartbeat even if a scheduled run errors.
|
||||
*******************************************************************************/
|
||||
public static VerificationResult run(Set<Long> selection, String trigger) throws QException
|
||||
{
|
||||
boolean scheduled = "SCHEDULE".equals(trigger);
|
||||
try
|
||||
{
|
||||
VerificationResult result = new WormAuditVerifier(config()).verify(selection);
|
||||
QRecord row = save(result, trigger);
|
||||
if(scheduled)
|
||||
{
|
||||
if(result.passed())
|
||||
{
|
||||
QAlerts.resolve("wormaudit.chain.break", "chain", "Audit chain verification passed");
|
||||
}
|
||||
else
|
||||
{
|
||||
long problems = result.getBreakCount();
|
||||
String summary = result.getBreaks().stream().limit(ALERT_SUMMARY_REASONS).map(b -> b.reason()).collect(Collectors.joining("; "));
|
||||
if(problems > ALERT_SUMMARY_REASONS)
|
||||
{
|
||||
summary += "; and " + (problems - ALERT_SUMMARY_REASONS) + " more";
|
||||
}
|
||||
QAlerts.raise(new AlertSignal("wormaudit.chain.break").withDedupKey("chain")
|
||||
.withTitle("Audit chain verification found " + problems + (problems == 1 ? " problem" : " problems")).withSummary(summary)
|
||||
.withRecordRef(new AlertRecordRef(WormAuditTables.VERIFICATION, row.getValue("id"))));
|
||||
}
|
||||
}
|
||||
return (result);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if(scheduled)
|
||||
{
|
||||
QAlerts.heartbeat("wormaudit.verify.daily");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Persist identifiers and a value-free first reason, never audit content.
|
||||
*******************************************************************************/
|
||||
public static QRecord save(VerificationResult result, String trigger) throws QException
|
||||
{
|
||||
return (save(result, trigger, null));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Append a chain or sealed-export result using the same immutable schema.
|
||||
*******************************************************************************/
|
||||
public static QRecord save(VerificationResult result, String trigger, Long exportId) throws QException
|
||||
{
|
||||
return (save(result, trigger, exportId, Instant.now()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Clock-injected persistence for genuine historical verification runs.
|
||||
*******************************************************************************/
|
||||
public static QRecord save(VerificationResult result, String trigger, Long exportId, Instant finishedAt) throws QException
|
||||
{
|
||||
String user = QContext.getQSession() == null || QContext.getQSession().getUser() == null ? "system" : QContext.getQSession().getUser().getIdReference();
|
||||
QRecord row = new QRecord().withValue("kind", exportId == null ? "CHAIN" : "SEALED_EXPORT").withValue("export_id", exportId).withValue("trigger", trigger)
|
||||
.withValue("started_at", result.getStartedAt()).withValue("finished_at", finishedAt)
|
||||
.withValue("from_link_id", result.getFromLinkId()).withValue("to_link_id", result.getToLinkId())
|
||||
.withValue("links_checked", result.getLinksChecked()).withValue("outcome", result.passed() ? "PASS" : "FAIL")
|
||||
.withValue("break_count", result.getBreakCount()).withValue("run_by", user);
|
||||
if(!result.passed())
|
||||
{
|
||||
var first = result.getBreaks().getFirst();
|
||||
row.withValue("first_break_link_id", first.linkId()).withValue("first_break_audit_id", first.auditId()).withValue("first_break_reason", first.reason());
|
||||
}
|
||||
return (new AuditStore(null).insert(WormAuditTables.VERIFICATION, List.of(row)).getFirst());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,313 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.processes;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.VerificationResult;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.UseOtherPermissionNameCustomPermissionChecker;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunInput;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunOutput;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.CouldNotFindQueryFilterForExtractStepException;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Standard streamed ETL Review/Result action plus two scheduled-only processes.
|
||||
** No process is an app navigation entry.
|
||||
*******************************************************************************/
|
||||
public final class WormAuditProcesses
|
||||
{
|
||||
public static final String VERIFY = "worm_audit_verify_chain";
|
||||
public static final String CHAIN = "worm_audit_chain_tick";
|
||||
public static final String DAILY = "worm_audit_verify_daily";
|
||||
public static final String PERMISSION = "worm_audit.verify";
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private WormAuditProcesses()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Metadata for table/record verification and scheduled background work.
|
||||
*******************************************************************************/
|
||||
public static List<QProcessMetaData> processes(WormAuditQBitConfig config)
|
||||
{
|
||||
var settings = config.effectiveSettings();
|
||||
Map<String, Serializable> values = new HashMap<>();
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, WormAuditTables.CHAIN);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, WormAuditTables.CHAIN);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION, true);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, "Verify live audit content, link continuity and signed checkpoints.");
|
||||
QProcessMetaData verify = StreamedETLWithFrontendProcess.defineProcessMetaData(Extract.class, Review.class, Load.class, values)
|
||||
.withName(VERIFY).withLabel("Verify audit chain").withTableName(WormAuditTables.CHAIN).withPermissionRules(permission());
|
||||
QProcessMetaData chain = new QProcessMetaData().withName(CHAIN).withLabel("Chain settled audit records")
|
||||
.withStep(new QBackendStepMetaData().withName("chain").withCode(new QCodeReference(ChainTick.class)))
|
||||
.withPermissionRules(permission()).withSchedule(new QScheduleMetaData().withSchedulerName(config.getSchedulerName())
|
||||
.withRepeatSeconds(settings.chain_interval_seconds()).withInitialDelaySeconds(settings.chain_interval_seconds()));
|
||||
QProcessMetaData daily = new QProcessMetaData().withName(DAILY).withLabel("Daily audit chain verification")
|
||||
.withStep(new QBackendStepMetaData().withName("verify").withCode(new QCodeReference(Daily.class)))
|
||||
.withPermissionRules(permission()).withSchedule(new QScheduleMetaData().withSchedulerName(config.getSchedulerName())
|
||||
.withCronExpression(settings.verify_cron()).withCronTimeZoneId("UTC"));
|
||||
return (List.of(verify, chain, daily));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One explicit permission, not process-name access suffixes.
|
||||
*******************************************************************************/
|
||||
private static QPermissionRules permission()
|
||||
{
|
||||
return (new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION)
|
||||
.withCustomPermissionChecker(UseOtherPermissionNameCustomPermissionChecker.build(PERMISSION)));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Use core selection/filter interpretation, with a full-chain default. An
|
||||
** empty frontend filter means a full run, so completeness is still checked.
|
||||
*******************************************************************************/
|
||||
public static class Extract extends ExtractViaQueryStep
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Extract links in id order and make the effective scope available to Load.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QQueryFilter getQueryFilter(RunBackendStepInput input) throws QException
|
||||
{
|
||||
input.addValue(FIELD_SOURCE_TABLE, WormAuditTables.CHAIN);
|
||||
QQueryFilter filter;
|
||||
try
|
||||
{
|
||||
filter = super.getQueryFilter(input).clone();
|
||||
}
|
||||
catch(CouldNotFindQueryFilterForExtractStepException expected)
|
||||
{
|
||||
filter = new QQueryFilter();
|
||||
}
|
||||
filter.setOrderBys(List.of(new QFilterOrderBy("id", true)));
|
||||
return (filter);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Review counts the streamed selection without verifying or writing rows.
|
||||
*******************************************************************************/
|
||||
public static class Review extends AbstractTransformStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private long count;
|
||||
private long min = Long.MAX_VALUE;
|
||||
private long max = Long.MIN_VALUE;
|
||||
private Long checkpoints;
|
||||
|
||||
/*******************************************************************************
|
||||
** Accumulate the scope and forward links to the loader.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
for(QRecord record : input.getRecords())
|
||||
{
|
||||
count++;
|
||||
min = Math.min(min, record.getValueLong("id"));
|
||||
max = Math.max(max, record.getValueLong("id"));
|
||||
}
|
||||
checkpoints = null;
|
||||
output.getRecords().addAll(input.getRecords());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Standard Review line, including the link range and checkpoint count.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
if(checkpoints == null)
|
||||
{
|
||||
try
|
||||
{
|
||||
CountInput input = new CountInput(WormAuditTables.CHECKPOINT).withFilter(new QQueryFilter(
|
||||
new QFilterCriteria("from_link_id", QCriteriaOperator.LESS_THAN_OR_EQUALS, max),
|
||||
new QFilterCriteria("to_link_id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, min)));
|
||||
input.withPermissionBypass("WORM audit internal review");
|
||||
checkpoints = count == 0 ? 0L : new CountAction().execute(input).getCount().longValue();
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
return (new ArrayList<>(List.of(new ProcessSummaryLine(Status.ERROR, "Cannot count overlapping audit checkpoints"))));
|
||||
}
|
||||
}
|
||||
String range = count == 0 ? "none" : WormAuditVerifier.number(min) + "–" + WormAuditVerifier.number(max);
|
||||
return (new ArrayList<>(List.of(new ProcessSummaryLine(Status.INFO, WormAuditVerifier.number(count)
|
||||
+ " chain links will be verified (links " + range + ", " + WormAuditVerifier.number(checkpoints) + " checkpoints)"))));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Load collects selected ids, then performs one snapshot verification in
|
||||
** postRun and appends one result row, including for an empty chain.
|
||||
*******************************************************************************/
|
||||
public static class Load extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private Set<Long> ids;
|
||||
private VerificationResult result;
|
||||
|
||||
/*******************************************************************************
|
||||
** Retain ids only, not audit content, across streamed pages.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
if(selected(new Extract().getQueryFilter(input)))
|
||||
{
|
||||
if(ids == null)
|
||||
{
|
||||
ids = new HashSet<>();
|
||||
}
|
||||
input.getRecords().forEach(r -> ids.add(r.getValueLong("id")));
|
||||
}
|
||||
output.getRecords().addAll(input.getRecords());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Run once after all pages. Explicit filters are selection runs.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void postRun(BackendStepPostRunInput input, BackendStepPostRunOutput output) throws QException
|
||||
{
|
||||
QQueryFilter filter = new Extract().getQueryFilter(input);
|
||||
if(selected(filter) && ids == null)
|
||||
{
|
||||
// Some callers supply the effective selection only to postRun.
|
||||
ids = new HashSet<>();
|
||||
new AuditStore(null).query(WormAuditTables.CHAIN, filter).forEach(r -> ids.add(r.getValueLong("id")));
|
||||
}
|
||||
result = VerificationRuns.run(selected(filter) ? ids : null, "USER");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Empty filters preserve full completeness verification and retain no ids.
|
||||
*******************************************************************************/
|
||||
private boolean selected(QQueryFilter filter)
|
||||
{
|
||||
return ((filter.getCriteria() != null && !filter.getCriteria().isEmpty())
|
||||
|| (filter.getSubFilters() != null && !filter.getSubFilters().isEmpty())
|
||||
|| filter.getLimit() != null || (filter.getSkip() != null && filter.getSkip() > 0));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One ERROR per exact break; warnings remain visible on a passing run.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
if(result == null)
|
||||
{
|
||||
return (lines);
|
||||
}
|
||||
if(result.passed())
|
||||
{
|
||||
lines.add(new ProcessSummaryLine(Status.OK, WormAuditVerifier.number(result.getLinksChecked()) + " links verified"));
|
||||
}
|
||||
result.getBreaks().forEach(b -> lines.add(new ProcessSummaryLine(Status.ERROR, b.reason())));
|
||||
result.getWarnings().forEach(w -> lines.add(new ProcessSummaryLine(Status.WARNING, w)));
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Scheduled writer tick.
|
||||
*******************************************************************************/
|
||||
public static class ChainTick implements BackendStep
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Append settled work under the shared writer lock.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
output.addValue("linksChained", new WormAuditChainer(VerificationRuns.config()).chainOnce().links());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Daily verification, including alerts and heartbeat.
|
||||
*******************************************************************************/
|
||||
public static class Daily implements BackendStep
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Verify the full settled chain.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
output.addValue("outcome", VerificationRuns.run(null, "SCHEDULE").passed() ? "PASS" : "FAIL");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.verify;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Ordered, value-free findings. Break identifiers are separately available for
|
||||
** record navigation and alert correlation without parsing human-readable text.
|
||||
*******************************************************************************/
|
||||
public class VerificationResult
|
||||
{
|
||||
/*******************************************************************************
|
||||
** One break; null ids apply to coverage and checkpoint findings.
|
||||
*******************************************************************************/
|
||||
public record Break(Long linkId, Long auditId, String reason)
|
||||
{
|
||||
}
|
||||
|
||||
private final List<Break> breaks = new ArrayList<>();
|
||||
private final List<String> warnings = new ArrayList<>();
|
||||
private final Instant startedAt;
|
||||
private long breakCount;
|
||||
private long linksChecked;
|
||||
private Long fromLinkId;
|
||||
private Long toLinkId;
|
||||
|
||||
/*******************************************************************************
|
||||
** Use the production clock by default.
|
||||
*******************************************************************************/
|
||||
public VerificationResult()
|
||||
{
|
||||
this(Instant.now());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Bind the verification start to the caller's clock.
|
||||
*******************************************************************************/
|
||||
public VerificationResult(Instant startedAt)
|
||||
{
|
||||
this.startedAt = startedAt;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Add a finding in deterministic verification order.
|
||||
*******************************************************************************/
|
||||
public void add(Long linkId, Long auditId, String reason)
|
||||
{
|
||||
breakCount++;
|
||||
if(breaks.size() < 1000)
|
||||
{
|
||||
breaks.add(new Break(linkId, auditId, reason));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Count additional SQL offenders without retaining their content.
|
||||
*******************************************************************************/
|
||||
public void omitted(long count)
|
||||
{
|
||||
breakCount += count;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Total findings, including diagnostics omitted by the bounded display limit.
|
||||
*******************************************************************************/
|
||||
public long getBreakCount()
|
||||
{
|
||||
return (breakCount);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Keep warnings bounded even when many checkpoint ranges have been purged.
|
||||
*******************************************************************************/
|
||||
public void warn(String warning)
|
||||
{
|
||||
if(warnings.size() < 100)
|
||||
{
|
||||
warnings.add(warning);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Record the scope that was checked.
|
||||
*******************************************************************************/
|
||||
public void scope(long count, Long from, Long to)
|
||||
{
|
||||
linksChecked = count;
|
||||
fromLinkId = from;
|
||||
toLinkId = to;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Ordered findings.
|
||||
*******************************************************************************/
|
||||
public List<Break> getBreaks()
|
||||
{
|
||||
return (List.copyOf(breaks));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Warnings do not break the chain.
|
||||
*******************************************************************************/
|
||||
public List<String> getWarnings()
|
||||
{
|
||||
return (warnings);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether all checked invariants hold.
|
||||
*******************************************************************************/
|
||||
public boolean passed()
|
||||
{
|
||||
return (breakCount == 0);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Number of checked links.
|
||||
*******************************************************************************/
|
||||
public long getLinksChecked()
|
||||
{
|
||||
return (linksChecked);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** First checked link, null for an empty scope.
|
||||
*******************************************************************************/
|
||||
public Long getFromLinkId()
|
||||
{
|
||||
return (fromLinkId);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Last checked link, null for an empty scope.
|
||||
*******************************************************************************/
|
||||
public Long getToLinkId()
|
||||
{
|
||||
return (toLinkId);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verification start time.
|
||||
*******************************************************************************/
|
||||
public Instant getStartedAt()
|
||||
{
|
||||
return (startedAt);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,573 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.verify;
|
||||
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.chain.Hashes;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.checkpoint.Checkpoints;
|
||||
import com.kingsrook.qbits.wormaudit.guard.AppendOnlyGuard;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSTransaction;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Snapshot verification of live content, continuity, exact coverage and signed
|
||||
** checkpoints. Selection runs verify predecessor boundaries but deliberately
|
||||
** omit global completeness; full runs begin at the latest retention anchor.
|
||||
*******************************************************************************/
|
||||
public class WormAuditVerifier
|
||||
{
|
||||
public static final int PAGE_SIZE = 1000;
|
||||
private final WormAuditQBitConfig config;
|
||||
private final int pageSize;
|
||||
|
||||
/*******************************************************************************
|
||||
** Bind trust pins and settle horizon.
|
||||
*******************************************************************************/
|
||||
public WormAuditVerifier(WormAuditQBitConfig config)
|
||||
{
|
||||
this(config, PAGE_SIZE);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Smaller pages support constrained hosts and boundary regression tests.
|
||||
*******************************************************************************/
|
||||
public WormAuditVerifier(WormAuditQBitConfig config, int pageSize)
|
||||
{
|
||||
if(pageSize < 1 || pageSize > PAGE_SIZE)
|
||||
{
|
||||
throw (new IllegalArgumentException("Verification page size must be between 1 and 1000"));
|
||||
}
|
||||
this.config = config;
|
||||
this.pageSize = pageSize;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify all live links and settled coverage.
|
||||
*******************************************************************************/
|
||||
public VerificationResult verify() throws QException
|
||||
{
|
||||
return (verify(null));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Null means full verification; an empty set is an empty selection.
|
||||
*******************************************************************************/
|
||||
public VerificationResult verify(Set<Long> selection) throws QException
|
||||
{
|
||||
return (verify(selection, Instant.now()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Clock-injected verification uses the same horizon and snapshot as production.
|
||||
*******************************************************************************/
|
||||
public VerificationResult verify(Set<Long> selection, Instant now) throws QException
|
||||
{
|
||||
WormAuditChainer.JVM_LOCK.lock();
|
||||
try(QBackendTransaction transaction = QBackendTransaction.openFor(new QueryInput(WormAuditTables.CHAIN)))
|
||||
{
|
||||
try
|
||||
{
|
||||
if(transaction instanceof RDBMSTransaction rdbms)
|
||||
{
|
||||
rdbms.getConnection().setTransactionIsolation(Connection.TRANSACTION_REPEATABLE_READ);
|
||||
}
|
||||
VerificationResult result = verify(new AuditStore(transaction), selection, now);
|
||||
transaction.commit();
|
||||
return (result);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
transaction.rollback();
|
||||
throw (AppendOnlyGuard.map(e));
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
WormAuditChainer.JVM_LOCK.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify one consistent storage snapshot.
|
||||
*******************************************************************************/
|
||||
VerificationResult verify(AuditStore store, Set<Long> selection, Instant now) throws QException
|
||||
{
|
||||
var settings = config.effectiveSettings();
|
||||
VerificationResult result = new VerificationResult(now);
|
||||
if(config.getSigningKey() == null)
|
||||
{
|
||||
result.warn("no signing key configured");
|
||||
}
|
||||
QRecord anchor = store.anchor();
|
||||
long anchorId = anchor.getValueLong("anchor_link_id");
|
||||
long lastId = anchorId;
|
||||
String previous = anchor.getValueString("anchor_hash");
|
||||
long checked = 0;
|
||||
Long first = null;
|
||||
Long last = null;
|
||||
Set<Long> missing = selection == null ? null : new HashSet<>(selection);
|
||||
CheckpointCursor checkpoints = new CheckpointCursor(store, selection, anchor, result);
|
||||
while(true)
|
||||
{
|
||||
List<QRecord> page = store.page(WormAuditTables.CHAIN, lastId, pageSize);
|
||||
if(page.isEmpty())
|
||||
{
|
||||
break;
|
||||
}
|
||||
List<QRecord> selected = page.stream().filter(r -> selection == null || selection.contains(r.getValueLong("id"))).toList();
|
||||
Map<Long, QRecord> headers = AuditStore.index(store.in("audit", "id", selected.stream().map(r -> r.getValueLong("audit_id")).toList()));
|
||||
Set<Long> detailIds = new HashSet<>();
|
||||
for(QRecord link : selected)
|
||||
{
|
||||
try
|
||||
{
|
||||
List<Long> ids = AuditStore.detailIds(link);
|
||||
if(ids.size() <= 250)
|
||||
{
|
||||
detailIds.addAll(ids);
|
||||
}
|
||||
}
|
||||
catch(IllegalArgumentException e)
|
||||
{
|
||||
// Content verification reports malformed coverage with its link id.
|
||||
}
|
||||
}
|
||||
Map<Long, QRecord> details = new HashMap<>();
|
||||
List<Long> ids = new ArrayList<>(detailIds);
|
||||
for(int start = 0; start < ids.size(); start += pageSize)
|
||||
{
|
||||
details.putAll(AuditStore.index(store.in("auditDetail", "id", ids.subList(start, Math.min(start + pageSize, ids.size())))));
|
||||
}
|
||||
for(QRecord link : page)
|
||||
{
|
||||
long id = link.getValueLong("id");
|
||||
long auditId = link.getValueLong("audit_id");
|
||||
if(selection == null || selection.contains(id))
|
||||
{
|
||||
if(missing != null)
|
||||
{
|
||||
missing.remove(id);
|
||||
}
|
||||
checked++;
|
||||
first = first == null ? id : first;
|
||||
last = id;
|
||||
if(selection == null && id != lastId + 1)
|
||||
{
|
||||
result.add(id, auditId, "Missing link ids " + number(lastId + 1) + "–" + number(id - 1) + " (gap)");
|
||||
}
|
||||
if(id != lastId + 1 || previous == null || !previous.equals(link.getValueString("prev_hash")))
|
||||
{
|
||||
result.add(id, auditId, "Break at link " + number(id) + " (audit " + number(auditId) + "): prev-hash discontinuity");
|
||||
}
|
||||
verifyContent(store, link, headers.get(auditId), details, result);
|
||||
}
|
||||
previous = link.getValueString("link_hash");
|
||||
lastId = id;
|
||||
}
|
||||
}
|
||||
// The main scan's page is out of scope before checkpoint pages are read.
|
||||
checkpoints.through(Long.MAX_VALUE);
|
||||
result.scope(checked, first, last);
|
||||
if(missing != null)
|
||||
{
|
||||
missing.stream().sorted().forEach(id -> result.add(id, null, "Missing selected link " + number(id)));
|
||||
}
|
||||
if(selection == null)
|
||||
{
|
||||
Instant cutoff = now.minusSeconds(settings.settle_horizon_seconds());
|
||||
if(store.isRdbms())
|
||||
{
|
||||
store.completeness(cutoff, result);
|
||||
}
|
||||
else if(store.isMemory())
|
||||
{
|
||||
completeness(store, store.all(WormAuditTables.CHAIN), cutoff, result);
|
||||
}
|
||||
else
|
||||
{
|
||||
pagedCompleteness(store, cutoff, result);
|
||||
}
|
||||
}
|
||||
return (result);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Recompute exact stored content. Diagnostics name covered detail ids because
|
||||
** v1 stores a document hash, not per-field hashes or prior values.
|
||||
*******************************************************************************/
|
||||
private void verifyContent(AuditStore store, QRecord link, QRecord audit, Map<Long, QRecord> rows, VerificationResult result) throws QException
|
||||
{
|
||||
long id = link.getValueLong("id");
|
||||
long auditId = link.getValueLong("audit_id");
|
||||
if(List.of("kind", "detail_ids", "prev_hash", "link_hash", "chained_at").stream().anyMatch(field -> link.getValue(field) == null))
|
||||
{
|
||||
result.add(id, auditId, "Break at link " + number(id) + " (audit " + number(auditId) + "): malformed link");
|
||||
return;
|
||||
}
|
||||
if(audit == null)
|
||||
{
|
||||
result.add(id, auditId, "Break at link " + number(id) + " (audit " + number(auditId) + "): audit header is missing");
|
||||
return;
|
||||
}
|
||||
if(audit.getValueInstant("timestamp") == null)
|
||||
{
|
||||
result.add(id, auditId, "Break at link " + number(id) + " (audit " + number(auditId) + "): missing audit timestamp");
|
||||
return;
|
||||
}
|
||||
try
|
||||
{
|
||||
List<Long> details = AuditStore.detailIds(link);
|
||||
String kind = link.getValueString("kind");
|
||||
if(!List.of("AUDIT", "LATE_DETAIL").contains(kind) || details.size() > 250 || ("LATE_DETAIL".equals(kind) && details.isEmpty()))
|
||||
{
|
||||
result.add(id, auditId, "Break at link " + number(id) + ": invalid kind or detail coverage");
|
||||
}
|
||||
if(details.size() > 250)
|
||||
{
|
||||
return;
|
||||
}
|
||||
Set<Long> unique = new HashSet<>();
|
||||
for(long detail : details)
|
||||
{
|
||||
if(!unique.add(detail))
|
||||
{
|
||||
result.add(id, auditId, "audit_detail " + number(detail) + " is covered twice or more by the chain");
|
||||
}
|
||||
}
|
||||
String hash = Hashes.link(link.getValueString("prev_hash"), store.canonical(audit, details, rows));
|
||||
if(!hash.equals(link.getValueString("link_hash")))
|
||||
{
|
||||
String table = store.name("auditTable", audit.getValueLong("auditTableId"));
|
||||
result.add(id, auditId, "Break at link " + number(id) + " (audit " + number(auditId) + ", table " + table + ", record "
|
||||
+ audit.getValueString("recordId") + "): content differs from the chained hash"
|
||||
+ (details.isEmpty() ? "" : " (audit_detail " + details.stream().map(WormAuditVerifier::number).collect(java.util.stream.Collectors.joining(", ")) + ")"));
|
||||
}
|
||||
}
|
||||
catch(QException | IllegalArgumentException e)
|
||||
{
|
||||
result.add(id, auditId, "Break at link " + number(id) + " (audit " + number(auditId) + "): " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Portable completeness for non-SQL backends. Trade repeated scans for bounded
|
||||
** memory; only the memory backend uses the whole-chain coverage index.
|
||||
*******************************************************************************/
|
||||
private void pagedCompleteness(AuditStore store, Instant cutoff, VerificationResult result) throws QException
|
||||
{
|
||||
for(String table : List.of("audit", "auditDetail"))
|
||||
{
|
||||
long after = 0;
|
||||
while(true)
|
||||
{
|
||||
List<QRecord> rows = store.page(table, after, pageSize);
|
||||
if(rows.isEmpty())
|
||||
{
|
||||
break;
|
||||
}
|
||||
Map<Long, Integer> counts = new HashMap<>();
|
||||
for(QRecord row : rows)
|
||||
{
|
||||
QRecord audit = table.equals("audit") ? row : store.get("audit", row.getValueLong("auditId"));
|
||||
if(audit == null || audit.getValueInstant("timestamp") == null || audit.getValueInstant("timestamp").isBefore(cutoff))
|
||||
{
|
||||
counts.put(row.getValueLong("id"), 0);
|
||||
if(table.equals("audit") && audit.getValueInstant("timestamp") == null)
|
||||
{
|
||||
result.add(null, row.getValueLong("id"), "audit " + number(row.getValueLong("id")) + " has no timestamp");
|
||||
}
|
||||
}
|
||||
}
|
||||
long linkAfter = 0;
|
||||
while(!counts.isEmpty())
|
||||
{
|
||||
List<QRecord> links = store.page(WormAuditTables.CHAIN, linkAfter, pageSize);
|
||||
if(links.isEmpty())
|
||||
{
|
||||
break;
|
||||
}
|
||||
for(QRecord link : links)
|
||||
{
|
||||
if(table.equals("audit"))
|
||||
{
|
||||
if("AUDIT".equals(link.getValueString("kind")))
|
||||
{
|
||||
counts.computeIfPresent(link.getValueLong("audit_id"), (id, count) -> count + 1);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
try
|
||||
{
|
||||
AuditStore.detailIds(link).forEach(id -> counts.computeIfPresent(id, (key, count) -> count + 1));
|
||||
}
|
||||
catch(IllegalArgumentException e)
|
||||
{
|
||||
// Malformed coverage was reported by content verification.
|
||||
}
|
||||
}
|
||||
}
|
||||
linkAfter = links.getLast().getValueLong("id");
|
||||
}
|
||||
counts.forEach((id, count) -> coverage(table.equals("audit") ? "audit" : "audit_detail", id, count, result));
|
||||
after = rows.getLast().getValueLong("id");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every settled header and detail must occur once; orphan details also fail.
|
||||
*******************************************************************************/
|
||||
private void completeness(AuditStore store, List<QRecord> links, Instant cutoff, VerificationResult result) throws QException
|
||||
{
|
||||
Map<Long, Integer> headers = new HashMap<>();
|
||||
Map<Long, Integer> details = new HashMap<>();
|
||||
for(QRecord link : links)
|
||||
{
|
||||
if("AUDIT".equals(link.getValueString("kind")))
|
||||
{
|
||||
headers.merge(link.getValueLong("audit_id"), 1, Integer::sum);
|
||||
}
|
||||
try
|
||||
{
|
||||
AuditStore.detailIds(link).forEach(id -> details.merge(id, 1, Integer::sum));
|
||||
}
|
||||
catch(IllegalArgumentException e)
|
||||
{
|
||||
// Invalid coverage was already reported while checking link content.
|
||||
}
|
||||
}
|
||||
Map<Long, QRecord> audits = new HashMap<>();
|
||||
for(QRecord audit : store.all("audit"))
|
||||
{
|
||||
long id = audit.getValueLong("id");
|
||||
audits.put(id, audit);
|
||||
if(audit.getValueInstant("timestamp") == null)
|
||||
{
|
||||
result.add(null, id, "audit " + number(id) + " has no timestamp");
|
||||
}
|
||||
else if(audit.getValueInstant("timestamp").isBefore(cutoff))
|
||||
{
|
||||
coverage("audit", id, headers.getOrDefault(id, 0), result);
|
||||
}
|
||||
}
|
||||
for(QRecord detail : store.all("auditDetail"))
|
||||
{
|
||||
QRecord audit = audits.get(detail.getValueLong("auditId"));
|
||||
if(audit == null || audit.getValueInstant("timestamp") == null || audit.getValueInstant("timestamp").isBefore(cutoff))
|
||||
{
|
||||
coverage("audit_detail", detail.getValueLong("id"), details.getOrDefault(detail.getValueLong("id"), 0), result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Report coverage without printing any audit content.
|
||||
*******************************************************************************/
|
||||
private void coverage(String table, long id, int count, VerificationResult result)
|
||||
{
|
||||
if(count != 1)
|
||||
{
|
||||
result.add(null, table.equals("audit") ? id : null, table + " " + number(id)
|
||||
+ (count == 0 ? " is not covered by the chain" : " is covered twice or more by the chain"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Check pinned signatures even for purged ranges. A partially purged range's
|
||||
** Merkle tree requires its sealed export; never claim to recompute it live.
|
||||
*******************************************************************************/
|
||||
private class CheckpointCursor
|
||||
{
|
||||
private final AuditStore store;
|
||||
private final Set<Long> selection;
|
||||
private final QRecord anchor;
|
||||
private final VerificationResult result;
|
||||
private long after;
|
||||
private QRecord next;
|
||||
|
||||
/*******************************************************************************
|
||||
** Keep only one checkpoint pending as pages complete.
|
||||
*******************************************************************************/
|
||||
CheckpointCursor(AuditStore store, Set<Long> selection, QRecord anchor, VerificationResult result)
|
||||
{
|
||||
this.store = store;
|
||||
this.selection = selection;
|
||||
this.anchor = anchor;
|
||||
this.result = result;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify completed ranges, including missing tails at end of the scan.
|
||||
*******************************************************************************/
|
||||
void through(long lastId) throws QException
|
||||
{
|
||||
while(true)
|
||||
{
|
||||
if(next == null)
|
||||
{
|
||||
List<QRecord> rows = store.page(WormAuditTables.CHECKPOINT, after, 1);
|
||||
if(rows.isEmpty())
|
||||
{
|
||||
return;
|
||||
}
|
||||
next = rows.getFirst();
|
||||
}
|
||||
Long to = next.getValueLong("to_link_id");
|
||||
if(to != null && to > lastId)
|
||||
{
|
||||
return;
|
||||
}
|
||||
check(next);
|
||||
after = next.getValueLong("id");
|
||||
next = null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Re-read at most one capped range of hashes in the same snapshot.
|
||||
*******************************************************************************/
|
||||
private void check(QRecord checkpoint) throws QException
|
||||
{
|
||||
String prefix = "checkpoint " + number(checkpoint.getValueLong("id")) + ": ";
|
||||
if(List.of("from_link_id", "to_link_id", "link_count", "head_hash", "merkle_root", "algorithm", "key_id", "public_key", "signature")
|
||||
.stream().anyMatch(field -> checkpoint.getValue(field) == null))
|
||||
{
|
||||
result.add(null, null, prefix + "malformed checkpoint");
|
||||
return;
|
||||
}
|
||||
long from = checkpoint.getValueLong("from_link_id");
|
||||
long to = checkpoint.getValueLong("to_link_id");
|
||||
long anchorId = anchor.getValueLong("anchor_link_id");
|
||||
if(selection != null && selection.stream().noneMatch(id -> id >= from && id <= to))
|
||||
{
|
||||
return;
|
||||
}
|
||||
var pin = config.getTrustedPublicKeys().get(checkpoint.getValueString("key_id"));
|
||||
if(pin == null)
|
||||
{
|
||||
result.add(null, null, prefix + "untrusted key");
|
||||
}
|
||||
else if(!Checkpoints.verify(checkpoint, pin))
|
||||
{
|
||||
result.add(null, null, prefix + "signature does not verify");
|
||||
}
|
||||
if(from < 1 || to < from || checkpoint.getValueLong("link_count") != to - from + 1
|
||||
|| to - from + 1 > Integer.MAX_VALUE)
|
||||
{
|
||||
result.add(null, null, prefix + "invalid link range or count");
|
||||
return;
|
||||
}
|
||||
if(from <= anchorId)
|
||||
{
|
||||
QRecord head = to > anchorId ? store.get(WormAuditTables.CHAIN, to) : null;
|
||||
if(to > anchorId && (head == null || !Objects.equals(head.getValueString("link_hash"), checkpoint.getValueString("head_hash"))))
|
||||
{
|
||||
result.add(to, null, prefix + "head mismatch");
|
||||
}
|
||||
if(to == anchorId && !Objects.equals(anchor.getValueString("anchor_hash"), checkpoint.getValueString("head_hash")))
|
||||
{
|
||||
result.add(to, null, prefix + "head mismatch with purge anchor");
|
||||
}
|
||||
result.warn(prefix + "purged range; Merkle proof requires the sealed export");
|
||||
return;
|
||||
}
|
||||
List<String> hashes = new ArrayList<>();
|
||||
long cursor = from - 1;
|
||||
String head = null;
|
||||
while(cursor < to)
|
||||
{
|
||||
List<QRecord> page = store.page(WormAuditTables.CHAIN, cursor, (int) Math.min(pageSize, to - cursor));
|
||||
if(page.isEmpty())
|
||||
{
|
||||
break;
|
||||
}
|
||||
for(QRecord link : page)
|
||||
{
|
||||
cursor = link.getValueLong("id");
|
||||
if(cursor <= to)
|
||||
{
|
||||
hashes.add(link.getValueString("link_hash"));
|
||||
}
|
||||
if(cursor == to)
|
||||
{
|
||||
head = link.getValueString("link_hash");
|
||||
}
|
||||
}
|
||||
}
|
||||
if(!Objects.equals(head, checkpoint.getValueString("head_hash")))
|
||||
{
|
||||
result.add(to, null, prefix + "head mismatch");
|
||||
}
|
||||
try
|
||||
{
|
||||
if(hashes.size() != checkpoint.getValueLong("link_count") || hashes.isEmpty()
|
||||
|| hashes.stream().anyMatch(Objects::isNull) || !Hashes.merkle(hashes).equals(checkpoint.getValueString("merkle_root")))
|
||||
{
|
||||
result.add(null, null, prefix + "merkle mismatch");
|
||||
}
|
||||
}
|
||||
catch(IllegalArgumentException e)
|
||||
{
|
||||
result.add(null, null, prefix + "merkle mismatch");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stable human-readable ids matching the process summary contract.
|
||||
*******************************************************************************/
|
||||
public static String number(long value)
|
||||
{
|
||||
return (String.format(Locale.US, "%,d", value));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,556 @@
|
||||
-- liquibase formatted sql
|
||||
|
||||
-- changeset worm-audit:1
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_chain (
|
||||
id bigint PRIMARY KEY,
|
||||
kind text NOT NULL,
|
||||
audit_id bigint NOT NULL,
|
||||
detail_ids text NOT NULL,
|
||||
prev_hash text NOT NULL,
|
||||
link_hash text NOT NULL,
|
||||
chained_at timestamptz NOT NULL,
|
||||
CHECK (id > 0),
|
||||
CHECK (kind IN ('AUDIT', 'LATE_DETAIL')),
|
||||
CHECK (prev_hash ~ '^[0-9a-f]{64}$' AND link_hash ~ '^[0-9a-f]{64}$'),
|
||||
CHECK (detail_ids = '' OR (detail_ids ~ '^(0|[1-9][0-9]*)(,(0|[1-9][0-9]*))*$' AND cardinality(string_to_array(detail_ids, ',')) <= 250))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_checkpoint (
|
||||
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
from_link_id bigint,
|
||||
to_link_id bigint,
|
||||
link_count bigint,
|
||||
head_hash text,
|
||||
merkle_root text,
|
||||
algorithm text,
|
||||
key_id text,
|
||||
public_key text,
|
||||
signature text,
|
||||
created_at timestamptz
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_export (
|
||||
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
from_link_id bigint,
|
||||
to_link_id bigint,
|
||||
from_checkpoint_id bigint,
|
||||
to_checkpoint_id bigint,
|
||||
bucket_alias text,
|
||||
object_prefix text,
|
||||
object_versions text,
|
||||
manifest_key text,
|
||||
manifest_sha256 text,
|
||||
prev_manifest_sha256 text,
|
||||
object_count bigint,
|
||||
byte_count bigint,
|
||||
lock_mode text,
|
||||
retain_until timestamptz,
|
||||
sealed_at timestamptz
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_verification (
|
||||
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
kind text,
|
||||
"trigger" text,
|
||||
started_at timestamptz,
|
||||
finished_at timestamptz,
|
||||
from_link_id bigint,
|
||||
to_link_id bigint,
|
||||
links_checked bigint,
|
||||
outcome text,
|
||||
break_count bigint,
|
||||
first_break_link_id bigint,
|
||||
first_break_audit_id bigint,
|
||||
first_break_reason text,
|
||||
export_id bigint,
|
||||
run_by text
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_purge (
|
||||
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
requested_at timestamptz,
|
||||
requested_by text,
|
||||
cutoff timestamptz,
|
||||
retention_interval text,
|
||||
outcome text,
|
||||
reason text,
|
||||
audits_deleted bigint,
|
||||
details_deleted bigint,
|
||||
links_deleted bigint,
|
||||
anchor_link_id bigint,
|
||||
anchor_hash text
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS worm_audit_chain_audit ON public.worm_audit_chain(audit_id) WHERE kind = 'AUDIT';
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_chain_audit_all ON public.worm_audit_chain(audit_id);
|
||||
|
||||
-- changeset worm-audit:2 splitStatements:false
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_reject() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp SET timezone = 'UTC' AS $$
|
||||
DECLARE
|
||||
old_time timestamptz;
|
||||
function_owner name;
|
||||
stack text;
|
||||
row_identity text := 'all';
|
||||
BEGIN
|
||||
IF TG_OP = 'DELETE' AND current_setting('worm_audit.purging', true) = 'on' THEN
|
||||
SELECT pg_get_userbyid(proowner) INTO function_owner FROM pg_proc
|
||||
WHERE oid = 'public.worm_audit_purge(timestamptz)'::regprocedure;
|
||||
GET DIAGNOSTICS stack = PG_CONTEXT;
|
||||
IF current_user = function_owner AND position('function worm_audit_purge(timestamp with time zone)' in stack) > 0 THEN
|
||||
IF TG_RELID = '${audit}'::regclass THEN
|
||||
old_time := OLD.timestamp;
|
||||
ELSIF TG_RELID = '${audit_detail}'::regclass THEN
|
||||
SELECT timestamp INTO old_time FROM ${audit} WHERE id = OLD.audit_id;
|
||||
ELSIF TG_RELID = 'public.worm_audit_chain'::regclass THEN
|
||||
SELECT timestamp INTO old_time FROM ${audit} WHERE id = OLD.audit_id;
|
||||
END IF;
|
||||
IF old_time < now() - interval '${retention}' THEN
|
||||
RETURN OLD;
|
||||
END IF;
|
||||
END IF;
|
||||
END IF;
|
||||
IF TG_LEVEL = 'ROW' THEN
|
||||
row_identity := OLD.id::text;
|
||||
END IF;
|
||||
RAISE EXCEPTION USING ERRCODE = 'P0001',
|
||||
MESSAGE = format('worm_audit: audit rows are write-once; %s of "%s" is not allowed (id=%s). Expired rows are removed only by worm_audit_purge().',
|
||||
TG_OP, TG_TABLE_NAME, row_identity),
|
||||
HINT = 'see qbit-worm-audit docs/DESIGN.md section 4';
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:3 splitStatements:false
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_lookup_reject() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $$
|
||||
BEGIN
|
||||
IF TG_OP = 'UPDATE' AND NEW.id IS NOT DISTINCT FROM OLD.id AND NEW.name IS NOT DISTINCT FROM OLD.name THEN
|
||||
RETURN NEW;
|
||||
END IF;
|
||||
RAISE EXCEPTION USING ERRCODE = 'P0001',
|
||||
MESSAGE = format('worm_audit: audit rows are write-once; %s of "%s" is not allowed (id=%s). Expired rows are removed only by worm_audit_purge().',
|
||||
TG_OP, TG_TABLE_NAME, OLD.id),
|
||||
HINT = 'see qbit-worm-audit docs/DESIGN.md section 4';
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:4 splitStatements:false
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'worm_audit_purger') THEN
|
||||
CREATE ROLE worm_audit_purger NOLOGIN;
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:5 splitStatements:false
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_purge(p_cutoff timestamptz) RETURNS void
|
||||
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, public, pg_temp SET timezone = 'UTC' AS $$
|
||||
DECLARE
|
||||
boundary bigint;
|
||||
smaller bigint;
|
||||
anchor text;
|
||||
audits_count bigint;
|
||||
details_count bigint;
|
||||
links_count bigint;
|
||||
BEGIN
|
||||
IF p_cutoff IS NULL OR p_cutoff > now() - interval '${retention}' THEN
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome, reason)
|
||||
VALUES(now(), session_user, p_cutoff, '${retention}', 'REFUSED', 'Cutoff is within retention or null');
|
||||
RETURN;
|
||||
END IF;
|
||||
-- Same lock as the Java chainer; table locks also exclude concurrent audit inserts.
|
||||
PERFORM pg_advisory_xact_lock(86753090421);
|
||||
LOCK TABLE ${audit}, ${audit_detail}, public.worm_audit_chain IN SHARE ROW EXCLUSIVE MODE;
|
||||
SELECT coalesce(min(c.id) - 1, (SELECT max(id) FROM public.worm_audit_chain)) INTO boundary
|
||||
FROM public.worm_audit_chain c LEFT JOIN ${audit} a ON a.id = c.audit_id
|
||||
WHERE a.id IS NULL OR a.timestamp IS NULL OR a.timestamp >= p_cutoff;
|
||||
-- Shrink to a prefix closed over every audit's links AND all its stored details.
|
||||
LOOP
|
||||
SELECT min(c.id) - 1 INTO smaller FROM public.worm_audit_chain c
|
||||
WHERE c.id <= boundary AND (
|
||||
EXISTS (SELECT FROM public.worm_audit_chain later WHERE later.audit_id = c.audit_id AND later.id > boundary)
|
||||
OR EXISTS (SELECT FROM ${audit_detail} d WHERE d.audit_id = c.audit_id AND NOT EXISTS (
|
||||
SELECT FROM public.worm_audit_chain covered WHERE covered.audit_id = c.audit_id
|
||||
AND d.id::text = ANY(string_to_array(covered.detail_ids, ',')))));
|
||||
EXIT WHEN smaller IS NULL;
|
||||
boundary := smaller;
|
||||
END LOOP;
|
||||
SELECT link_hash INTO anchor FROM public.worm_audit_chain WHERE id = boundary;
|
||||
IF anchor IS NULL THEN
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome, reason)
|
||||
VALUES(now(), session_user, p_cutoff, '${retention}', 'REFUSED', 'No complete expired chain prefix');
|
||||
RETURN;
|
||||
END IF;
|
||||
PERFORM set_config('worm_audit.purging', 'on', true);
|
||||
DELETE FROM ${audit_detail} d WHERE EXISTS
|
||||
(SELECT FROM public.worm_audit_chain c WHERE c.audit_id = d.audit_id AND c.id <= boundary);
|
||||
GET DIAGNOSTICS details_count = ROW_COUNT;
|
||||
-- Keep ids locally because links must be deleted before their audit headers.
|
||||
WITH removed AS (DELETE FROM public.worm_audit_chain WHERE id <= boundary RETURNING audit_id),
|
||||
removed_audits AS (DELETE FROM ${audit} WHERE id IN (SELECT audit_id FROM removed) RETURNING id)
|
||||
SELECT (SELECT count(*) FROM removed), (SELECT count(*) FROM removed_audits) INTO links_count, audits_count;
|
||||
PERFORM set_config('worm_audit.purging', 'off', true);
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome,
|
||||
audits_deleted, details_deleted, links_deleted, anchor_link_id, anchor_hash)
|
||||
VALUES(now(), session_user, p_cutoff, '${retention}', 'PURGED', audits_count, details_count, links_count, boundary, anchor);
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:6
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON ${audit};
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON ${audit}
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON ${audit};
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON ${audit}
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON ${audit} FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON ${audit} TO ${app_role};
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON ${audit_detail};
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON ${audit_detail}
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON ${audit_detail};
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON ${audit_detail}
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON ${audit_detail} FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON ${audit_detail} TO ${app_role};
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_chain;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_chain
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_chain;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_chain
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_chain FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON public.worm_audit_chain TO ${app_role};
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_checkpoint;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_checkpoint
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_checkpoint;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_checkpoint
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_checkpoint FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON public.worm_audit_checkpoint TO ${app_role};
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_export;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_export
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_export;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_export
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_export FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON public.worm_audit_export TO ${app_role};
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_verification;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_verification
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_verification;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_verification
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_verification FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON public.worm_audit_verification TO ${app_role};
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_purge;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_purge
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_purge;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_purge
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_purge FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON public.worm_audit_purge TO ${app_role};
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON ${audit_table};
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON ${audit_table}
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_lookup_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON ${audit_table};
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON ${audit_table}
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON ${audit_table} FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON ${audit_table} TO ${app_role};
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON ${audit_user};
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON ${audit_user}
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_lookup_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON ${audit_user};
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON ${audit_user}
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON ${audit_user} FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON ${audit_user} TO ${app_role};
|
||||
|
||||
-- Lookups may refresh display metadata, but never identities or names.
|
||||
GRANT UPDATE (label, create_date, modify_date) ON ${audit_table} TO ${app_role};
|
||||
GRANT UPDATE (create_date, modify_date) ON ${audit_user} TO ${app_role};
|
||||
|
||||
REVOKE ALL ON FUNCTION public.worm_audit_purge(timestamptz) FROM PUBLIC;
|
||||
REVOKE ALL ON FUNCTION public.worm_audit_purge(timestamptz) FROM ${app_role};
|
||||
GRANT EXECUTE ON FUNCTION public.worm_audit_purge(timestamptz) TO worm_audit_purger;
|
||||
|
||||
-- changeset worm-audit:7 splitStatements:false
|
||||
DO $$
|
||||
DECLARE
|
||||
table_name text;
|
||||
sequence_name text;
|
||||
BEGIN
|
||||
FOREACH table_name IN ARRAY ARRAY['${audit}', '${audit_detail}', '${audit_table}', '${audit_user}',
|
||||
'public.worm_audit_checkpoint', 'public.worm_audit_export', 'public.worm_audit_verification', 'public.worm_audit_purge'] LOOP
|
||||
sequence_name := pg_get_serial_sequence(table_name, 'id');
|
||||
IF sequence_name IS NOT NULL THEN
|
||||
EXECUTE format('GRANT USAGE ON SEQUENCE %s TO ${app_role}', sequence_name);
|
||||
END IF;
|
||||
END LOOP;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:8 splitStatements:false
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_detail_coverage() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $$
|
||||
BEGIN
|
||||
-- Serialize even direct SQL appends; the GIN index makes overlap checks indexed.
|
||||
PERFORM pg_advisory_xact_lock(86753090421);
|
||||
IF NEW.detail_ids <> '' AND (
|
||||
EXISTS (SELECT FROM unnest(string_to_array(NEW.detail_ids, ',')) AS d(id) GROUP BY d.id HAVING count(*) > 1)
|
||||
OR EXISTS (SELECT FROM public.worm_audit_chain c
|
||||
WHERE string_to_array(c.detail_ids, ',') && string_to_array(NEW.detail_ids, ','))) THEN
|
||||
RAISE EXCEPTION USING ERRCODE = '23505', MESSAGE = 'worm_audit: audit detail is already covered by the chain';
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:9
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_chain_detail_coverage ON public.worm_audit_chain USING gin (string_to_array(detail_ids, ','));
|
||||
DROP TRIGGER IF EXISTS worm_audit_detail_coverage_guard ON public.worm_audit_chain;
|
||||
CREATE TRIGGER worm_audit_detail_coverage_guard BEFORE INSERT ON public.worm_audit_chain
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_detail_coverage();
|
||||
|
||||
-- changeset worm-audit:10
|
||||
-- Editable singleton: deliberately excluded from append-only triggers.
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_settings (
|
||||
id integer PRIMARY KEY CHECK (id = 1),
|
||||
audit_retention_years integer NOT NULL DEFAULT 6 CHECK (audit_retention_years BETWEEN 1 AND 100),
|
||||
export_lock_mode text NOT NULL DEFAULT 'GOVERNANCE' CHECK (export_lock_mode IN ('GOVERNANCE', 'COMPLIANCE')),
|
||||
export_retention_days integer NOT NULL DEFAULT 30 CHECK (export_retention_days BETWEEN 1 AND 36500),
|
||||
checkpoint_every_links integer NOT NULL DEFAULT 10000 CHECK (checkpoint_every_links > 0),
|
||||
checkpoint_every_minutes integer NOT NULL DEFAULT 60 CHECK (checkpoint_every_minutes > 0),
|
||||
chain_interval_seconds integer NOT NULL DEFAULT 30 CHECK (chain_interval_seconds > 0),
|
||||
settle_horizon_seconds integer NOT NULL DEFAULT 60 CHECK (settle_horizon_seconds >= 0),
|
||||
late_detail_lookback_hours integer NOT NULL DEFAULT 24 CHECK (late_detail_lookback_hours > 0),
|
||||
export_cron text NOT NULL DEFAULT '0 0 2 * * ?',
|
||||
verify_cron text NOT NULL DEFAULT '0 0 3 * * ?',
|
||||
notes text,
|
||||
modify_date timestamptz NOT NULL DEFAULT now(),
|
||||
modified_by text NOT NULL DEFAULT current_user
|
||||
);
|
||||
REVOKE DELETE, TRUNCATE ON public.worm_audit_settings FROM ${app_role};
|
||||
GRANT SELECT, INSERT, UPDATE ON public.worm_audit_settings TO ${app_role};
|
||||
INSERT INTO public.worm_audit_settings
|
||||
(id, audit_retention_years, export_lock_mode, export_retention_days, checkpoint_every_links,
|
||||
checkpoint_every_minutes, chain_interval_seconds, settle_horizon_seconds, late_detail_lookback_hours,
|
||||
export_cron, verify_cron)
|
||||
SELECT ${settings_values}
|
||||
WHERE NOT EXISTS (SELECT 1 FROM public.worm_audit_settings WHERE id = 1);
|
||||
|
||||
-- changeset worm-audit:11
|
||||
-- comment: Index the foreign key columns added by worm-audit:12 (worm_audit_chain.audit_id already leads worm_audit_chain_audit_all).
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_export_from_checkpoint ON public.worm_audit_export(from_checkpoint_id);
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_export_to_checkpoint ON public.worm_audit_export(to_checkpoint_id);
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_verification_export ON public.worm_audit_verification(export_id);
|
||||
|
||||
-- changeset worm-audit:12 splitStatements:false
|
||||
-- comment: Foreign keys, ON DELETE RESTRICT only: nothing cascades into or out of audit evidence. A chain link needs its audit header (the purge deletes links before headers); a sealed export needs its signed checkpoints and an export verification its export (neither is ever deleted). Links referenced by checkpoints, exports, verifications and purges are not foreign keys: the purge deletes the sealed prefix those rows outlive. Rows whose parent is already missing are evidence and are never changed: they are copied to qqq_fk_quarantine (action KEPT) and the constraint is left NOT VALID (enforced for new rows only).
|
||||
DO $$
|
||||
DECLARE
|
||||
fk record;
|
||||
orphans bigint;
|
||||
BEGIN
|
||||
CREATE TABLE IF NOT EXISTS qqq_fk_quarantine (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, table_name VARCHAR(100) NOT NULL, column_name VARCHAR(100) NOT NULL, row_id VARCHAR(100), missing_parent_id VARCHAR(100), action VARCHAR(20) NOT NULL, row_data TEXT, changeset VARCHAR(200), quarantined_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP);
|
||||
FOR fk IN SELECT * FROM (VALUES
|
||||
('fk_worm_audit_chain_audit_id', 'worm_audit_chain', 'audit_id', '${audit}'),
|
||||
('fk_worm_audit_export_from_checkpoint_id', 'worm_audit_export', 'from_checkpoint_id', 'public.worm_audit_checkpoint'),
|
||||
('fk_worm_audit_export_to_checkpoint_id', 'worm_audit_export', 'to_checkpoint_id', 'public.worm_audit_checkpoint'),
|
||||
('fk_worm_audit_verification_export_id', 'worm_audit_verification', 'export_id', 'public.worm_audit_export')
|
||||
) AS t(name, child, col, parent) LOOP
|
||||
IF NOT EXISTS (SELECT FROM pg_constraint WHERE conname = fk.name) THEN
|
||||
EXECUTE format('INSERT INTO qqq_fk_quarantine (table_name, column_name, row_id, missing_parent_id, action, row_data, changeset)'
|
||||
|| ' SELECT %L, %L, c.id::text, c.%I::text, ''KEPT'', row_to_json(c)::text, ''worm-audit:12'' FROM public.%I c'
|
||||
|| ' WHERE c.%I IS NOT NULL AND NOT EXISTS (SELECT FROM %s p WHERE p.id = c.%I)', fk.child, fk.col, fk.col, fk.child, fk.col, fk.parent, fk.col);
|
||||
GET DIAGNOSTICS orphans = ROW_COUNT;
|
||||
EXECUTE format('ALTER TABLE public.%I ADD CONSTRAINT %I FOREIGN KEY (%I) REFERENCES %s (id) ON DELETE RESTRICT NOT VALID', fk.child, fk.name, fk.col, fk.parent);
|
||||
IF orphans = 0 THEN
|
||||
EXECUTE format('ALTER TABLE public.%I VALIDATE CONSTRAINT %I', fk.child, fk.name);
|
||||
END IF;
|
||||
END IF;
|
||||
END LOOP;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:13 splitStatements:false
|
||||
-- comment: Normalized detail coverage. One insert-only row per audit_detail id covered by a chain link, written by the coverage trigger in the same statement as the link, so the chain writer probes a primary key instead of the GIN array index. Append-only like the chain, with no foreign keys (the owner cannot UPDATE WORM tables, so FK checks would be denied). The advisory lock keeps a running chainer from appending between the backfill and the new trigger body.
|
||||
DO $$
|
||||
BEGIN
|
||||
PERFORM pg_advisory_xact_lock(86753090421);
|
||||
END;
|
||||
$$;
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_chain_detail (
|
||||
detail_id bigint PRIMARY KEY,
|
||||
chain_id bigint NOT NULL,
|
||||
CHECK (detail_id >= 0),
|
||||
CHECK (chain_id > 0)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_chain_detail_chain ON public.worm_audit_chain_detail(chain_id);
|
||||
-- The first link wins if a tampered chain covers a detail twice; the verifier still reports the duplicate from the chain itself.
|
||||
INSERT INTO public.worm_audit_chain_detail(detail_id, chain_id)
|
||||
SELECT x.id::bigint, c.id FROM public.worm_audit_chain c CROSS JOIN LATERAL unnest(string_to_array(c.detail_ids, ',')) AS x(id)
|
||||
WHERE c.detail_ids <> '' ORDER BY c.id
|
||||
ON CONFLICT (detail_id) DO NOTHING;
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_reject() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp SET timezone = 'UTC' AS $fn$
|
||||
DECLARE
|
||||
old_time timestamptz;
|
||||
function_owner name;
|
||||
stack text;
|
||||
row_identity text := 'all';
|
||||
BEGIN
|
||||
IF TG_OP = 'DELETE' AND current_setting('worm_audit.purging', true) = 'on' THEN
|
||||
SELECT pg_get_userbyid(proowner) INTO function_owner FROM pg_proc
|
||||
WHERE oid = 'public.worm_audit_purge(timestamptz)'::regprocedure;
|
||||
GET DIAGNOSTICS stack = PG_CONTEXT;
|
||||
IF current_user = function_owner AND position('function worm_audit_purge(timestamp with time zone)' in stack) > 0 THEN
|
||||
IF TG_RELID = '${audit}'::regclass THEN
|
||||
old_time := OLD.timestamp;
|
||||
ELSIF TG_RELID = '${audit_detail}'::regclass THEN
|
||||
SELECT timestamp INTO old_time FROM ${audit} WHERE id = OLD.audit_id;
|
||||
ELSIF TG_RELID = 'public.worm_audit_chain'::regclass THEN
|
||||
SELECT timestamp INTO old_time FROM ${audit} WHERE id = OLD.audit_id;
|
||||
ELSIF TG_RELID = 'public.worm_audit_chain_detail'::regclass THEN
|
||||
SELECT a.timestamp INTO old_time FROM public.worm_audit_chain c JOIN ${audit} a ON a.id = c.audit_id WHERE c.id = OLD.chain_id;
|
||||
END IF;
|
||||
IF old_time < now() - interval '${retention}' THEN
|
||||
RETURN OLD;
|
||||
END IF;
|
||||
END IF;
|
||||
END IF;
|
||||
IF TG_LEVEL = 'ROW' THEN
|
||||
-- Record fields resolve per statement, so each table's key is read in its own branch.
|
||||
IF TG_RELID = 'public.worm_audit_chain_detail'::regclass THEN
|
||||
row_identity := OLD.detail_id::text;
|
||||
ELSE
|
||||
row_identity := OLD.id::text;
|
||||
END IF;
|
||||
END IF;
|
||||
RAISE EXCEPTION USING ERRCODE = 'P0001',
|
||||
MESSAGE = format('worm_audit: audit rows are write-once; %s of "%s" is not allowed (id=%s). Expired rows are removed only by worm_audit_purge().',
|
||||
TG_OP, TG_TABLE_NAME, row_identity),
|
||||
HINT = 'see qbit-worm-audit docs/DESIGN.md section 4';
|
||||
END;
|
||||
$fn$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_detail_coverage() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $fn$
|
||||
BEGIN
|
||||
-- Serialize even direct SQL appends, then record coverage in the same statement as the link.
|
||||
-- The primary key rejects a detail covered twice, within this link or by an earlier one. The
|
||||
-- previous EXISTS overlap check ran as a cached generic plan that chose a sequential scan of
|
||||
-- the whole chain for every inserted link.
|
||||
PERFORM pg_advisory_xact_lock(86753090421);
|
||||
IF NEW.detail_ids = '' THEN
|
||||
RETURN NEW;
|
||||
END IF;
|
||||
BEGIN
|
||||
INSERT INTO public.worm_audit_chain_detail(detail_id, chain_id)
|
||||
SELECT d, NEW.id FROM unnest(string_to_array(NEW.detail_ids, ',')::bigint[]) AS d;
|
||||
EXCEPTION WHEN unique_violation THEN
|
||||
RAISE EXCEPTION USING ERRCODE = '23505', MESSAGE = 'worm_audit: audit detail is already covered by the chain';
|
||||
END;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$fn$;
|
||||
|
||||
-- The array index worm_audit_chain_detail_coverage is no longer used by this release. It stays until
|
||||
-- a later migration drops it, so a previous-release writer still running during a rolling deploy keeps its indexed probe.
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_chain_detail;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_chain_detail
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_chain_detail;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_chain_detail
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_chain_detail FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON public.worm_audit_chain_detail TO ${app_role};
|
||||
|
||||
-- Same function as worm-audit:5, except that coverage rows are locked and purged before their links.
|
||||
-- The migrating role may have revoked its own EXECUTE (a runtime role that owns the schema, as on CNPG);
|
||||
-- PostgreSQL then refuses CREATE OR REPLACE. Grant it back for this statement only, then restore.
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT has_function_privilege(current_user, 'public.worm_audit_purge(timestamptz)', 'EXECUTE') THEN
|
||||
EXECUTE format('GRANT EXECUTE ON FUNCTION public.worm_audit_purge(timestamptz) TO %I', current_user);
|
||||
PERFORM set_config('worm_audit.purge_regrant', 'on', true);
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_purge(p_cutoff timestamptz) RETURNS void
|
||||
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, public, pg_temp SET timezone = 'UTC' AS $fn$
|
||||
DECLARE
|
||||
boundary bigint;
|
||||
smaller bigint;
|
||||
anchor text;
|
||||
audits_count bigint;
|
||||
details_count bigint;
|
||||
links_count bigint;
|
||||
BEGIN
|
||||
IF p_cutoff IS NULL OR p_cutoff > now() - interval '${retention}' THEN
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome, reason)
|
||||
VALUES(now(), session_user, p_cutoff, '${retention}', 'REFUSED', 'Cutoff is within retention or null');
|
||||
RETURN;
|
||||
END IF;
|
||||
-- Same lock as the Java chainer; table locks also exclude concurrent audit inserts.
|
||||
PERFORM pg_advisory_xact_lock(86753090421);
|
||||
LOCK TABLE ${audit}, ${audit_detail}, public.worm_audit_chain, public.worm_audit_chain_detail IN SHARE ROW EXCLUSIVE MODE;
|
||||
SELECT coalesce(min(c.id) - 1, (SELECT max(id) FROM public.worm_audit_chain)) INTO boundary
|
||||
FROM public.worm_audit_chain c LEFT JOIN ${audit} a ON a.id = c.audit_id
|
||||
WHERE a.id IS NULL OR a.timestamp IS NULL OR a.timestamp >= p_cutoff;
|
||||
-- Shrink to a prefix closed over every audit's links AND all its stored details.
|
||||
LOOP
|
||||
SELECT min(c.id) - 1 INTO smaller FROM public.worm_audit_chain c
|
||||
WHERE c.id <= boundary AND (
|
||||
EXISTS (SELECT FROM public.worm_audit_chain later WHERE later.audit_id = c.audit_id AND later.id > boundary)
|
||||
OR EXISTS (SELECT FROM ${audit_detail} d WHERE d.audit_id = c.audit_id AND NOT EXISTS (
|
||||
SELECT FROM public.worm_audit_chain covered WHERE covered.audit_id = c.audit_id
|
||||
AND d.id::text = ANY(string_to_array(covered.detail_ids, ',')))));
|
||||
EXIT WHEN smaller IS NULL;
|
||||
boundary := smaller;
|
||||
END LOOP;
|
||||
SELECT link_hash INTO anchor FROM public.worm_audit_chain WHERE id = boundary;
|
||||
IF anchor IS NULL THEN
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome, reason)
|
||||
VALUES(now(), session_user, p_cutoff, '${retention}', 'REFUSED', 'No complete expired chain prefix');
|
||||
RETURN;
|
||||
END IF;
|
||||
PERFORM set_config('worm_audit.purging', 'on', true);
|
||||
DELETE FROM ${audit_detail} d WHERE EXISTS
|
||||
(SELECT FROM public.worm_audit_chain c WHERE c.audit_id = d.audit_id AND c.id <= boundary);
|
||||
GET DIAGNOSTICS details_count = ROW_COUNT;
|
||||
-- Coverage rows go before their links: the guard finds their age through the link.
|
||||
DELETE FROM public.worm_audit_chain_detail WHERE chain_id <= boundary;
|
||||
-- Keep ids locally because links must be deleted before their audit headers.
|
||||
WITH removed AS (DELETE FROM public.worm_audit_chain WHERE id <= boundary RETURNING audit_id),
|
||||
removed_audits AS (DELETE FROM ${audit} WHERE id IN (SELECT audit_id FROM removed) RETURNING id)
|
||||
SELECT (SELECT count(*) FROM removed), (SELECT count(*) FROM removed_audits) INTO links_count, audits_count;
|
||||
PERFORM set_config('worm_audit.purging', 'off', true);
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome,
|
||||
audits_deleted, details_deleted, links_deleted, anchor_link_id, anchor_hash)
|
||||
VALUES(now(), session_user, p_cutoff, '${retention}', 'PURGED', audits_count, details_count, links_count, boundary, anchor);
|
||||
END;
|
||||
$fn$;
|
||||
DO $$
|
||||
BEGIN
|
||||
IF current_setting('worm_audit.purge_regrant', true) = 'on' THEN
|
||||
EXECUTE format('REVOKE EXECUTE ON FUNCTION public.worm_audit_purge(timestamptz) FROM %I', current_user);
|
||||
PERFORM set_config('worm_audit.purge_regrant', 'off', true);
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
@@ -0,0 +1,50 @@
|
||||
bundle: qbit-worm-audit
|
||||
version: 1
|
||||
layer: QBIT
|
||||
# Help for the WORM Audit app and every worm_audit_* table, in the QQQ help-bundle
|
||||
# format (qqq#1198). Each table entry says what the table is for and how its rows
|
||||
# get created, so an empty table explains itself. WormAuditHelp applies it until
|
||||
# the core HelpBundles registry ships in a QQQ release
|
||||
# (STAND-IN(core=QRun-IO/qqq#1198, removeBy=4.1.0-RC.7)). Summaries stay under 120
|
||||
# characters. Only the settings table is seeded (by the database script, see the
|
||||
# README); every other table is written by the qbit's jobs and starts empty.
|
||||
entries:
|
||||
- key: app:worm_audit;slot:help
|
||||
summary: "Tamper-evident audit: a hash chain over every audit record, signed checkpoints, and sealed exports."
|
||||
content: |
|
||||
Install the database script (the Setup warnings on the home list what is missing) and the one settings row is created with it. Everything else fills by itself once the first audit record has settled: the chain job links records every 30 seconds by default, checkpoints follow, the daily job verifies and exports. Use Verify audit chain to check at any time.
|
||||
|
||||
- key: table:worm_audit_settings;slot:help
|
||||
summary: "The one settings row: retention, export lock, checkpoint and chain cadence, and the job schedules."
|
||||
content: |
|
||||
**How rows get created:** the database script inserts the one row (ID 1) with the host's configured values, only when it is absent; it can never be added to or deleted. Edit record 1 to change a value; changes are audited and take effect within 30 seconds, except the schedules and chain interval, which need a restart. If the row is missing the qbit runs on the host's configured values, so nothing stops, but run the script to make them visible and editable here. Audit retention here is the next purge date only; the purge interval in the database changes through a reviewed migration.
|
||||
|
||||
- key: table:worm_audit_chain;slot:help
|
||||
summary: "One link per core audit record: a hash that includes the previous link, so any change breaks the chain."
|
||||
content: |
|
||||
**How rows get created:** the Chain settled audit records job appends links for audit records older than the settle horizon (every 30 seconds by default, set in Settings). Nobody adds, edits or deletes a link; the database refuses. An empty table means no audit record has settled yet, or the chain job is not running.
|
||||
|
||||
- key: table:worm_audit_chain_detail;slot:help
|
||||
summary: "Which chain link covers each audit detail: one row per detail, written with the link."
|
||||
content: |
|
||||
**How rows get created:** whenever the chain job appends a link, the database writes one row here for every audit detail the link covers, in the same transaction (the database script also fills it for links that existed before this table). It lets the chain job find details that are not chained yet without scanning the chain. Nobody adds, edits or deletes a row; the database refuses, and verification checks that it matches the chain exactly. An empty table means no audit detail has been chained yet.
|
||||
|
||||
- key: table:worm_audit_checkpoint;slot:help
|
||||
summary: "Signed ranges of links (Ed25519), proving the chain's head at a point in time."
|
||||
content: |
|
||||
**How rows get created:** the chain job writes one after the configured number of links or minutes (Settings), signing the range. Sealed exports and verification check against them. An empty table means the chain is too short for a first checkpoint yet.
|
||||
|
||||
- key: table:worm_audit_export;slot:help
|
||||
summary: "Sealed exports of chain ranges to object-lock storage, with their retention and manifest."
|
||||
content: |
|
||||
**How rows get created:** the Daily sealed audit export job (schedule in Settings) and the Export now action seal the links since the last export into the configured bucket, then record it here. Verify sealed export re-reads one. An empty table means no export has run: check the bucket setup warning on the home.
|
||||
|
||||
- key: table:worm_audit_verification;slot:help
|
||||
summary: "Every run of the chain verification: what was checked and whether it passed or broke."
|
||||
content: |
|
||||
**How rows get created:** the Verify audit chain action and the Daily audit chain verification job (schedule in Settings) each add a row, with the first break found, if any. A break also raises an alert. An empty table means verification has not run yet.
|
||||
|
||||
- key: table:worm_audit_purge;slot:help
|
||||
summary: "Record of each retention purge: the cutoff, what was removed and the new anchor link."
|
||||
content: |
|
||||
**How rows get created:** only the database's owner role can purge, by calling the purge function after the retention date has passed; the function writes this row. The application can never delete audit rows. An empty table means nothing has been purged, which is normal until records pass retention.
|
||||
@@ -0,0 +1,207 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.KeyPairGenerator;
|
||||
import java.time.Instant;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.wormaudit.canonical.AuditCanonical;
|
||||
import com.kingsrook.qbits.wormaudit.canonical.Jcs;
|
||||
import com.kingsrook.qbits.wormaudit.chain.Hashes;
|
||||
import com.kingsrook.qbits.wormaudit.checkpoint.Checkpoints;
|
||||
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
|
||||
import com.kingsrook.qbits.wormaudit.guard.AppendOnlyGuard;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for canonical and crypto test.
|
||||
*******************************************************************************/
|
||||
class CanonicalAndCryptoTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Verify jcs ordering escaping and invalid types.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void jcsOrderingEscapingAndInvalidTypes()
|
||||
{
|
||||
var value = new LinkedHashMap<String, Object>();
|
||||
value.put("\u20ac", "Euro");
|
||||
value.put("\ud83d\ude00", "Emoji");
|
||||
value.put("\ufb33", "Hebrew");
|
||||
value.put("a", null);
|
||||
String result = Jcs.string(value);
|
||||
assertThat(result)
|
||||
.isEqualTo(
|
||||
"{\"a\":null,\"\u20ac\":\"Euro\",\"\ud83d\ude00\":\"Emoji\",\"\ufb33\":\"Hebrew\"}");
|
||||
assertThat(Jcs.string("\"\\\b\f\n\r\t\u0001/é"))
|
||||
.isEqualTo("\"\\\"\\\\\\b\\f\\n\\r\\t\\u0001/é\"");
|
||||
assertThat(Jcs.string(List.of(Long.MIN_VALUE, Long.MAX_VALUE)))
|
||||
.isEqualTo("[-9223372036854775808,9223372036854775807]");
|
||||
assertThatThrownBy(() -> Jcs.string(1.5)).isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatThrownBy(() -> Jcs.string("\ud800")).isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatThrownBy(() -> Jcs.string("\udc00")).isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatThrownBy(() -> Jcs.string("\ud800x"))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify exact canonical document.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void exactCanonicalDocument()
|
||||
{
|
||||
var audit =
|
||||
new QRecord()
|
||||
.withValue("id", 4211L)
|
||||
.withValue("recordId", 17)
|
||||
.withValue("message", "Record was Edited")
|
||||
.withValue("timestamp", Instant.parse("2026-10-13T21:04:05.123456Z"));
|
||||
var detail =
|
||||
new QRecord()
|
||||
.withValue("id", 9001L)
|
||||
.withValue("fieldName", "title")
|
||||
.withValue("message", "Changed Title")
|
||||
.withValue("oldValue", "hmac:v1:test:old")
|
||||
.withValue("newValue", "hmac:v1:test:new");
|
||||
assertThat(
|
||||
new String(
|
||||
AuditCanonical.bytes(audit, "cms_page", "james", List.of(detail)),
|
||||
StandardCharsets.UTF_8))
|
||||
.isEqualTo(
|
||||
"{\"audit\":{\"clientChannel\":null,\"id\":4211,\"message\":\"Record was"
|
||||
+ " Edited\",\"recordId\":\"17\",\"table\":\"cms_page\",\"timestamp\":\"2026-10-13T21:04:05.123456Z\",\"user\":\"james\"},\"details\":[{\"fieldName\":\"title\",\"id\":9001,\"message\":\"Changed"
|
||||
+ " Title\",\"newValue\":\"hmac:v1:test:new\",\"oldValue\":\"hmac:v1:test:old\"}],\"v\":1}");
|
||||
assertThat(
|
||||
new String(
|
||||
AuditCanonical.bytes(
|
||||
audit,
|
||||
null,
|
||||
null,
|
||||
List.of(
|
||||
new QRecord().withValue("id", 2L),
|
||||
new QRecord().withValue("id", 1L))),
|
||||
StandardCharsets.UTF_8))
|
||||
.containsSubsequence("\"id\":1,", "\"id\":2,");
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify independent hash and merkle vectors.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void independentHashAndMerkleVectors() throws Exception
|
||||
{
|
||||
assertThat(Hashes.link(Hashes.GENESIS, "{}".getBytes(StandardCharsets.UTF_8)))
|
||||
.isEqualTo("72f1d7c888d85ce3159bd8bbf38f3046725d3d4d6cef2b9449d9fb4b58cf7a9c");
|
||||
String a = "11".repeat(32), b = "22".repeat(32), c = "33".repeat(32);
|
||||
assertThat(Hashes.merkle(List.of(a))).isEqualTo(a);
|
||||
assertThat(Hashes.merkle(List.of(a, b, c)))
|
||||
.isEqualTo("2470b93c7954a79f41c0106caeeeae350559c2af06bd9a9ab203b163951f64f7");
|
||||
assertThatThrownBy(() -> Hashes.merkle(List.of()))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify checkpoint pinning and signing.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void checkpointPinningAndSigning() throws Exception
|
||||
{
|
||||
var keys = KeyPairGenerator.getInstance("Ed25519").generateKeyPair();
|
||||
var other = KeyPairGenerator.getInstance("Ed25519").generateKeyPair();
|
||||
var config =
|
||||
new WormAuditQBitConfig()
|
||||
.withSigningKey(
|
||||
() -> new WormAuditQBitConfig.SigningKey("a", keys.getPrivate()))
|
||||
.withTrustedPublicKeys(Map.of("a", keys.getPublic()));
|
||||
var links =
|
||||
List.of(new QRecord().withValue("id", 1L).withValue("link_hash", "11".repeat(32)));
|
||||
var checkpoint = Checkpoints.sign(links, config, Instant.now());
|
||||
assertThat(Checkpoints.verify(checkpoint, keys.getPublic())).isTrue();
|
||||
assertThat(Checkpoints.verify(checkpoint, other.getPublic())).isFalse();
|
||||
assertThat(Checkpoints.verify(checkpoint, null)).isFalse();
|
||||
checkpoint.setValue("signature", "bad");
|
||||
assertThat(Checkpoints.verify(checkpoint, keys.getPublic())).isFalse();
|
||||
checkpoint.setValue("algorithm", "RSA");
|
||||
assertThat(Checkpoints.verify(checkpoint, keys.getPublic())).isFalse();
|
||||
config.withTrustedPublicKeys(Map.of());
|
||||
assertThatThrownBy(() -> Checkpoints.sign(links, config, Instant.now()))
|
||||
.hasMessageContaining("not pinned");
|
||||
config.withTrustedPublicKeys(Map.of("a", other.getPublic()));
|
||||
assertThatThrownBy(() -> Checkpoints.sign(links, config, Instant.now()))
|
||||
.hasMessageContaining("does not match");
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify ddl is reviewed rendering and rejects injection.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void ddlIsReviewedRenderingAndRejectsInjection() throws Exception
|
||||
{
|
||||
String sql = WormAuditDdl.postgres(new WormAuditQBitConfig());
|
||||
assertThat(sql)
|
||||
.isEqualTo(
|
||||
java.nio.file.Files.readString(
|
||||
java.nio.file.Path.of("docs/ddl/worm-audit-postgres.sql")));
|
||||
assertThat(sql).doesNotContain("${");
|
||||
// Liquibase parses every line starting with "-- changeset" as a directive, comments included.
|
||||
assertThat(sql.lines().filter(line -> line.startsWith("-- changeset")))
|
||||
.allMatch(line -> line.matches("-- changeset worm-audit:[0-9]+( splitStatements:false)?"));
|
||||
assertThat(sql).contains("GET DIAGNOSTICS stack = PG_CONTEXT", "function worm_audit_purge(timestamp with time zone)", "SET timezone = 'UTC'");
|
||||
// The documented migration contract deliberately uses physical snake_case columns.
|
||||
assertThat(sql).contains("OLD.audit_id", "OLD.timestamp", "NEW.name", "GRANT UPDATE (label, create_date, modify_date)");
|
||||
assertThat(java.nio.file.Files.readString(java.nio.file.Path.of("README.md")))
|
||||
.contains("renders table names only", "audit_detail.audit_id", "audit_user.id/name/create_date/modify_date");
|
||||
assertThat(
|
||||
WormAuditDdl.postgres(
|
||||
new WormAuditQBitConfig()
|
||||
.withRetentionYears(7)
|
||||
.withAppRole("runtime")
|
||||
.withCoreAuditTableNames(Map.of("audit", "history.audit"))))
|
||||
.contains("\"history\".\"audit\"", "7 years", "TO \"runtime\"");
|
||||
assertThatThrownBy(() -> WormAuditDdl.identifier("a; DROP TABLE audit"))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
WormAuditDdl.postgres(
|
||||
new WormAuditQBitConfig().withRetentionYears(0)))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatThrownBy(
|
||||
() -> WormAuditDdl.postgres(new WormAuditQBitConfig().withAppRole("a.b")))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
var sqlError = new java.sql.SQLException("worm_audit: rejected", "P0001");
|
||||
assertThat(AppendOnlyGuard.map(new Exception(sqlError)))
|
||||
.hasMessage(AppendOnlyGuard.MESSAGE);
|
||||
assertThat(
|
||||
AppendOnlyGuard.map(
|
||||
new java.sql.SQLException("ERROR: worm_audit: rejected", "P0001")))
|
||||
.hasMessage(AppendOnlyGuard.MESSAGE);
|
||||
assertThat(AppendOnlyGuard.map(new java.sql.SQLException("other", "P0001")))
|
||||
.hasMessage("WORM audit operation failed");
|
||||
var original = new com.kingsrook.qqq.backend.core.exceptions.QException("original");
|
||||
assertThat(AppendOnlyGuard.map(original)).isSameAs(original);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,347 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import java.security.KeyPairGenerator;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.chain.Hashes;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.OLD;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.STORE;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.core;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.detail;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.remove;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.tamper;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for chain verification test.
|
||||
*******************************************************************************/
|
||||
class ChainVerificationTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Verify signed.
|
||||
*******************************************************************************/
|
||||
static WormAuditQBitConfig signed() throws Exception
|
||||
{
|
||||
var pair = KeyPairGenerator.getInstance("Ed25519").generateKeyPair();
|
||||
return (new WormAuditQBitConfig()
|
||||
.withCheckpointEveryLinks(1)
|
||||
.withSigningKey(() -> new WormAuditQBitConfig.SigningKey("test", pair.getPrivate()))
|
||||
.withTrustedPublicKeys(Map.of("test", pair.getPublic())));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify late headers details and chunk boundaries.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void lateHeadersDetailsAndChunkBoundaries() throws Exception
|
||||
{
|
||||
var config = signed().withBatchSize(1);
|
||||
init(config);
|
||||
audit(20);
|
||||
detail(1, 20);
|
||||
var writer = new WormAuditChainer(config);
|
||||
assertThat(writer.chainOnce().audits()).isEqualTo(1);
|
||||
audit(1);
|
||||
detail(2, 1);
|
||||
detail(3, 20);
|
||||
assertThat(writer.chainOnce().lateDetails()).isEqualTo(1);
|
||||
for(long i = 4; i <= 260; i++)
|
||||
{
|
||||
detail(i, 1);
|
||||
}
|
||||
assertThat(writer.chainOnce().lateDetails()).isEqualTo(257);
|
||||
var links = STORE.all(WormAuditTables.CHAIN);
|
||||
assertThat(links).extracting(r -> r.getValueLong("id")).containsExactly(1L, 2L, 3L, 4L, 5L);
|
||||
assertThat(links).allMatch(r -> AuditStore.detailIds(r).size() <= 250);
|
||||
var result = new WormAuditVerifier(config).verify();
|
||||
assertThat(result.getBreaks()).isEmpty();
|
||||
assertThat(result.getLinksChecked()).isEqualTo(5);
|
||||
assertThat(result.getFromLinkId()).isEqualTo(1);
|
||||
assertThat(result.getToLinkId()).isEqualTo(5);
|
||||
assertThat(writer.chainOnce().links()).isZero();
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify overflow at initial chain and settling and elapsed checkpoint.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void overflowAtInitialChainAndSettlingAndElapsedCheckpoint() throws Exception
|
||||
{
|
||||
var config = signed().withCheckpointEveryLinks(10000).withCheckpointEveryMinutes(1);
|
||||
init(config);
|
||||
audit(1);
|
||||
for(long i = 1; i <= 251; i++)
|
||||
{
|
||||
detail(i, 1);
|
||||
}
|
||||
var writer = new WormAuditChainer(config);
|
||||
assertThat(writer.chainOnce(OLD.plusSeconds(30)).links()).isZero();
|
||||
Instant now = Instant.now();
|
||||
assertThat(writer.chainOnce(now).links()).isEqualTo(2);
|
||||
assertThat(STORE.all(WormAuditTables.CHECKPOINT)).isEmpty();
|
||||
assertThat(writer.chainOnce(now.plusSeconds(61)).checkpoints()).isEqualTo(1);
|
||||
assertThat(new WormAuditVerifier(config).verify().passed()).isTrue();
|
||||
config.withSigningKey(null);
|
||||
assertThat(new WormAuditVerifier(config).verify().getWarnings())
|
||||
.contains("no signing key configured");
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify detects breaks.
|
||||
*******************************************************************************/
|
||||
@ParameterizedTest
|
||||
@ValueSource(
|
||||
strings =
|
||||
{
|
||||
"content",
|
||||
"headerDeleted",
|
||||
"detailDeleted",
|
||||
"inserted",
|
||||
"detailInserted",
|
||||
"gap",
|
||||
"previous",
|
||||
"reorder",
|
||||
"forged",
|
||||
"untrusted",
|
||||
"merkle",
|
||||
"head",
|
||||
"duplicate",
|
||||
"kind",
|
||||
"lookup",
|
||||
"range",
|
||||
"nullCheckpoint",
|
||||
"nullPrevious",
|
||||
"nullTimestamp"
|
||||
})
|
||||
void detectsBreaks(String scenario) throws Exception
|
||||
{
|
||||
var config = signed();
|
||||
init(config);
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
audit(2);
|
||||
detail(2, 2);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
String expected;
|
||||
switch(scenario)
|
||||
{
|
||||
case "content" ->
|
||||
{
|
||||
tamper("auditDetail", 1, "newValue", "tampered");
|
||||
expected =
|
||||
"Break at link 1 (audit 1, table cms_page, record 17): content differs from"
|
||||
+ " the chained hash (audit_detail 1)";
|
||||
}
|
||||
case "headerDeleted" ->
|
||||
{
|
||||
remove("audit", 1);
|
||||
expected = "audit header is missing";
|
||||
}
|
||||
case "detailDeleted" ->
|
||||
{
|
||||
remove("auditDetail", 1);
|
||||
expected = "audit_detail 1 is missing";
|
||||
}
|
||||
case "inserted" ->
|
||||
{
|
||||
audit(3);
|
||||
expected = "audit 3 is not covered";
|
||||
}
|
||||
case "detailInserted" ->
|
||||
{
|
||||
detail(3, 1);
|
||||
expected = "audit_detail 3 is not covered";
|
||||
}
|
||||
case "gap" ->
|
||||
{
|
||||
remove(WormAuditTables.CHAIN, 1);
|
||||
expected = "gap";
|
||||
}
|
||||
case "previous" ->
|
||||
{
|
||||
tamper(WormAuditTables.CHAIN, 2, "prev_hash", Hashes.GENESIS);
|
||||
expected = "prev-hash discontinuity";
|
||||
}
|
||||
case "reorder" ->
|
||||
{
|
||||
tamper(WormAuditTables.CHAIN, 1, "audit_id", 2L);
|
||||
expected = "another audit";
|
||||
}
|
||||
case "forged" ->
|
||||
{
|
||||
tamper(WormAuditTables.CHECKPOINT, 1, "signature", "AA==");
|
||||
expected = "checkpoint 1: signature does not verify";
|
||||
}
|
||||
case "untrusted" ->
|
||||
{
|
||||
config.withTrustedPublicKeys(Map.of());
|
||||
expected = "untrusted key";
|
||||
}
|
||||
case "merkle" ->
|
||||
{
|
||||
tamper(WormAuditTables.CHECKPOINT, 1, "merkle_root", Hashes.GENESIS);
|
||||
expected = "merkle mismatch";
|
||||
}
|
||||
case "head" ->
|
||||
{
|
||||
tamper(WormAuditTables.CHECKPOINT, 1, "head_hash", Hashes.GENESIS);
|
||||
expected = "head mismatch";
|
||||
}
|
||||
case "duplicate" ->
|
||||
{
|
||||
tamper(WormAuditTables.CHAIN, 1, "detail_ids", "1,1");
|
||||
expected = "audit_detail 1 is covered twice";
|
||||
}
|
||||
case "kind" ->
|
||||
{
|
||||
tamper(WormAuditTables.CHAIN, 1, "kind", "INVALID");
|
||||
expected = "invalid kind";
|
||||
}
|
||||
case "lookup" ->
|
||||
{
|
||||
tamper("auditTable", 1, "name", "renamed");
|
||||
expected = "content differs";
|
||||
}
|
||||
case "nullCheckpoint" ->
|
||||
{
|
||||
tamper(WormAuditTables.CHECKPOINT, 1, "signature", null);
|
||||
expected = "malformed checkpoint";
|
||||
}
|
||||
case "nullPrevious" ->
|
||||
{
|
||||
tamper(WormAuditTables.CHAIN, 1, "prev_hash", null);
|
||||
expected = "malformed link";
|
||||
}
|
||||
case "nullTimestamp" ->
|
||||
{
|
||||
tamper("audit", 1, "timestamp", null);
|
||||
expected = "missing audit timestamp";
|
||||
}
|
||||
default ->
|
||||
{
|
||||
tamper(WormAuditTables.CHECKPOINT, 1, "link_count", 22L);
|
||||
expected = "invalid link range";
|
||||
}
|
||||
}
|
||||
var result = new WormAuditVerifier(config).verify();
|
||||
assertThat(result.passed()).isFalse();
|
||||
assertThat(result.getBreaks()).anyMatch(b -> b.reason().contains(expected));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify selection does not run completeness and checks boundary.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void selectionDoesNotRunCompletenessAndChecksBoundary() throws Exception
|
||||
{
|
||||
var config = signed();
|
||||
init(config);
|
||||
audit(1);
|
||||
audit(2);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
audit(3);
|
||||
assertThat(new WormAuditVerifier(config).verify(Set.of(2L)).passed()).isTrue();
|
||||
assertThat(new WormAuditVerifier(config).verify(Set.of()).getLinksChecked()).isZero();
|
||||
assertThat(new WormAuditVerifier(config).verify(Set.of(99L)).getBreaks())
|
||||
.anyMatch(b -> b.reason().contains("Missing selected link 99"));
|
||||
tamper(WormAuditTables.CHAIN, 2, "prev_hash", Hashes.GENESIS);
|
||||
assertThat(new WormAuditVerifier(config).verify(Set.of(2L)).getBreaks())
|
||||
.anyMatch(b -> b.reason().contains("prev-hash"));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify continues after purge anchor including empty live chain.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void continuesAfterPurgeAnchorIncludingEmptyLiveChain() throws Exception
|
||||
{
|
||||
var config = signed();
|
||||
init(config);
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
audit(2);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
var first = STORE.get(WormAuditTables.CHAIN, 1);
|
||||
STORE.insert(
|
||||
WormAuditTables.PURGE,
|
||||
List.of(
|
||||
new com.kingsrook.qqq.backend.core.model.data.QRecord()
|
||||
.withValue("outcome", "PURGED")
|
||||
.withValue("anchor_link_id", 1L)
|
||||
.withValue("anchor_hash", first.getValueString("link_hash"))));
|
||||
remove("auditDetail", 1);
|
||||
remove("audit", 1);
|
||||
remove(WormAuditTables.CHAIN, 1);
|
||||
assertThat(new WormAuditVerifier(config).verify().passed()).isTrue();
|
||||
var last = STORE.get(WormAuditTables.CHAIN, 2);
|
||||
STORE.insert(
|
||||
WormAuditTables.PURGE,
|
||||
List.of(
|
||||
new com.kingsrook.qqq.backend.core.model.data.QRecord()
|
||||
.withValue("outcome", "PURGED")
|
||||
.withValue("anchor_link_id", 2L)
|
||||
.withValue("anchor_hash", last.getValueString("link_hash"))));
|
||||
remove("audit", 2);
|
||||
remove(WormAuditTables.CHAIN, 2);
|
||||
assertThat(new WormAuditVerifier(config).verify().passed()).isTrue();
|
||||
audit(3);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
assertThat(STORE.all(WormAuditTables.CHAIN))
|
||||
.extracting(r -> r.getValueLong("id"))
|
||||
.containsExactly(3L);
|
||||
assertThat(new WormAuditVerifier(config).verify().passed()).isTrue();
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Findings retain bounded diagnostics while recording the complete failure count.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void boundedDiagnosticsPreserveTotalCount() throws Exception
|
||||
{
|
||||
init(new WormAuditQBitConfig());
|
||||
var result = new com.kingsrook.qbits.wormaudit.verify.VerificationResult();
|
||||
for(long id = 1; id <= 1200; id++)
|
||||
{
|
||||
result.add(id, id, "gap");
|
||||
result.warn("warning " + id);
|
||||
}
|
||||
result.omitted(20);
|
||||
assertThat(result.getBreaks()).hasSize(1000);
|
||||
assertThat(result.getWarnings()).hasSize(100);
|
||||
assertThat(result.getBreakCount()).isEqualTo(1220);
|
||||
assertThat(result.passed()).isFalse();
|
||||
assertThat(com.kingsrook.qbits.wormaudit.processes.VerificationRuns.save(result, "USER").getValueLong("break_count")).isEqualTo(1220);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for docker condition test.
|
||||
*******************************************************************************/
|
||||
class DockerConditionTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Verify never skips in ci.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void neverSkipsInCi()
|
||||
{
|
||||
assertThat(RequiresDockerCondition.evaluate(false, null).isDisabled()).isTrue();
|
||||
assertThat(RequiresDockerCondition.evaluate(true, "true").isDisabled()).isFalse();
|
||||
assertThatThrownBy(() -> RequiresDockerCondition.evaluate(false, "false"))
|
||||
.hasMessageContaining("CI is set");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import java.net.URI;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.objectstorage.ObjectStorageQBitConfig;
|
||||
import com.kingsrook.qbits.objectstorage.api.Capability;
|
||||
import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
|
||||
import com.kingsrook.qbits.objectstorage.api.ListRequest;
|
||||
import com.kingsrook.qbits.objectstorage.api.ObjectStorage;
|
||||
import com.kingsrook.qbits.objectstorage.config.AddressingStyle;
|
||||
import com.kingsrook.qbits.objectstorage.config.BucketRole;
|
||||
import com.kingsrook.qbits.objectstorage.config.ChecksumMode;
|
||||
import com.kingsrook.qbits.objectstorage.config.ProviderType;
|
||||
import com.kingsrook.qbits.objectstorage.config.StorageBucketConfig;
|
||||
import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig;
|
||||
import com.kingsrook.qbits.objectstorage.internal.CapabilityProbe;
|
||||
import com.kingsrook.qbits.objectstorage.internal.CapabilityStore;
|
||||
import com.kingsrook.qbits.objectstorage.internal.ObjectStorageRuntime;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.export.ExportFiles;
|
||||
import com.kingsrook.qbits.wormaudit.export.ObjectStorageSealedStore;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedExportVerifier;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedExporter;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.ExportProcesses;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.testcontainers.containers.GenericContainer;
|
||||
import org.testcontainers.containers.wait.strategy.Wait;
|
||||
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
|
||||
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
||||
import software.amazon.awssdk.core.checksums.RequestChecksumCalculation;
|
||||
import software.amazon.awssdk.core.checksums.ResponseChecksumValidation;
|
||||
import software.amazon.awssdk.core.sync.RequestBody;
|
||||
import software.amazon.awssdk.regions.Region;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
import software.amazon.awssdk.services.s3.model.S3Exception;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.STORE;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.detail;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.tamper;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Real Object Lock acceptance. Missing Docker skips locally and fails in CI.
|
||||
*******************************************************************************/
|
||||
@ExtendWith(RequiresDockerCondition.class)
|
||||
class MinioExportAcceptanceTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Production adapter, GOVERNANCE and COMPLIANCE retention, version preservation and UI findings.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void lockedRoundTripVersionRetentionLiveTamperAndUnsupportedBucket() throws Exception
|
||||
{
|
||||
String image = System.getProperty("qqq.test.minioImage",
|
||||
"cgr.dev/chainguard/minio@sha256:dc455cdd4db6e390b1b26514f15ff37681a777bf8dab41aac445a0259f751e56");
|
||||
try(GenericContainer<?> minio = new GenericContainer<>(image)
|
||||
.withEnv("MINIO_ROOT_USER", "worm-test-access").withEnv("MINIO_ROOT_PASSWORD", "worm-test-secret")
|
||||
.withCommand("server", "/tmp/minio-data").withExposedPorts(9000)
|
||||
.waitingFor(Wait.forHttp("/minio/health/live").forPort(9000).withStartupTimeout(Duration.ofMinutes(2))))
|
||||
{
|
||||
minio.start();
|
||||
String endpoint = "http://" + minio.getHost() + ":" + minio.getMappedPort(9000);
|
||||
try(S3Client s3 = S3Client.builder().endpointOverride(URI.create(endpoint)).region(Region.US_EAST_1).forcePathStyle(true)
|
||||
.credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("worm-test-access", "worm-test-secret")))
|
||||
.requestChecksumCalculation(RequestChecksumCalculation.WHEN_REQUIRED)
|
||||
.responseChecksumValidation(ResponseChecksumValidation.WHEN_REQUIRED).build())
|
||||
{
|
||||
s3.createBucket(b -> b.bucket("worm-locked").objectLockEnabledForBucket(true));
|
||||
s3.createBucket(b -> b.bucket("worm-plain"));
|
||||
var config = ChainVerificationTest.signed();
|
||||
var instance = init(config);
|
||||
AlertsInstanceMetaData.ofOrWithNew(instance).withSink(new QCodeReference(ProcessesAndAlertsTest.Sink.class));
|
||||
ProcessesAndAlertsTest.Sink.raised.clear();
|
||||
System.setProperty("worm.test.access", "worm-test-access");
|
||||
System.setProperty("worm.test.secret", "worm-test-secret");
|
||||
ObjectStorage.configureStandalone(new ObjectStorageQBitConfig().withBackendName("memory")
|
||||
.withAllowedSecretRefPatterns(List.of("prop.worm.test.*"))
|
||||
.withConnection(new StorageConnectionConfig("minio").withProviderType(ProviderType.MINIO).withEndpoint(endpoint)
|
||||
.withRegion("us-east-1").withAddressingStyle(AddressingStyle.PATH).withChecksumMode(ChecksumMode.WHEN_REQUIRED)
|
||||
.withAccessKeyRef("${prop.worm.test.access}").withSecretKeyRef("${prop.worm.test.secret}"))
|
||||
.withBucket(new StorageBucketConfig("worm_audit").withConnection("minio").withBucketName("worm-locked").withRole(BucketRole.BACKUP).withIndexed(false))
|
||||
.withBucket(new StorageBucketConfig("plain").withConnection("minio").withBucketName("worm-plain").withRole(BucketRole.BACKUP).withIndexed(false)));
|
||||
CapabilityProbe.probeConnection("minio", false);
|
||||
assertThat(ObjectStorage.asSystem("test", "1").bucket("worm_audit").capability(Capability.OBJECT_LOCK)).isEqualTo(CapabilityStatus.SUPPORTED);
|
||||
assertThat(ObjectStorage.asSystem("test", "1").bucket("plain").capability(Capability.OBJECT_LOCK)).isEqualTo(CapabilityStatus.UNSUPPORTED);
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
Instant now = Instant.now();
|
||||
var exported = new SealedExporter(config).run("USER", now);
|
||||
assertThat(exported.failed()).as(exported.message()).isFalse();
|
||||
var row = exported.row();
|
||||
var storage = new ObjectStorageSealedStore("worm_audit", "acceptance");
|
||||
assertThat(storage.list(ListRequest.prefix(row.getValueString("object_prefix"))).getItems()).hasSize(3);
|
||||
var verifier = new SealedExportVerifier(config);
|
||||
assertThat(verifier.verify(row, "USER").result().passed()).isTrue();
|
||||
for(String name : ExportFiles.NAMES)
|
||||
{
|
||||
String key = row.getValueString("object_prefix") + name;
|
||||
var head = storage.head(key);
|
||||
assertThat(head.getLockMode()).isEqualTo("GOVERNANCE");
|
||||
assertThat(Duration.between(now.plus(Duration.ofDays(30)), head.getLockRetainUntil()).abs()).isLessThan(Duration.ofSeconds(2));
|
||||
String originalVersion = head.getVersionId();
|
||||
assertThat(originalVersion).isNotBlank();
|
||||
assertThat(ExportFiles.parse(row.getValueString("object_versions").getBytes(java.nio.charset.StandardCharsets.UTF_8)))
|
||||
.containsEntry(name, originalVersion);
|
||||
assertThatThrownBy(() -> s3.deleteObject(b -> b.bucket("worm-locked").key(key).versionId(originalVersion).bypassGovernanceRetention(false)))
|
||||
.isInstanceOf(S3Exception.class);
|
||||
byte[] original = s3.getObjectAsBytes(b -> b.bucket("worm-locked").key(key).versionId(originalVersion)).asByteArray();
|
||||
// S3 overwrites create new versions; they cannot replace the locked version.
|
||||
var replacement = s3.putObject(b -> b.bucket("worm-locked").key(key).objectLockMode("GOVERNANCE")
|
||||
.objectLockRetainUntilDate(row.getValueInstant("retain_until")), RequestBody.fromString("different replacement bytes"));
|
||||
assertThat(replacement.versionId()).isNotEqualTo(originalVersion);
|
||||
assertThat(s3.getObjectAsBytes(b -> b.bucket("worm-locked").key(key).versionId(originalVersion)).asByteArray()).isEqualTo(original);
|
||||
try(var sealed = storage.getVersion(key, originalVersion))
|
||||
{
|
||||
assertThat(sealed.readAllBytes()).isEqualTo(original);
|
||||
}
|
||||
var report = verifier.verify(row, "USER");
|
||||
assertThat(report.result().passed()).isFalse();
|
||||
assertThat(report.result().getLinksChecked()).isEqualTo(1);
|
||||
assertThat(report.result().getBreaks()).allSatisfy(b -> assertThat(b.reason()).contains("was overwritten after sealing"));
|
||||
assertThat(report.result().getBreaks().toString()).contains("object " + name + " was overwritten after sealing (current version "
|
||||
+ replacement.versionId() + ", sealed version " + originalVersion + " is still retained under Object Lock)");
|
||||
}
|
||||
tamper("auditDetail", 1, "newValue", "tampered live value");
|
||||
var load = new ExportProcesses.VerifyLoad();
|
||||
load.runOnePage(new RunBackendStepInput().withRecords(List.of(row)), new RunBackendStepOutput());
|
||||
assertThat(load.getProcessSummary(new RunBackendStepOutput(), true).toString())
|
||||
.contains("ERROR", "link 1 (audit 1): live content differs from the sealed copy");
|
||||
tamper("auditDetail", 1, "newValue", "new");
|
||||
audit(2);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
config.withExportLockMode("COMPLIANCE");
|
||||
var compliance = new SealedExporter(config).run("USER", Instant.now());
|
||||
assertThat(compliance.failed()).isFalse();
|
||||
assertThat(compliance.row().getValueString("lock_mode")).isEqualTo("COMPLIANCE");
|
||||
assertThat(verifier.verify(compliance.row(), "USER").result().passed()).isTrue();
|
||||
for(String name : ExportFiles.NAMES)
|
||||
{
|
||||
String key = compliance.row().getValueString("object_prefix") + name;
|
||||
var head = storage.head(key);
|
||||
assertThat(head.getLockMode()).isEqualTo("COMPLIANCE");
|
||||
assertThatThrownBy(() -> s3.deleteObject(b -> b.bucket("worm-locked").key(key).versionId(head.getVersionId())))
|
||||
.isInstanceOf(S3Exception.class);
|
||||
}
|
||||
audit(3);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
config.withExportBucketAlias("plain");
|
||||
var failed = new SealedExporter(config).run("SCHEDULE", Instant.now());
|
||||
assertThat(failed.failed()).isTrue();
|
||||
assertThat(failed.message()).contains("does not support Object Lock");
|
||||
assertThat(STORE.all(WormAuditTables.EXPORT)).hasSize(2);
|
||||
assertThat(ProcessesAndAlertsTest.Sink.raised).hasSize(1);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ObjectStorageRuntime.reset();
|
||||
CapabilityStore.clearCache();
|
||||
System.clearProperty("worm.test.access");
|
||||
System.clearProperty("worm.test.secret");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import javax.crypto.Mac;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Base64;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.UUID;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.actions.audits.AuditAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.audits.AuditActionCustomizerInterface;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QSupplementalFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.OLD;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.STORE;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.core;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.detail;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for phi canary test.
|
||||
*******************************************************************************/
|
||||
public class PhiCanaryTest
|
||||
{
|
||||
// RC.5 has AiFieldMetaData but not DataClassification; use a test-local marker.
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for phi.
|
||||
*******************************************************************************/
|
||||
public static class Phi extends QSupplementalFieldMetaData
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Verify get type.
|
||||
*******************************************************************************/
|
||||
public String getType()
|
||||
{
|
||||
return ("test-phi");
|
||||
}
|
||||
}
|
||||
|
||||
// Real RC.5 AuditAction customization point: rewrite details BEFORE insertion.
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for token hook.
|
||||
*******************************************************************************/
|
||||
public static class TokenHook implements AuditActionCustomizerInterface
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Verify customize input.
|
||||
*******************************************************************************/
|
||||
public void customizeInput(AuditSingleInput input)
|
||||
{
|
||||
for(QRecord detail : input.getDetails())
|
||||
{
|
||||
var field =
|
||||
QContext.getQInstance()
|
||||
.getTable(input.getAuditTableName())
|
||||
.getField(detail.getValueString("fieldName"));
|
||||
if(field.getSupplementalMetaData("test-phi") != null)
|
||||
{
|
||||
for(String column : List.of("oldValue", "newValue"))
|
||||
{
|
||||
if(detail.getValueString(column) == null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
Mac mac = Mac.getInstance("HmacSHA256");
|
||||
mac.init(
|
||||
new SecretKeySpec(
|
||||
"fixture-key-not-production"
|
||||
.getBytes(StandardCharsets.UTF_8),
|
||||
"HmacSHA256"));
|
||||
detail.setValue(
|
||||
column,
|
||||
"hmac:v1:test:"
|
||||
+ Base64.getUrlEncoder()
|
||||
.withoutPadding()
|
||||
.encodeToString(
|
||||
mac.doFinal(
|
||||
detail.getValueString(column)
|
||||
.getBytes(
|
||||
StandardCharsets
|
||||
.UTF_8))));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
}
|
||||
detail.setValue("message", "Changed Diagnosis");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify token hook keeps every audit and chain value free of canary.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void tokenHookKeepsEveryAuditAndChainValueFreeOfCanary() throws Exception
|
||||
{
|
||||
var config = ChainVerificationTest.signed();
|
||||
var instance = init(config);
|
||||
instance.addTable(
|
||||
new QTableMetaData()
|
||||
.withName("patient")
|
||||
.withBackendName("memory")
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
|
||||
.withField(
|
||||
new QFieldMetaData("diagnosis", QFieldType.STRING)
|
||||
.withSupplementalMetaData(new Phi())));
|
||||
instance.addSupplementalCustomizer(
|
||||
AuditActionCustomizerInterface.CUSTOMIZER_TYPE,
|
||||
new QCodeReference(TokenHook.class));
|
||||
String canary = "CANARY-PHI-" + UUID.randomUUID();
|
||||
var detail =
|
||||
new QRecord()
|
||||
.withValue("fieldName", "diagnosis")
|
||||
.withValue("oldValue", canary)
|
||||
.withValue("newValue", canary)
|
||||
.withValue("message", "Changed Diagnosis from " + canary + " to " + canary);
|
||||
new AuditAction()
|
||||
.execute(
|
||||
new AuditInput()
|
||||
.withAuditSingleInput(
|
||||
new AuditSingleInput()
|
||||
.withAuditTableName("patient")
|
||||
.withAuditUserName("james")
|
||||
.withRecordId(17)
|
||||
.withTimestamp(OLD)
|
||||
.withMessage("Record was Edited")
|
||||
.withDetails(List.of(detail))));
|
||||
assertThat(STORE.all("audit")).hasSize(1);
|
||||
assertThat(STORE.all("auditDetail")).hasSize(1);
|
||||
assertThat(STORE.all("auditDetail").getFirst().getValueString("newValue"))
|
||||
.startsWith("hmac:v1:test:");
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
List<String> values = new ArrayList<>();
|
||||
for(String table :
|
||||
List.of(
|
||||
"audit",
|
||||
"auditDetail",
|
||||
WormAuditTables.CHAIN,
|
||||
WormAuditTables.CHECKPOINT))
|
||||
{
|
||||
STORE.all(table)
|
||||
.forEach(
|
||||
r ->
|
||||
r.getValues().values().stream()
|
||||
.filter(Objects::nonNull)
|
||||
.map(Object::toString)
|
||||
.forEach(values::add));
|
||||
}
|
||||
for(var link : STORE.all(WormAuditTables.CHAIN))
|
||||
{
|
||||
values.add(
|
||||
new String(
|
||||
STORE.canonical(
|
||||
STORE.get("audit", link.getValueLong("audit_id")),
|
||||
AuditStore.detailIds(link)),
|
||||
StandardCharsets.UTF_8));
|
||||
}
|
||||
for(String value : values)
|
||||
{
|
||||
assertThat(value).doesNotContain(canary);
|
||||
// Every longer substring contains a 5-character substring.
|
||||
for(int start = 0; start <= canary.length() - 5; start++)
|
||||
{
|
||||
assertThat(value).doesNotContain(canary.substring(start, start + 5));
|
||||
}
|
||||
}
|
||||
assertThat(new WormAuditVerifier(config).verify().passed()).isTrue();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,762 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.testcontainers.containers.PostgreSQLContainer;
|
||||
import org.testcontainers.junit.jupiter.Container;
|
||||
import org.testcontainers.junit.jupiter.Testcontainers;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.STORE;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.core;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.detail;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.lookups;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for postgres acceptance test.
|
||||
*******************************************************************************/
|
||||
@ExtendWith(RequiresDockerCondition.class)
|
||||
@Testcontainers
|
||||
class PostgresAcceptanceTest
|
||||
{
|
||||
@Container
|
||||
static final PostgreSQLContainer<?> POSTGRES = new PostgreSQLContainer<>("postgres:16-alpine");
|
||||
|
||||
WormAuditQBitConfig config;
|
||||
QInstance instance;
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify owner.
|
||||
*******************************************************************************/
|
||||
Connection owner() throws SQLException
|
||||
{
|
||||
return (DriverManager.getConnection(
|
||||
POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword()));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify app.
|
||||
*******************************************************************************/
|
||||
Connection app() throws SQLException
|
||||
{
|
||||
return (DriverManager.getConnection(POSTGRES.getJdbcUrl(), "app", "fixture"));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify sql.
|
||||
*******************************************************************************/
|
||||
void sql(Connection c, String sql) throws SQLException
|
||||
{
|
||||
try(var s = c.createStatement())
|
||||
{
|
||||
s.execute(sql);
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify count.
|
||||
*******************************************************************************/
|
||||
long count(Connection c, String sql) throws SQLException
|
||||
{
|
||||
try(var s = c.createStatement();
|
||||
var r = s.executeQuery(sql))
|
||||
{
|
||||
r.next();
|
||||
return (r.getLong(1));
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify database.
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void database() throws Exception
|
||||
{
|
||||
config = ChainVerificationTest.signed().withSettleHorizonSeconds(0);
|
||||
try(var c = owner())
|
||||
{
|
||||
sql(
|
||||
c,
|
||||
"DROP SCHEMA public CASCADE; CREATE SCHEMA public; GRANT USAGE ON SCHEMA public"
|
||||
+ " TO PUBLIC");
|
||||
sql(
|
||||
c,
|
||||
"DO $$ BEGIN IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='app') THEN"
|
||||
+ " CREATE ROLE app LOGIN PASSWORD 'fixture'; END IF; END $$");
|
||||
instance = RdbmsTestSupport.instance(POSTGRES.getJdbcUrl(), "app", "fixture", config, true);
|
||||
RdbmsTestSupport.createTables(c, instance, true);
|
||||
String ddl = WormAuditDdl.postgres(config);
|
||||
sql(c, ddl);
|
||||
sql(c, ddl);
|
||||
}
|
||||
QContext.init(instance, new QSession());
|
||||
lookups();
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Home setup probes see dropped/disabled guards, missing purge and runtime ownership.
|
||||
** This test uses the same Docker condition as the existing PostgreSQL suite.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void homeSetupWarnings() throws Exception
|
||||
{
|
||||
var storage = new com.kingsrook.qbits.wormaudit.demo.MemorySealedStore();
|
||||
var clean = com.kingsrook.qbits.wormaudit.app.WormAuditHomeData.load(config, storage, java.time.Instant.now());
|
||||
assertThat(clean.warnings()).contains("Daily verification has never run").noneMatch(w -> w.contains("guard") && w.contains("settings"));
|
||||
try(var connection = owner())
|
||||
{
|
||||
sql(connection, "UPDATE worm_audit_settings SET audit_retention_years=7 WHERE id=1");
|
||||
config.invalidateSettings();
|
||||
assertThat(com.kingsrook.qbits.wormaudit.app.WormAuditHomeData.load(config, storage, java.time.Instant.now()).warnings())
|
||||
.anyMatch(w -> w.contains("purge function's compiled interval (6 years)"));
|
||||
sql(connection, WormAuditDdl.postgres(config));
|
||||
assertThat(count(connection, "SELECT audit_retention_years FROM worm_audit_settings WHERE id=1")).isEqualTo(7);
|
||||
sql(connection, "UPDATE worm_audit_settings SET audit_retention_years=6 WHERE id=1");
|
||||
config.invalidateSettings();
|
||||
sql(connection, "DROP TRIGGER worm_audit_row_guard ON audit_detail");
|
||||
sql(connection, "ALTER TABLE worm_audit_chain DISABLE TRIGGER worm_audit_truncate_guard");
|
||||
sql(connection, "DROP FUNCTION public.worm_audit_purge(timestamptz)");
|
||||
}
|
||||
var missing = com.kingsrook.qbits.wormaudit.app.WormAuditHomeData.load(config, storage, java.time.Instant.now());
|
||||
assertThat(missing.warnings()).contains("Missing or disabled worm_audit_row_guard on audit_detail",
|
||||
"Missing or disabled worm_audit_truncate_guard on worm_audit_chain", "Retention purge function is missing");
|
||||
instance = RdbmsTestSupport.instance(POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword(), config, true);
|
||||
QContext.init(instance, new QSession());
|
||||
assertThat(com.kingsrook.qbits.wormaudit.app.WormAuditHomeData.load(config, storage, java.time.Instant.now()).warnings())
|
||||
.contains("Runtime role owns audit table audit", "Runtime role owns audit table worm_audit_purge");
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify grants triggers lookup rules and spoofed flag.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void grantsTriggersLookupRulesAndSpoofedFlag() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
try(var c = owner();
|
||||
var runtime = app())
|
||||
{
|
||||
assertThatThrownBy(() -> sql(runtime,
|
||||
"INSERT INTO worm_audit_chain(id,kind,audit_id,detail_ids,prev_hash,link_hash,chained_at) "
|
||||
+ "SELECT 99,'LATE_DETAIL',audit_id,detail_ids,prev_hash,link_hash,chained_at FROM worm_audit_chain WHERE id=1"))
|
||||
.isInstanceOf(SQLException.class).satisfies(e -> assertThat(((SQLException) e).getSQLState()).isEqualTo("23505"));
|
||||
|
||||
for(String table :
|
||||
List.of("worm_audit_export", "worm_audit_verification", "worm_audit_purge"))
|
||||
{
|
||||
sql(c, "INSERT INTO " + table + " DEFAULT VALUES");
|
||||
}
|
||||
List<String> tables =
|
||||
List.of(
|
||||
"audit",
|
||||
"audit_detail",
|
||||
"worm_audit_chain",
|
||||
"worm_audit_checkpoint",
|
||||
"worm_audit_export",
|
||||
"worm_audit_verification",
|
||||
"worm_audit_purge");
|
||||
for(String table : tables)
|
||||
{
|
||||
assertThat(
|
||||
count(
|
||||
c,
|
||||
"SELECT has_table_privilege('app','"
|
||||
+ table
|
||||
+ "','UPDATE')::int"))
|
||||
.isZero();
|
||||
assertThat(
|
||||
count(
|
||||
c,
|
||||
"SELECT has_table_privilege('app','"
|
||||
+ table
|
||||
+ "','DELETE')::int"))
|
||||
.isZero();
|
||||
assertThat(
|
||||
count(
|
||||
c,
|
||||
"SELECT has_table_privilege('app','"
|
||||
+ table
|
||||
+ "','TRUNCATE')::int"))
|
||||
.isZero();
|
||||
// PostgreSQL checks grants before firing triggers: an unprivileged app
|
||||
// receives 42501. Temporarily grant DML to prove the trigger defense too.
|
||||
assertThatThrownBy(() -> sql(runtime, "UPDATE " + table + " SET id=id"))
|
||||
.isInstanceOf(SQLException.class)
|
||||
.satisfies(
|
||||
e ->
|
||||
assertThat(((SQLException) e).getSQLState())
|
||||
.isEqualTo("42501"));
|
||||
sql(c, "GRANT UPDATE, DELETE, TRUNCATE ON " + table + " TO app");
|
||||
// A table a foreign key references (audit, checkpoint, export) refuses TRUNCATE before
|
||||
// its guard runs; TRUNCATE ... CASCADE (owner) reaches the guards, which refuse it.
|
||||
if(count(c, "SELECT count(*) FROM pg_constraint WHERE contype='f' AND confrelid='" + table + "'::regclass") > 0)
|
||||
{
|
||||
assertThatThrownBy(() -> sql(runtime, "TRUNCATE " + table))
|
||||
.isInstanceOf(SQLException.class)
|
||||
.hasMessageContaining("referenced in a foreign key constraint");
|
||||
assertThatThrownBy(() -> sql(c, "TRUNCATE " + table + " CASCADE"))
|
||||
.isInstanceOf(SQLException.class)
|
||||
.hasMessageContaining("worm_audit:");
|
||||
}
|
||||
boolean referenced = count(c, "SELECT count(*) FROM pg_constraint WHERE contype='f' AND confrelid='" + table + "'::regclass") > 0;
|
||||
for(String operation : referenced
|
||||
? List.of("UPDATE " + table + " SET id=id", "DELETE FROM " + table)
|
||||
: List.of("UPDATE " + table + " SET id=id", "DELETE FROM " + table, "TRUNCATE " + table))
|
||||
{
|
||||
assertThatThrownBy(() -> sql(runtime, operation))
|
||||
.isInstanceOf(SQLException.class)
|
||||
.hasMessageContaining("worm_audit:")
|
||||
.satisfies(
|
||||
e ->
|
||||
assertThat(((SQLException) e).getSQLState())
|
||||
.isEqualTo("P0001"));
|
||||
}
|
||||
}
|
||||
sql(runtime, "SELECT set_config('worm_audit.purging','on',false)");
|
||||
assertThatThrownBy(() -> sql(runtime, "DELETE FROM audit"))
|
||||
.hasMessageContaining("worm_audit:");
|
||||
sql(runtime, "UPDATE audit_table SET label='Allowed label'");
|
||||
sql(runtime, "UPDATE audit_user SET modify_date=now()");
|
||||
for(String table : List.of("audit_table", "audit_user"))
|
||||
{
|
||||
sql(c, "GRANT UPDATE, DELETE ON " + table + " TO app");
|
||||
assertThatThrownBy(() -> sql(runtime, "UPDATE " + table + " SET name='renamed'"))
|
||||
.hasMessageContaining("worm_audit:");
|
||||
assertThatThrownBy(() -> sql(runtime, "UPDATE " + table + " SET id=id+1"))
|
||||
.hasMessageContaining("worm_audit:");
|
||||
assertThatThrownBy(() -> sql(runtime, "DELETE FROM " + table))
|
||||
.hasMessageContaining("worm_audit:");
|
||||
assertThatThrownBy(() -> sql(c, "TRUNCATE " + table))
|
||||
.hasMessageContaining("worm_audit:");
|
||||
sql(runtime, "UPDATE " + table + " SET modify_date=now()");
|
||||
}
|
||||
sql(runtime, "UPDATE audit_table SET label='New label'");
|
||||
sql(c, WormAuditDdl.postgres(config));
|
||||
assertThat(
|
||||
count(
|
||||
c,
|
||||
"SELECT"
|
||||
+ " has_function_privilege('app','worm_audit_purge(timestamptz)','EXECUTE')::int"))
|
||||
.isZero();
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify retention refusal purge anchor and real backend tamper.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void retentionRefusalPurgeAnchorAndRealBackendTamper() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
audit(2);
|
||||
detail(2, 2);
|
||||
// A live suffix remains after the expired prefix is removed.
|
||||
try(var c = owner())
|
||||
{
|
||||
sql(
|
||||
c,
|
||||
"INSERT INTO audit(id,audit_table_id,audit_user_id,record_id,message,timestamp)"
|
||||
+ " VALUES(3,1,1,17,'Record was Edited',now()-interval '2 minutes')");
|
||||
}
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).isEmpty();
|
||||
String anchor = STORE.get(WormAuditTables.CHAIN, 2).getValueString("link_hash");
|
||||
try(var c = owner())
|
||||
{
|
||||
sql(
|
||||
c,
|
||||
"DO $$ BEGIN IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='purge_login')"
|
||||
+ " THEN CREATE ROLE purge_login LOGIN PASSWORD 'purger-fixture'; END IF;"
|
||||
+ " END $$; GRANT worm_audit_purger TO purge_login");
|
||||
}
|
||||
try(var purger =
|
||||
DriverManager.getConnection(
|
||||
POSTGRES.getJdbcUrl(), "purge_login", "purger-fixture"))
|
||||
{
|
||||
sql(purger, "SELECT worm_audit_purge(now())");
|
||||
assertThat(STORE.all(WormAuditTables.PURGE).getFirst().getValueString("outcome"))
|
||||
.isEqualTo("REFUSED");
|
||||
sql(purger, "SELECT worm_audit_purge(now()-interval '6 years')");
|
||||
}
|
||||
var purge = STORE.all(WormAuditTables.PURGE).getLast();
|
||||
assertThat(purge.getValueString("outcome")).isEqualTo("PURGED");
|
||||
assertThat(purge.getValueString("requested_by")).isEqualTo("purge_login");
|
||||
assertThat(purge.getValueLong("audits_deleted")).isEqualTo(2);
|
||||
assertThat(purge.getValueLong("details_deleted")).isEqualTo(2);
|
||||
assertThat(purge.getValueLong("links_deleted")).isEqualTo(2);
|
||||
assertThat(purge.getValueString("anchor_hash")).isEqualTo(anchor);
|
||||
try(var c = owner())
|
||||
{
|
||||
assertThat(count(c, "SELECT count(*) FROM worm_audit_chain_detail WHERE chain_id <= 2")).as("coverage purged with its links").isZero();
|
||||
assertThat(count(c, "SELECT count(*) FROM worm_audit_chain_detail")).isZero();
|
||||
}
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).isEmpty();
|
||||
detail(3, 3);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
try(var c = owner())
|
||||
{
|
||||
sql(
|
||||
c,
|
||||
"ALTER TABLE audit_detail DISABLE TRIGGER ALL; UPDATE audit_detail SET"
|
||||
+ " new_value='tampered' WHERE id=3; ALTER TABLE audit_detail ENABLE"
|
||||
+ " TRIGGER ALL");
|
||||
}
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks())
|
||||
.anyMatch(
|
||||
b ->
|
||||
b.reason()
|
||||
.contains(
|
||||
"Break at link 4 (audit 3, table cms_page,"
|
||||
+ " record 17)")
|
||||
&& b.reason().contains("audit_detail 3"));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify prefix cannot split an audit and advisory lock skips tick.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void prefixCannotSplitAnAuditAndAdvisoryLockSkipsTick() throws Exception
|
||||
{
|
||||
// This retention fixture deliberately polls a detail for a six-year-old header.
|
||||
setting("late_detail_lookback_hours", 24 * 365 * 10);
|
||||
audit(1);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
try(var c = owner())
|
||||
{
|
||||
sql(
|
||||
c,
|
||||
"INSERT INTO audit(id,audit_table_id,audit_user_id,record_id,message,timestamp)"
|
||||
+ " VALUES(2,1,1,17,'Record was Edited',now()-interval '2 minutes')");
|
||||
}
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
detail(1, 1);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
try(var c = owner())
|
||||
{
|
||||
sql(c, "SELECT worm_audit_purge(now()-interval '6 years')");
|
||||
assertThat(STORE.all(WormAuditTables.PURGE).getLast().getValueString("outcome"))
|
||||
.isEqualTo("REFUSED");
|
||||
assertThat(STORE.all(WormAuditTables.CHAIN)).hasSize(3);
|
||||
c.setAutoCommit(false);
|
||||
sql(c, "SELECT pg_advisory_xact_lock(" + WormAuditChainer.ADVISORY_LOCK + ")");
|
||||
assertThat(new WormAuditChainer(config).chainOnce().skipped()).isTrue();
|
||||
c.rollback();
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify eight writers and three chainers with late commits.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void eightWritersAndThreeChainersWithLateCommits() throws Exception
|
||||
{
|
||||
var done = new AtomicBoolean(false);
|
||||
var executor = Executors.newFixedThreadPool(11);
|
||||
List<Future<?>> writers = new ArrayList<>();
|
||||
List<Future<?>> chainers = new ArrayList<>();
|
||||
try
|
||||
{
|
||||
for(int thread = 0; thread < 8; thread++)
|
||||
{
|
||||
final int writer = thread;
|
||||
writers.add(
|
||||
executor.submit(
|
||||
() ->
|
||||
{
|
||||
try(var c = app())
|
||||
{
|
||||
for(int i = 0; i < 8; i++)
|
||||
{
|
||||
long id = 1000L + writer * 100 + i;
|
||||
c.setAutoCommit(false);
|
||||
sql(
|
||||
c,
|
||||
"INSERT INTO"
|
||||
+ " audit(id,audit_table_id,audit_user_id,record_id,message,timestamp)"
|
||||
+ " VALUES("
|
||||
+ id
|
||||
+ ",1,1,17,'Record was"
|
||||
+ " Edited',now()-interval '2"
|
||||
+ " minutes')");
|
||||
if(i % 2 == 0)
|
||||
{
|
||||
c.commit();
|
||||
}
|
||||
Thread.sleep(5L * (writer + 1));
|
||||
sql(
|
||||
c,
|
||||
"INSERT INTO"
|
||||
+ " audit_detail(id,audit_id,field_name,message,old_value,new_value)"
|
||||
+ " VALUES("
|
||||
+ id
|
||||
+ ","
|
||||
+ id
|
||||
+ ",'title','Changed"
|
||||
+ " Title','old','new')");
|
||||
c.commit();
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}));
|
||||
}
|
||||
for(int i = 0; i < 3; i++)
|
||||
{
|
||||
chainers.add(
|
||||
executor.submit(
|
||||
() ->
|
||||
{
|
||||
QContext.init(instance, new QSession());
|
||||
try
|
||||
{
|
||||
do
|
||||
{
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
Thread.sleep(10);
|
||||
} while(!done.get());
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}));
|
||||
}
|
||||
for(var writer : writers)
|
||||
{
|
||||
writer.get(90, TimeUnit.SECONDS);
|
||||
}
|
||||
done.set(true);
|
||||
for(var chainer : chainers)
|
||||
{
|
||||
chainer.get(90, TimeUnit.SECONDS);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
done.set(true);
|
||||
executor.shutdownNow();
|
||||
}
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
var links = STORE.all(WormAuditTables.CHAIN);
|
||||
for(int i = 0; i < links.size(); i++)
|
||||
{
|
||||
assertThat(links.get(i).getValueLong("id")).isEqualTo(i + 1L);
|
||||
}
|
||||
assertThat(links.stream().filter(r -> "AUDIT".equals(r.getValueString("kind"))).count())
|
||||
.isEqualTo(64);
|
||||
assertThat(links.stream().flatMap(r -> AuditStore.detailIds(r).stream()).toList())
|
||||
.hasSize(64)
|
||||
.doesNotHaveDuplicates();
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).isEmpty();
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Even the function owner cannot authorize direct deletes by setting a GUC.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void ownerCannotSpoofPurgeStack() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
try(var connection = owner())
|
||||
{
|
||||
sql(connection, "ALTER TABLE audit ALTER COLUMN timestamp TYPE timestamp without time zone USING timestamp AT TIME ZONE 'UTC'");
|
||||
sql(connection, "INSERT INTO audit(id,audit_table_id,audit_user_id,timestamp) VALUES"
|
||||
+ "(2,1,1,(now() AT TIME ZONE 'UTC')-interval '6 years'-interval '1 hour')");
|
||||
}
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
try(var connection = owner())
|
||||
{
|
||||
sql(connection, "SELECT set_config('worm_audit.purging','on',false)");
|
||||
for(String table : List.of("audit", "audit_detail", "worm_audit_chain"))
|
||||
{
|
||||
assertThatThrownBy(() -> sql(connection, "DELETE FROM " + table + " WHERE id=1"))
|
||||
.isInstanceOf(SQLException.class).hasMessageContaining("worm_audit:");
|
||||
}
|
||||
assertThat(count(connection, "SELECT count(*) FROM worm_audit_purge")).isZero();
|
||||
// This succeeds only if the PG16 context text matches the stack guard.
|
||||
sql(connection, "SET timezone='America/Chicago'");
|
||||
sql(connection, "SELECT worm_audit_purge(now()-interval '6 years')");
|
||||
assertThat(count(connection, "SELECT count(*) FROM audit")).isZero();
|
||||
assertThat(count(connection, "SELECT count(*) FROM worm_audit_purge WHERE outcome='PURGED'")).isEqualTo(1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A UTC without-tz audit column settles identically in a Chicago JVM/session.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void chainerUsesUtcCutoffInChicago() throws Exception
|
||||
{
|
||||
var original = java.util.TimeZone.getDefault();
|
||||
setting("settle_horizon_seconds", 60);
|
||||
try(var connection = owner())
|
||||
{
|
||||
sql(connection, "ALTER TABLE audit ALTER COLUMN timestamp TYPE timestamp without time zone USING timestamp AT TIME ZONE 'UTC'");
|
||||
sql(connection, "INSERT INTO audit(id,audit_table_id,audit_user_id,timestamp) VALUES"
|
||||
+ "(1,1,1,TIMESTAMP '2026-10-13 12:00:00'),(2,1,1,TIMESTAMP '2026-10-13 12:01:30')");
|
||||
// New runtime connections inherit this non-UTC session setting.
|
||||
sql(connection, "ALTER ROLE app SET timezone='America/Chicago'");
|
||||
java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("America/Chicago"));
|
||||
assertThat(new WormAuditChainer(config).chainOnce(java.time.Instant.parse("2026-10-13T12:02:00Z")).audits()).isEqualTo(1);
|
||||
assertThat(STORE.all(WormAuditTables.CHAIN)).extracting(r -> r.getValueLong("audit_id")).containsExactly(1L);
|
||||
java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("UTC"));
|
||||
assertThat(new WormAuditVerifier(config).verify(java.util.Set.of(1L)).getBreaks()).isEmpty();
|
||||
}
|
||||
finally
|
||||
{
|
||||
java.util.TimeZone.setDefault(original);
|
||||
try(var connection = owner())
|
||||
{
|
||||
sql(connection, "ALTER ROLE app RESET timezone");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** PostgreSQL containment polling includes the lower window boundary only.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void postgresLateDetailWindowAndCompleteness() throws Exception
|
||||
{
|
||||
setting("settle_horizon_seconds", 60);
|
||||
var now = java.time.Instant.now().truncatedTo(java.time.temporal.ChronoUnit.SECONDS);
|
||||
var cutoff = now.minusSeconds(60);
|
||||
try(var connection = owner(); var statement = connection.prepareStatement(
|
||||
"INSERT INTO audit(id,audit_table_id,audit_user_id,timestamp) VALUES(?,1,1,?)"))
|
||||
{
|
||||
for(int id = 1; id <= 4; id++)
|
||||
{
|
||||
long seconds = switch(id)
|
||||
{
|
||||
case 1 -> 86401;
|
||||
case 2 -> 86400;
|
||||
case 3 -> 1;
|
||||
default -> 0;
|
||||
};
|
||||
statement.setInt(1, id);
|
||||
statement.setObject(2, cutoff.minusSeconds(seconds).atOffset(java.time.ZoneOffset.UTC));
|
||||
statement.executeUpdate();
|
||||
}
|
||||
}
|
||||
var writer = new WormAuditChainer(config);
|
||||
assertThat(writer.chainOnce(now).audits()).isEqualTo(3);
|
||||
for(long id = 1; id <= 4; id++)
|
||||
{
|
||||
detail(id, id);
|
||||
}
|
||||
assertThat(writer.chainOnce(now).lateDetails()).isEqualTo(2);
|
||||
assertThat(STORE.all(WormAuditTables.CHAIN).stream().flatMap(r -> AuditStore.detailIds(r).stream()).toList()).containsExactly(2L, 3L);
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).anyMatch(b -> b.reason().contains("audit_detail 1 is not covered"));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The chain trigger writes one coverage row per detail with its link; the
|
||||
** table is write-once, backfilled from the chain by the DDL, purged with its
|
||||
** links, and verification breaks if it ever disagrees with the chain.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void detailCoverageTableIsWrittenByTheTriggerBackfilledAndWriteOnce() throws Exception
|
||||
{
|
||||
// The fixture's audit headers are years old; poll them as late-detail candidates.
|
||||
setting("late_detail_lookback_hours", 24 * 365 * 10);
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
detail(2, 1);
|
||||
audit(2);
|
||||
detail(3, 2);
|
||||
var writer = new WormAuditChainer(config);
|
||||
assertThat(writer.chainOnce().links()).isEqualTo(2);
|
||||
detail(4, 1);
|
||||
assertThat(writer.chainOnce().lateDetails()).isEqualTo(1);
|
||||
assertThat(writer.chainOnce().links()).isZero();
|
||||
String pairs = "SELECT string_agg(detail_id || '@' || chain_id, ',' ORDER BY detail_id) FROM worm_audit_chain_detail";
|
||||
try(var c = owner(); var runtime = app(); var s = c.createStatement())
|
||||
{
|
||||
try(var r = s.executeQuery(pairs))
|
||||
{
|
||||
r.next();
|
||||
assertThat(r.getString(1)).isEqualTo("1@1,2@1,3@2,4@3");
|
||||
}
|
||||
assertThatThrownBy(() -> sql(runtime, "INSERT INTO worm_audit_chain_detail(detail_id,chain_id) VALUES(4,1)"))
|
||||
.isInstanceOf(SQLException.class).satisfies(e -> assertThat(((SQLException) e).getSQLState()).isEqualTo("23505"));
|
||||
for(String privilege : List.of("UPDATE", "DELETE", "TRUNCATE"))
|
||||
{
|
||||
assertThat(count(c, "SELECT has_table_privilege('app','worm_audit_chain_detail','" + privilege + "')::int")).isZero();
|
||||
}
|
||||
sql(c, "GRANT UPDATE, DELETE, TRUNCATE ON worm_audit_chain_detail TO app");
|
||||
for(String operation : List.of("UPDATE worm_audit_chain_detail SET chain_id=chain_id", "DELETE FROM worm_audit_chain_detail", "TRUNCATE worm_audit_chain_detail"))
|
||||
{
|
||||
assertThatThrownBy(() -> sql(runtime, operation)).isInstanceOf(SQLException.class).hasMessageContaining("worm_audit:")
|
||||
.satisfies(e -> assertThat(((SQLException) e).getSQLState()).isEqualTo("P0001"));
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
// An install that predates the table: the DDL backfills it from the //
|
||||
// chain, and the writer then finds nothing left to chain. //
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
sql(c, "DROP TABLE worm_audit_chain_detail");
|
||||
sql(c, WormAuditDdl.postgres(config));
|
||||
try(var r = s.executeQuery(pairs))
|
||||
{
|
||||
r.next();
|
||||
assertThat(r.getString(1)).isEqualTo("1@1,2@1,3@2,4@3");
|
||||
}
|
||||
assertThat(writer.chainOnce().links()).isZero();
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).isEmpty();
|
||||
|
||||
sql(c, "ALTER TABLE worm_audit_chain_detail DISABLE TRIGGER ALL; UPDATE worm_audit_chain_detail SET chain_id=1 WHERE detail_id=3;"
|
||||
+ " ALTER TABLE worm_audit_chain_detail ENABLE TRIGGER ALL");
|
||||
}
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).anyMatch(b -> b.reason().contains("audit_detail 3 has a detail coverage row that disagrees"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Referential integrity guardrail: every table reference of the WORM tables
|
||||
** (possible-value fields, and the chain's audit header) has a RESTRICT
|
||||
** foreign key, or an exemption with its reason. Nothing may cascade into or
|
||||
** out of audit evidence.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void everyReferenceHasARestrictForeignKeyOrAReason() throws Exception
|
||||
{
|
||||
String links = "the retention purge deletes the sealed prefix of chain links; this evidence row outlives the links it names";
|
||||
java.util.Map<String, String> exempt = java.util.Map.of(
|
||||
"worm_audit_checkpoint.from_link_id", links,
|
||||
"worm_audit_checkpoint.to_link_id", links,
|
||||
"worm_audit_export.from_link_id", links,
|
||||
"worm_audit_export.to_link_id", links,
|
||||
"worm_audit_verification.from_link_id", links,
|
||||
"worm_audit_verification.to_link_id", links,
|
||||
"worm_audit_verification.first_break_link_id", links + "; a break may name a link that is missing",
|
||||
"worm_audit_purge.anchor_link_id", "the anchor is the last link the same purge deleted",
|
||||
"worm_audit_chain_detail.chain_id", "the chain is WORM: the owner has no UPDATE, so a foreign key check into it is denied;"
|
||||
+ " the coverage trigger writes the row with its link and the purge deletes it first");
|
||||
|
||||
java.util.Set<String> references = new java.util.TreeSet<>(List.of("worm_audit_chain.audit_id"));
|
||||
for(var table : instance.getTables().values())
|
||||
{
|
||||
if(!table.getName().startsWith("worm_audit_"))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
for(var field : table.getFields().values())
|
||||
{
|
||||
if(field.getPossibleValueSourceName() != null && instance.getTable(field.getPossibleValueSourceName()) != null)
|
||||
{
|
||||
references.add(table.getName() + "." + field.getBackendName());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
java.util.Map<String, String> foreignKeys = new java.util.HashMap<>();
|
||||
try(var c = owner(); var s = c.createStatement(); var r = s.executeQuery("""
|
||||
SELECT c.conrelid::regclass::text || '.' || a.attname, c.confdeltype::text || c.convalidated::text
|
||||
FROM pg_constraint c JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = c.conkey[1]
|
||||
WHERE c.contype = 'f' AND c.conrelid::regclass::text LIKE 'worm_audit_%'"""))
|
||||
{
|
||||
while(r.next())
|
||||
{
|
||||
foreignKeys.put(r.getString(1), r.getString(2));
|
||||
}
|
||||
}
|
||||
assertThat(references.stream().filter(ref -> !foreignKeys.containsKey(ref) && !exempt.containsKey(ref)).toList())
|
||||
.as("references without a foreign key (add one to worm-audit.sql, or an exemption with a reason)").isEmpty();
|
||||
assertThat(foreignKeys).as("RESTRICT and validated").containsOnlyKeys("worm_audit_chain.audit_id", "worm_audit_export.from_checkpoint_id",
|
||||
"worm_audit_export.to_checkpoint_id", "worm_audit_verification.export_id").allSatisfy((k, v) -> assertThat(v).isEqualTo("rtrue"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Upgrading a schema whose chain names a missing audit header: the link is
|
||||
** evidence, so it is copied to qqq_fk_quarantine (KEPT), never changed, and
|
||||
** the constraint is added NOT VALID (new links are still checked). A re-run
|
||||
** changes nothing.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void orphanEvidenceIsKeptAndTheConstraintIsNotValidated() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
try(var c = owner())
|
||||
{
|
||||
sql(c, "ALTER TABLE worm_audit_chain DROP CONSTRAINT fk_worm_audit_chain_audit_id");
|
||||
sql(c, "ALTER TABLE audit DISABLE TRIGGER worm_audit_row_guard; DELETE FROM audit WHERE id = 1; ALTER TABLE audit ENABLE TRIGGER worm_audit_row_guard");
|
||||
sql(c, WormAuditDdl.postgres(config));
|
||||
sql(c, WormAuditDdl.postgres(config));
|
||||
assertThat(count(c, "SELECT count(*) FROM qqq_fk_quarantine WHERE table_name = 'worm_audit_chain' AND column_name = 'audit_id'"
|
||||
+ " AND row_id = '1' AND missing_parent_id = '1' AND action = 'KEPT' AND row_data LIKE '%\"link_hash\"%'")).isEqualTo(1);
|
||||
assertThat(count(c, "SELECT count(*) FROM qqq_fk_quarantine")).isEqualTo(1);
|
||||
assertThat(count(c, "SELECT count(*) FROM worm_audit_chain WHERE id = 1 AND audit_id = 1")).isEqualTo(1);
|
||||
assertThat(count(c, "SELECT count(*) FROM pg_constraint WHERE conname = 'fk_worm_audit_chain_audit_id' AND NOT convalidated")).isEqualTo(1);
|
||||
assertThatThrownBy(() -> sql(c, "INSERT INTO worm_audit_chain(id,kind,audit_id,detail_ids,prev_hash,link_hash,chained_at)"
|
||||
+ " VALUES(99,'AUDIT',404,'',repeat('0',64),repeat('0',64),now())"))
|
||||
.hasMessageContaining("fk_worm_audit_chain_audit_id");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The settings row (seeded by the DDL) overrides config: change it there.
|
||||
*******************************************************************************/
|
||||
private void setting(String column, int value) throws Exception
|
||||
{
|
||||
try(var connection = owner())
|
||||
{
|
||||
sql(connection, "UPDATE worm_audit_settings SET " + column + "=" + value + " WHERE id=1");
|
||||
}
|
||||
config.invalidateSettings();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,267 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.VerificationRuns;
|
||||
import com.kingsrook.qbits.wormaudit.processes.WormAuditProcesses;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertResolution;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertSignal;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertSinkInterface;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunInput;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunOutput;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.STORE;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.core;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.detail;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.tamper;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for processes and alerts test.
|
||||
*******************************************************************************/
|
||||
class ProcessesAndAlertsTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for sink.
|
||||
*******************************************************************************/
|
||||
public static class Sink implements AlertSinkInterface
|
||||
{
|
||||
static final List<AlertSignal> raised = new ArrayList<>();
|
||||
static final List<AlertResolution> resolved = new ArrayList<>();
|
||||
static final List<String> beats = new ArrayList<>();
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify raise.
|
||||
*******************************************************************************/
|
||||
public void raise(AlertSignal signal)
|
||||
{
|
||||
raised.add(signal);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify resolve.
|
||||
*******************************************************************************/
|
||||
public void resolve(AlertResolution resolution)
|
||||
{
|
||||
resolved.add(resolution);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify heartbeat.
|
||||
*******************************************************************************/
|
||||
public void heartbeat(String key, Instant at)
|
||||
{
|
||||
beats.add(key);
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** RC.5 validates declared severities, names and daily heartbeat expectations.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void alertTypesAndDailyMonitors() throws Exception
|
||||
{
|
||||
var instance = init(new WormAuditQBitConfig());
|
||||
new com.kingsrook.qqq.backend.core.instances.QInstanceValidator().validate(instance);
|
||||
assertThat(new com.kingsrook.qbits.wormaudit.chain.AuditStore(null).isMemory()).isTrue();
|
||||
var types = AlertsInstanceMetaData.of(instance).getAlertTypes();
|
||||
assertThat(types).containsOnlyKeys("wormaudit.chain.break", "wormaudit.export.failed", "wormaudit.verify.daily", "wormaudit.export.daily");
|
||||
assertThat(types.get("wormaudit.chain.break").getDefaultSeverity())
|
||||
.isEqualTo(com.kingsrook.qqq.backend.core.alerts.AlertSeverity.CRITICAL);
|
||||
assertThat(types.get("wormaudit.export.failed").getDefaultSeverity())
|
||||
.isEqualTo(com.kingsrook.qqq.backend.core.alerts.AlertSeverity.ERROR);
|
||||
types.values().forEach(type ->
|
||||
{
|
||||
assertThat(type.getLabel()).isNotBlank();
|
||||
assertThat(type.getDescription()).isNotBlank();
|
||||
assertThat(AlertsInstanceMetaData.ALERT_TYPE_NAME_PATTERN.matcher(type.getName()).matches()).isTrue();
|
||||
});
|
||||
for(String name : List.of("wormaudit.verify.daily", "wormaudit.export.daily"))
|
||||
{
|
||||
var monitors = types.get(name).getDefaultMonitors();
|
||||
assertThat(monitors).hasSize(1);
|
||||
assertThat(monitors.getFirst().getKind()).isEqualTo(com.kingsrook.qqq.backend.core.alerts.AlertMonitorKind.HEARTBEAT);
|
||||
assertThat(monitors.getFirst().getDefinition()).containsEntry("heartbeatKey", name)
|
||||
.containsEntry("expectedEverySeconds", 86400).containsEntry("graceSeconds", 7200);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify daily raise resolve heartbeat and user result.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void dailyRaiseResolveHeartbeatAndUserResult() throws Exception
|
||||
{
|
||||
var config = ChainVerificationTest.signed();
|
||||
var instance = init(config);
|
||||
Sink.raised.clear();
|
||||
Sink.resolved.clear();
|
||||
Sink.beats.clear();
|
||||
AlertsInstanceMetaData.ofOrWithNew(instance).withSink(new QCodeReference(Sink.class));
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
new WormAuditProcesses.ChainTick()
|
||||
.run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
tamper("auditDetail", 1, "newValue", "broken");
|
||||
var daily = new WormAuditProcesses.Daily();
|
||||
var output = new RunBackendStepOutput();
|
||||
daily.run(new RunBackendStepInput(), output);
|
||||
assertThat(output.getValueString("outcome")).isEqualTo("FAIL");
|
||||
assertThat(Sink.raised).hasSize(1);
|
||||
assertThat(Sink.raised.getFirst().getAlertTypeName()).isEqualTo("wormaudit.chain.break");
|
||||
assertThat(Sink.raised.getFirst().getDedupKey()).isEqualTo("chain");
|
||||
assertThat(Sink.raised.getFirst().getRecordRefs().getFirst().getTableName())
|
||||
.isEqualTo(WormAuditTables.VERIFICATION);
|
||||
assertThat(Sink.raised.getFirst().getTitle()).isEqualTo("Audit chain verification found 1 problem");
|
||||
assertThat(Sink.raised.getFirst().getSummary()).contains("Break at link 1");
|
||||
var first = STORE.all(WormAuditTables.VERIFICATION).getFirst();
|
||||
assertThat(first.getValueLong("first_break_link_id")).isEqualTo(1);
|
||||
assertThat(first.getValueLong("first_break_audit_id")).isEqualTo(1);
|
||||
assertThat(first.getValueString("trigger")).isEqualTo("SCHEDULE");
|
||||
tamper("auditDetail", 1, "newValue", "new");
|
||||
daily.run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
assertThat(Sink.resolved).hasSize(1);
|
||||
assertThat(Sink.beats)
|
||||
.containsExactly("wormaudit.verify.daily", "wormaudit.verify.daily");
|
||||
assertThat(STORE.all(WormAuditTables.VERIFICATION)).hasSize(2);
|
||||
assertThat(VerificationRuns.run(null, "USER").passed()).isTrue();
|
||||
assertThat(Sink.beats).hasSize(2);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify review and load across pages including failures and empty scope.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void reviewAndLoadAcrossPagesIncludingFailuresAndEmptyScope() throws Exception
|
||||
{
|
||||
var config = new WormAuditQBitConfig();
|
||||
init(config);
|
||||
audit(1);
|
||||
audit(2);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
var links = STORE.all(WormAuditTables.CHAIN);
|
||||
var review = new WormAuditProcesses.Review();
|
||||
var load = new WormAuditProcesses.Load();
|
||||
var output = new RunBackendStepOutput();
|
||||
assertThat(load.getProcessSummary(output, true)).isEmpty();
|
||||
for(var link : links)
|
||||
{
|
||||
var input = new RunBackendStepInput().withRecords(List.of(link));
|
||||
review.runOnePage(input, output);
|
||||
load.runOnePage(input, output);
|
||||
}
|
||||
assertThat(review.getProcessSummary(output, false).toString())
|
||||
.contains("2 chain links will be verified", "0 checkpoints");
|
||||
load.postRun(
|
||||
new BackendStepPostRunInput(new RunBackendStepInput()),
|
||||
new BackendStepPostRunOutput(output));
|
||||
assertThat(STORE.all(WormAuditTables.VERIFICATION)).hasSize(1);
|
||||
assertThat(load.getProcessSummary(output, true).toString())
|
||||
.contains("2 links verified", "no signing key configured");
|
||||
audit(3);
|
||||
var fullWithEmptyFilter = new WormAuditProcesses.Load();
|
||||
fullWithEmptyFilter.runOnePage(new RunBackendStepInput().withRecords(links), output);
|
||||
fullWithEmptyFilter.postRun(new BackendStepPostRunInput(new RunBackendStepInput().addValue("queryFilterJson", "{}")),
|
||||
new BackendStepPostRunOutput(output));
|
||||
assertThat(fullWithEmptyFilter.getProcessSummary(output, true).toString()).contains("audit 3 is not covered by the chain");
|
||||
tamper("audit", 1, "message", "changed");
|
||||
var selected = new WormAuditProcesses.Load();
|
||||
selected.runOnePage(
|
||||
new RunBackendStepInput().withRecords(List.of(links.getFirst())), output);
|
||||
selected.postRun(
|
||||
new BackendStepPostRunInput(new RunBackendStepInput().addValue("recordIds", "1")),
|
||||
new BackendStepPostRunOutput(output));
|
||||
assertThat(selected.getProcessSummary(output, true).toString())
|
||||
.contains("ERROR", "content differs");
|
||||
init(config);
|
||||
var empty = new WormAuditProcesses.Load();
|
||||
empty.postRun(
|
||||
new BackendStepPostRunInput(new RunBackendStepInput()),
|
||||
new BackendStepPostRunOutput(output));
|
||||
assertThat(empty.getProcessSummary(output, true).toString()).contains("0 links verified");
|
||||
assertThat(new WormAuditProcesses.Review().getProcessSummary(output, false).toString())
|
||||
.contains("links none");
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Full loads retain no id collection; review summarizes disjoint streamed pages once.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void reviewRangesAndSelectionOnlyIds() throws Exception
|
||||
{
|
||||
var config = ChainVerificationTest.signed().withCheckpointEveryLinks(2);
|
||||
init(config);
|
||||
for(long id = 1; id <= 4; id++)
|
||||
{
|
||||
audit(id);
|
||||
}
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
var links = STORE.all(WormAuditTables.CHAIN);
|
||||
var review = new WormAuditProcesses.Review();
|
||||
var full = new WormAuditProcesses.Load();
|
||||
var selected = new WormAuditProcesses.Load();
|
||||
var output = new RunBackendStepOutput();
|
||||
var ids = WormAuditProcesses.Load.class.getDeclaredField("ids");
|
||||
ids.setAccessible(true);
|
||||
for(var link : links)
|
||||
{
|
||||
var input = new RunBackendStepInput().withRecords(List.of(link));
|
||||
review.runOnePage(input, output);
|
||||
full.runOnePage(input, output);
|
||||
assertThat(ids.get(full)).isNull();
|
||||
}
|
||||
String summary = review.getProcessSummary(output, false).toString();
|
||||
assertThat(summary).contains("4 chain links", "links 1–4", "2 checkpoints");
|
||||
// A second call reuses the count; later inserts cannot silently change Review.
|
||||
STORE.insert(WormAuditTables.CHECKPOINT, List.of(new com.kingsrook.qqq.backend.core.model.data.QRecord()
|
||||
.withValue("from_link_id", 1L).withValue("to_link_id", 4L)));
|
||||
assertThat(review.getProcessSummary(output, false).toString()).isEqualTo(summary);
|
||||
selected.runOnePage(new RunBackendStepInput().addValue("recordIds", "2").withRecords(List.of(links.get(1))), output);
|
||||
assertThat(ids.get(selected)).isEqualTo(java.util.Set.of(2L));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Docker dependencies must not enable debug wire logs during tests.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void dockerLoggersAreQuiet()
|
||||
{
|
||||
for(String name : List.of("org.testcontainers", "com.github.dockerjava", "org.apache.hc", "tc"))
|
||||
{
|
||||
assertThat(org.apache.logging.log4j.LogManager.getLogger(name).isDebugEnabled()).isFalse();
|
||||
assertThat(org.apache.logging.log4j.LogManager.getLogger(name).isWarnEnabled()).isTrue();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import java.sql.DriverManager;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import com.kingsrook.qbits.wormaudit.chain.SweepFloors;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.STORE;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.core;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.detail;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.lookups;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for rdbms chain test.
|
||||
*******************************************************************************/
|
||||
class RdbmsChainTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Verify real qqq rdbms actions anti join rollback and tamper.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void realQqqRdbmsActionsAntiJoinRollbackAndTamper() throws Exception
|
||||
{
|
||||
String url =
|
||||
"jdbc:h2:mem:worm_"
|
||||
+ UUID.randomUUID()
|
||||
+ ";DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE";
|
||||
var config = ChainVerificationTest.signed();
|
||||
var instance = RdbmsTestSupport.instance(url, "sa", "", config, false);
|
||||
try(var connection = DriverManager.getConnection(url, "sa", ""))
|
||||
{
|
||||
RdbmsTestSupport.createTables(connection, instance, false);
|
||||
QContext.init(instance, new QSession());
|
||||
lookups();
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
var writer = new WormAuditChainer(config);
|
||||
assertThat(writer.chainOnce().links()).isEqualTo(1);
|
||||
assertThat(writer.chainOnce().links()).isZero();
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).isEmpty();
|
||||
// Fail signing after append, verifying the real RDBMS transaction rolls back.
|
||||
var pins = config.getTrustedPublicKeys();
|
||||
config.withTrustedPublicKeys(Map.of());
|
||||
audit(2);
|
||||
assertThatThrownBy(writer::chainOnce)
|
||||
.hasMessageContaining("WORM audit operation failed");
|
||||
assertThat(STORE.all(WormAuditTables.CHAIN)).hasSize(1);
|
||||
config.withTrustedPublicKeys(pins);
|
||||
assertThat(writer.chainOnce().links()).isEqualTo(1);
|
||||
assertThat(STORE.all(WormAuditTables.CHAIN))
|
||||
.extracting(r -> r.getValueLong("id"))
|
||||
.containsExactly(1L, 2L);
|
||||
connection
|
||||
.createStatement()
|
||||
.execute("UPDATE audit_detail SET new_value='changed' WHERE id=1");
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks())
|
||||
.anyMatch(b -> b.reason().contains("audit_detail 1"));
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise multiple keyset pages, capped checkpoints and bounded SQL query counts.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void batchesPagesAndCheckpointRanges() throws Exception
|
||||
{
|
||||
String url = "jdbc:h2:mem:pages_" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE";
|
||||
var config = ChainVerificationTest.signed().withBatchSize(1500).withCheckpointEveryLinks(400).withCheckpointEveryMinutes(1);
|
||||
var instance = RdbmsTestSupport.instance(url, "sa", "", config, false);
|
||||
try(var connection = DriverManager.getConnection(url, "sa", ""); var statement = connection.createStatement())
|
||||
{
|
||||
RdbmsTestSupport.createTables(connection, instance, false);
|
||||
QContext.init(instance, new QSession());
|
||||
lookups();
|
||||
statement.execute("INSERT INTO audit(id,audit_table_id,audit_user_id,record_id,timestamp) SELECT x,1,1,17,CURRENT_TIMESTAMP - INTERVAL '2' MINUTE FROM SYSTEM_RANGE(1,1005) series(x)");
|
||||
statement.execute("INSERT INTO audit_detail(id,audit_id,field_name,new_value) SELECT x,x,'title','new' FROM SYSTEM_RANGE(1,1005) series(x)");
|
||||
statement.execute("SET QUERY_STATISTICS TRUE");
|
||||
var writer = new WormAuditChainer(config);
|
||||
var now = java.time.Instant.now();
|
||||
var counts = writer.chainOnce(now);
|
||||
assertThat(counts.links()).isEqualTo(1005);
|
||||
assertThat(counts.checkpoints()).isEqualTo(2);
|
||||
assertSelectCount(connection, 100);
|
||||
assertThat(STORE.last(WormAuditTables.CHECKPOINT).getValueInstant("created_at")).isCloseTo(now, org.assertj.core.api.Assertions.within(1, java.time.temporal.ChronoUnit.SECONDS));
|
||||
statement.execute("SET QUERY_STATISTICS FALSE");
|
||||
statement.execute("SET QUERY_STATISTICS TRUE");
|
||||
assertThat(writer.chainOnce(now.plusSeconds(61)).checkpoints()).isEqualTo(1);
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).isEmpty();
|
||||
assertSelectCount(connection, 100);
|
||||
assertThat(STORE.all(WormAuditTables.CHECKPOINT)).extracting(r -> r.getValueLong("link_count")).containsExactly(400L, 400L, 205L);
|
||||
statement.execute("UPDATE audit_detail SET new_value='changed' WHERE id=1001");
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).anyMatch(b -> b.reason().contains("audit_detail 1,001"));
|
||||
statement.execute("DELETE FROM worm_audit_chain WHERE id=1000");
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).anyMatch(b -> b.reason().contains("gap"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Assert actual query executions stay proportional to batches, not row count.
|
||||
*******************************************************************************/
|
||||
private void assertSelectCount(java.sql.Connection connection, int limit) throws Exception
|
||||
{
|
||||
try(var statement = connection.createStatement(); var rows = statement.executeQuery(
|
||||
"SELECT COALESCE(SUM(EXECUTION_COUNT),0) FROM INFORMATION_SCHEMA.QUERY_STATISTICS WHERE UPPER(SQL_STATEMENT) LIKE 'SELECT%'"))
|
||||
{
|
||||
rows.next();
|
||||
assertThat(rows.getLong(1)).isLessThan(limit);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Poll only the lookback window and report older omissions with exact bounded ids.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void lateWindowAndCompletenessOffenderLimit() throws Exception
|
||||
{
|
||||
String url = "jdbc:h2:mem:window_" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE";
|
||||
var config = new WormAuditQBitConfig().withBatchSize(500);
|
||||
var instance = RdbmsTestSupport.instance(url, "sa", "", config, false);
|
||||
try(var connection = DriverManager.getConnection(url, "sa", ""); var statement = connection.createStatement())
|
||||
{
|
||||
RdbmsTestSupport.createTables(connection, instance, false);
|
||||
QContext.init(instance, new QSession());
|
||||
lookups();
|
||||
audit(1);
|
||||
statement.execute("INSERT INTO audit(id,audit_table_id,audit_user_id,timestamp) VALUES(2,1,1,CURRENT_TIMESTAMP - INTERVAL '2' MINUTE)");
|
||||
var writer = new WormAuditChainer(config);
|
||||
writer.chainOnce();
|
||||
detail(1, 1);
|
||||
detail(2, 2);
|
||||
assertThat(writer.chainOnce().lateDetails()).isEqualTo(1);
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).anyMatch(b -> b.reason().contains("audit_detail 1 is not covered"));
|
||||
statement.execute("INSERT INTO worm_audit_chain SELECT 99,kind,audit_id,detail_ids,prev_hash,link_hash,chained_at FROM worm_audit_chain WHERE id=3");
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).anyMatch(b -> b.reason().contains("audit_detail 2 is covered twice"));
|
||||
statement.execute("DELETE FROM worm_audit_chain WHERE id=99");
|
||||
statement.execute("INSERT INTO audit_detail(id,audit_id) SELECT x,1 FROM SYSTEM_RANGE(3,107) series(x)");
|
||||
var result = new WormAuditVerifier(config).verify();
|
||||
assertThat(result.getBreakCount()).isEqualTo(106);
|
||||
assertThat(result.getBreaks()).hasSize(100);
|
||||
assertThat(result.getBreaks().getFirst().reason()).contains("audit_detail 1 is not covered");
|
||||
assertThat(result.getBreaks().getLast().reason()).contains("audit_detail 101 is not covered");
|
||||
assertThat(result.getWarnings()).anyMatch(w -> w.contains("106 coverage offenders"));
|
||||
statement.execute("INSERT INTO audit(id,timestamp) SELECT x,CURRENT_TIMESTAMP - INTERVAL '2' MINUTE FROM SYSTEM_RANGE(2000,2104) series(x)");
|
||||
result = new WormAuditVerifier(config).verify();
|
||||
assertThat(result.getBreakCount()).isEqualTo(211);
|
||||
assertThat(result.getBreaks()).hasSize(200);
|
||||
assertThat(result.getBreaks().getFirst().reason()).contains("audit 2,000 is not covered");
|
||||
assertThat(result.getBreaks().get(99).reason()).contains("audit 2,099 is not covered");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Cutoffs are UTC local values for timestamp columns, independent of JVM zone.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void withoutTimezoneCutoffInChicago() throws Exception
|
||||
{
|
||||
var original = java.util.TimeZone.getDefault();
|
||||
String url = "jdbc:h2:mem:timezone_" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE";
|
||||
var config = new WormAuditQBitConfig();
|
||||
var instance = RdbmsTestSupport.instance(url, "sa", "", config, false);
|
||||
try(var connection = DriverManager.getConnection(url, "sa", ""); var statement = connection.createStatement())
|
||||
{
|
||||
RdbmsTestSupport.createTables(connection, instance, false);
|
||||
statement.execute("ALTER TABLE audit ALTER COLUMN timestamp timestamp without time zone");
|
||||
QContext.init(instance, new QSession());
|
||||
lookups();
|
||||
statement.execute("INSERT INTO audit(id,audit_table_id,audit_user_id,timestamp) VALUES(1,1,1,TIMESTAMP '2026-10-13 12:00:00'),(2,1,1,TIMESTAMP '2026-10-13 12:01:30')");
|
||||
java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("America/Chicago"));
|
||||
assertThat(new WormAuditChainer(config).chainOnce(java.time.Instant.parse("2026-10-13T12:02:00Z")).audits()).isEqualTo(1);
|
||||
assertThat(STORE.all(WormAuditTables.CHAIN)).extracting(r -> r.getValueLong("audit_id")).containsExactly(1L);
|
||||
java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("UTC"));
|
||||
assertThat(new WormAuditVerifier(config).verify(java.util.Set.of(1L)).getBreaks()).isEmpty();
|
||||
}
|
||||
finally
|
||||
{
|
||||
java.util.TimeZone.setDefault(original);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Without a coverage trigger the writer records coverage itself, and the
|
||||
** sweep floors advance only past settled ids: a later header and a late
|
||||
** detail for an old, already chained header are both still chained.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void coverageRowsAndSweepFloors() throws Exception
|
||||
{
|
||||
String url = "jdbc:h2:mem:floors_" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE";
|
||||
var config = new WormAuditQBitConfig().withBatchSize(500);
|
||||
var instance = RdbmsTestSupport.instance(url, "sa", "", config, false);
|
||||
try(var connection = DriverManager.getConnection(url, "sa", ""); var statement = connection.createStatement())
|
||||
{
|
||||
RdbmsTestSupport.createTables(connection, instance, false);
|
||||
QContext.init(instance, new QSession());
|
||||
lookups();
|
||||
statement.execute("INSERT INTO audit(id,audit_table_id,audit_user_id,timestamp) VALUES(1,1,1,CURRENT_TIMESTAMP - INTERVAL '2' MINUTE)");
|
||||
detail(10, 1);
|
||||
detail(20, 1);
|
||||
var writer = new WormAuditChainer(config);
|
||||
var floors = SweepFloors.of(config);
|
||||
var now = java.time.Instant.now();
|
||||
assertThat(writer.chainOnce(now).audits()).isEqualTo(1);
|
||||
assertThat(STORE.query(WormAuditTables.CHAIN_DETAIL, new com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter())).extracting(r -> r.getValueLong("detail_id") + "@" + r.getValueLong("chain_id"))
|
||||
.containsExactlyInAnyOrder("10@1", "20@1");
|
||||
assertThat(floors.audit()).isZero();
|
||||
assertThat(writer.chainOnce(now.plusSeconds(600)).links()).isZero();
|
||||
assertThat(floors.audit()).isEqualTo(1);
|
||||
assertThat(floors.detail()).isEqualTo(20);
|
||||
detail(30, 1);
|
||||
statement.execute("INSERT INTO audit(id,audit_table_id,audit_user_id,timestamp) VALUES(2,1,1,CURRENT_TIMESTAMP - INTERVAL '2' MINUTE)");
|
||||
detail(40, 2);
|
||||
var counts = writer.chainOnce(now.plusSeconds(1200));
|
||||
assertThat(counts.audits()).isEqualTo(1);
|
||||
assertThat(counts.lateDetails()).isEqualTo(1);
|
||||
assertThat(STORE.query(WormAuditTables.CHAIN_DETAIL, new com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter())).hasSize(4);
|
||||
assertThat(writer.chainOnce(now.plusSeconds(1800)).links()).isZero();
|
||||
assertThat(floors.audit()).isEqualTo(2);
|
||||
assertThat(floors.detail()).isEqualTo(40);
|
||||
// A detail whose id sits below the floor (its transaction committed very late)
|
||||
// is skipped by floored ticks and chained by the next hourly full sweep.
|
||||
statement.execute("INSERT INTO audit_detail(id,audit_id,field_name,new_value) VALUES(5,2,'title','late')");
|
||||
assertThat(writer.chainOnce(now.plusSeconds(2400)).lateDetails()).isZero();
|
||||
assertThat(writer.chainOnce(now.plus(SweepFloors.FULL_SWEEP_EVERY)).lateDetails()).isEqualTo(1);
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).isEmpty();
|
||||
statement.execute("DELETE FROM worm_audit_chain_detail WHERE detail_id=30");
|
||||
statement.execute("INSERT INTO worm_audit_chain_detail(detail_id,chain_id) VALUES(30,1)");
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).anyMatch(b -> b.reason().contains("audit_detail 30 has a detail coverage row that disagrees"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.jdbc.SimpleConnectionProvider;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for rdbms test support.
|
||||
*******************************************************************************/
|
||||
class RdbmsTestSupport
|
||||
{
|
||||
// RC.5 supplies the RDBMS actions and a pluggable strategy. Hosts supply their
|
||||
// PostgreSQL quoting strategy; this fixture uses the core actions unchanged.
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for postgres strategy.
|
||||
*******************************************************************************/
|
||||
public static class PostgresStrategy extends BaseRDBMSActionStrategy
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Verify get identifier quote string.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getIdentifierQuoteString()
|
||||
{
|
||||
return ("\"");
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify get insert default values clause.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getInsertDefaultValuesClause()
|
||||
{
|
||||
return ("DEFAULT VALUES");
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify instance.
|
||||
*******************************************************************************/
|
||||
static QInstance instance(
|
||||
String url, String user, String password, WormAuditQBitConfig config, boolean postgres)
|
||||
throws Exception
|
||||
{
|
||||
com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager.resetConnectionProviders();
|
||||
var backend =
|
||||
new RDBMSBackendMetaData()
|
||||
.withName("db")
|
||||
.withVendor(postgres ? "postgresql" : "h2")
|
||||
.withJdbcUrl(url)
|
||||
.withJdbcDriverClassName(
|
||||
postgres ? "org.postgresql.Driver" : "org.h2.Driver")
|
||||
.withUsername(user)
|
||||
.withPassword(password)
|
||||
.withConnectionProvider(new QCodeReference(SimpleConnectionProvider.class));
|
||||
if(postgres)
|
||||
{
|
||||
backend.withActionStrategyCodeReference(new QCodeReference(PostgresStrategy.class));
|
||||
}
|
||||
QInstance instance = WormAuditTestSupport.core(backend);
|
||||
for(var table : instance.getTables().values())
|
||||
{
|
||||
table.withBackendDetails(
|
||||
new RDBMSTableBackendDetails().withTableName(snake(table.getName())));
|
||||
table.getFields().values().forEach(f -> f.withBackendName(snake(f.getName())));
|
||||
}
|
||||
new WormAuditQBitProducer().withQBitConfig(config.withBackendName("db")).produce(instance);
|
||||
return (instance);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify snake.
|
||||
*******************************************************************************/
|
||||
static String snake(String value)
|
||||
{
|
||||
return (value.replaceAll("([a-z])([A-Z])", "$1_$2").toLowerCase(Locale.ROOT));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify create tables.
|
||||
*******************************************************************************/
|
||||
static void createTables(Connection connection, QInstance instance, boolean coreOnly)
|
||||
throws Exception
|
||||
{
|
||||
for(var table : instance.getTables().values())
|
||||
{
|
||||
if(coreOnly && table.getName().startsWith("worm_audit_"))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
List<String> columns = new ArrayList<>();
|
||||
for(var field : table.getFields().values())
|
||||
{
|
||||
String type =
|
||||
switch(field.getType())
|
||||
{
|
||||
case INTEGER -> "integer";
|
||||
case LONG -> "bigint";
|
||||
case DATE_TIME -> "timestamp(6) with time zone";
|
||||
default -> "text";
|
||||
};
|
||||
if(field.getName().equals(table.getPrimaryKeyField()))
|
||||
{
|
||||
type +=
|
||||
table.getName().equals("worm_audit_chain") || table.getName().equals("worm_audit_chain_detail")
|
||||
? " PRIMARY KEY"
|
||||
: " GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY";
|
||||
}
|
||||
columns.add(
|
||||
WormAuditDdl.identifier(
|
||||
field.getBackendName() == null
|
||||
? field.getName()
|
||||
: field.getBackendName())
|
||||
+ " "
|
||||
+ type);
|
||||
}
|
||||
String name = ((RDBMSTableBackendDetails) table.getBackendDetails()).getTableName();
|
||||
try(var statement = connection.createStatement())
|
||||
{
|
||||
statement.execute(
|
||||
"CREATE TABLE "
|
||||
+ WormAuditDdl.identifier(name)
|
||||
+ " ("
|
||||
+ String.join(",", columns)
|
||||
+ ")");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.model.LinkRangeWidget;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.VerificationRuns;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeEvent;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeListenerInterface;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeType;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
|
||||
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChildRecordListData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for record integration test.
|
||||
*******************************************************************************/
|
||||
class RecordIntegrationTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** All record screens have readable labels and full-width sections and lists.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void recordScreensHaveReadableLabelsAndFullWidthLists() throws Exception
|
||||
{
|
||||
var instance = init(new WormAuditQBitConfig());
|
||||
Map<String, String> formats = Map.of(
|
||||
WormAuditTables.CHAIN, "Link #%s", WormAuditTables.CHAIN_DETAIL, "Detail #%s (link %s)", WormAuditTables.CHECKPOINT, "Checkpoint #%s",
|
||||
WormAuditTables.EXPORT, "Export #%s (links %s–%s)", WormAuditTables.VERIFICATION, "Verification #%s (%s)",
|
||||
WormAuditTables.PURGE, "Purge #%s (%s)", WormAuditTables.SETTINGS, "WORM Audit Settings");
|
||||
for(String name : WormAuditTables.ALL)
|
||||
{
|
||||
var table = instance.getTable(name);
|
||||
assertThat(table.getRecordLabelFormat()).isEqualTo(formats.get(name));
|
||||
assertThat(table.getRecordLabelFields()).allMatch(table.getFields()::containsKey);
|
||||
table.getFields().forEach((fieldName, field) ->
|
||||
assertThat(field.getLabel()).as(name + "." + fieldName).isNotBlank().isNotEqualTo(fieldName).doesNotContain("_"));
|
||||
table.getSections().forEach(section ->
|
||||
{
|
||||
assertThat(section.getGridColumns()).isEqualTo(12);
|
||||
if(section.getWidgetName() != null)
|
||||
{
|
||||
assertThat(instance.getWidget(section.getWidgetName()).getGridColumns()).isEqualTo(12);
|
||||
}
|
||||
});
|
||||
}
|
||||
assertThat(instance.getTable(WormAuditTables.EXPORT).getRecordLabelFields()).containsExactly("id", "from_link_id", "to_link_id");
|
||||
for(String name : List.of(WormAuditTables.VERIFICATION, WormAuditTables.PURGE))
|
||||
{
|
||||
assertThat(instance.getTable(name).getRecordLabelFields()).containsExactly("id", "outcome");
|
||||
}
|
||||
var verification = instance.getTable(WormAuditTables.VERIFICATION);
|
||||
assertThat(verification.getSections()).extracting(section -> section.getLabel()).containsExactly("Result", "Scope", "Run", "Chain links");
|
||||
assertThat(verification.getSections().stream().filter(section -> section.getWidgetName() == null)
|
||||
.flatMap(section -> section.getFieldNames().stream()).toList()).containsExactlyInAnyOrderElementsOf(verification.getFields().keySet());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for event sink.
|
||||
*******************************************************************************/
|
||||
public static class EventSink implements RecordChangeListenerInterface
|
||||
{
|
||||
static List<String> tables = new ArrayList<>();
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify applies to.
|
||||
*******************************************************************************/
|
||||
public boolean appliesTo(String table, RecordChangeType type)
|
||||
{
|
||||
return (List.of(WormAuditTables.VERIFICATION, WormAuditTables.EXPORT).contains(table)
|
||||
&& type == RecordChangeType.INSERT);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify on records changed.
|
||||
*******************************************************************************/
|
||||
public void onRecordsChanged(RecordChangeEvent event)
|
||||
{
|
||||
tables.add(event.getTableName());
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify checkpoint child list uses range and read permissions.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void checkpointChildListUsesRangeAndReadPermissions() throws Exception
|
||||
{
|
||||
var config = ChainVerificationTest.signed().withCheckpointEveryLinks(2);
|
||||
var instance = init(config);
|
||||
audit(1);
|
||||
audit(2);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
var input =
|
||||
new RenderWidgetInput()
|
||||
.withWidgetMetaData(
|
||||
instance.getWidget(WormAuditTables.CHECKPOINT + "_links"));
|
||||
input.setQueryParams(Map.of("id", "1"));
|
||||
assertThatThrownBy(() -> new LinkRangeWidget().render(input)).isInstanceOf(Exception.class);
|
||||
QContext.getQSession()
|
||||
.withPermissions("worm_audit_checkpoint.read", "worm_audit_chain.read");
|
||||
var data = (ChildRecordListData) new LinkRangeWidget().render(input).getWidgetData();
|
||||
assertThat(data.getTotalRows()).isEqualTo(2);
|
||||
assertThat(data.getQueryOutput().getRecords()).hasSize(2);
|
||||
assertThat(data.getViewAllLink()).contains("filter=");
|
||||
String json = com.kingsrook.qqq.backend.core.utils.JsonUtils.toJson(data);
|
||||
assertThat(json).contains("Chain links").doesNotContain("signingKey", "trustedPublicKeys", "sealedStoreSupplier", "privateKey");
|
||||
input.setQueryParams(Map.of("id", "999"));
|
||||
assertThatThrownBy(() -> new LinkRangeWidget().render(input))
|
||||
.hasMessageContaining("was not found");
|
||||
assertThat(
|
||||
instance.getTable(WormAuditTables.EXPORT)
|
||||
.getField("from_checkpoint_id")
|
||||
.getPossibleValueSourceName())
|
||||
.isEqualTo(WormAuditTables.CHECKPOINT);
|
||||
assertThat(
|
||||
instance.getTable(WormAuditTables.VERIFICATION)
|
||||
.getField("export_id")
|
||||
.getPossibleValueSourceName())
|
||||
.isEqualTo(WormAuditTables.EXPORT);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify verification inserts reach core event spi for host esb publications.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void verificationInsertsReachCoreEventSpiForHostEsbPublications() throws Exception
|
||||
{
|
||||
var instance = init(new WormAuditQBitConfig());
|
||||
EventSink.tables.clear();
|
||||
instance.withRecordChangeListener(new QCodeReference(EventSink.class));
|
||||
audit(1);
|
||||
new WormAuditChainer(VerificationRuns.config()).chainOnce();
|
||||
VerificationRuns.run(null, "USER");
|
||||
assertThat(EventSink.tables).containsExactly(WormAuditTables.VERIFICATION);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import org.junit.jupiter.api.extension.ConditionEvaluationResult;
|
||||
import org.junit.jupiter.api.extension.ExecutionCondition;
|
||||
import org.junit.jupiter.api.extension.ExtensionContext;
|
||||
import org.testcontainers.DockerClientFactory;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for requires docker condition.
|
||||
*******************************************************************************/
|
||||
public class RequiresDockerCondition implements ExecutionCondition
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Verify evaluate execution condition.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context)
|
||||
{
|
||||
boolean available;
|
||||
try
|
||||
{
|
||||
available = DockerClientFactory.instance().isDockerAvailable();
|
||||
}
|
||||
catch(Exception | LinkageError e)
|
||||
{
|
||||
available = false;
|
||||
}
|
||||
return (evaluate(available, System.getenv("CI")));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify evaluate.
|
||||
*******************************************************************************/
|
||||
static ConditionEvaluationResult evaluate(boolean available, String ci)
|
||||
{
|
||||
if(available)
|
||||
{
|
||||
return (ConditionEvaluationResult.enabled("Docker is available"));
|
||||
}
|
||||
if(ci != null)
|
||||
{
|
||||
throw new IllegalStateException("Docker is required when CI is set");
|
||||
}
|
||||
return (ConditionEvaluationResult.disabled(
|
||||
"Docker unavailable; coordinator must run PostgreSQL acceptance tests"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,710 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Instant;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.objectstorage.api.ListRequest;
|
||||
import com.kingsrook.qbits.objectstorage.api.ObjectSummary;
|
||||
import com.kingsrook.qbits.objectstorage.api.Page;
|
||||
import com.kingsrook.qbits.objectstorage.api.StoredObject;
|
||||
import com.kingsrook.qbits.wormaudit.canonical.Jcs;
|
||||
import com.kingsrook.qbits.wormaudit.chain.Hashes;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.export.ExportFiles;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedExportVerifier;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedExporter;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedStore;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.ExportProcesses;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunInput;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunOutput;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.STORE;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.detail;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.remove;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.tamper;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Offline sealed-export contract, corruption, retention, batching and UI tests.
|
||||
*******************************************************************************/
|
||||
class SealedExportTest
|
||||
{
|
||||
static final Instant NOW = Instant.parse("2026-10-14T02:00:00Z");
|
||||
WormAuditQBitConfig config;
|
||||
MemorySealedStore storage;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tiny object boundary; bytes exist only in tests.
|
||||
*******************************************************************************/
|
||||
static class MemorySealedStore implements SealedStore
|
||||
{
|
||||
final Map<String, byte[]> objects = new LinkedHashMap<>();
|
||||
final Map<String, StoredObject> heads = new LinkedHashMap<>();
|
||||
final Map<String, Map<String, byte[]>> history = new LinkedHashMap<>();
|
||||
int nextVersion;
|
||||
String missingVersion;
|
||||
boolean omitVersion;
|
||||
boolean racePut;
|
||||
boolean supported = true;
|
||||
String echoMode;
|
||||
long shortenSeconds;
|
||||
int failPut;
|
||||
|
||||
@Override
|
||||
public boolean supportsLock()
|
||||
{
|
||||
return (supported);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String put(String key, Path file, com.kingsrook.qbits.objectstorage.api.LockMode mode, Instant until) throws Exception
|
||||
{
|
||||
if(failPut > 0 && objects.size() + 1 == failPut)
|
||||
{
|
||||
throw (new IllegalStateException("CANARY-PRIVATE-PROVIDER-TEXT"));
|
||||
}
|
||||
if(objects.containsKey(key))
|
||||
{
|
||||
history.computeIfAbsent(key, k -> new LinkedHashMap<>()).put(heads.get(key).getVersionId(), objects.get(key));
|
||||
}
|
||||
String version = omitVersion ? missingVersion : "v" + ++nextVersion;
|
||||
objects.put(key, Files.readAllBytes(file));
|
||||
heads.put(key, new StoredObject().withKey(key).withVersionId(racePut ? "replacement" : version)
|
||||
.withLockMode(echoMode == null ? mode.name() : echoMode).withLockRetainUntil(until.minusSeconds(shortenSeconds)));
|
||||
return (version);
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoredObject head(String key)
|
||||
{
|
||||
return (heads.get(key));
|
||||
}
|
||||
|
||||
@Override
|
||||
public InputStream getVersion(String key, String versionId)
|
||||
{
|
||||
byte[] bytes = versionId.equals(heads.get(key).getVersionId()) ? objects.get(key) : history.get(key).get(versionId);
|
||||
return (new ByteArrayInputStream(bytes));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Page<ObjectSummary> list(ListRequest request)
|
||||
{
|
||||
return (new Page<ObjectSummary>().withItems(objects.keySet().stream().filter(k -> k.startsWith(request.getPrefix()))
|
||||
.map(k -> new ObjectSummary().withKey(k)).toList()));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise setUp.
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp() throws Exception
|
||||
{
|
||||
config = ChainVerificationTest.signed();
|
||||
var instance = init(config);
|
||||
storage = new MemorySealedStore();
|
||||
ProcessesAndAlertsTest.Sink.raised.clear();
|
||||
ProcessesAndAlertsTest.Sink.resolved.clear();
|
||||
ProcessesAndAlertsTest.Sink.beats.clear();
|
||||
AlertsInstanceMetaData.ofOrWithNew(instance).withSink(new QCodeReference(ProcessesAndAlertsTest.Sink.class));
|
||||
}
|
||||
|
||||
|
||||
QRecord seal() throws Exception
|
||||
{
|
||||
new WormAuditChainer(config).chainOnce(NOW);
|
||||
var result = new SealedExporter(config, storage).run("USER", NOW);
|
||||
assertThat(result.failed()).as(result.message()).isFalse();
|
||||
assertThat(result.row()).isNotNull();
|
||||
return (result.row());
|
||||
}
|
||||
|
||||
|
||||
SealedExportVerifier.Report verify(QRecord row) throws Exception
|
||||
{
|
||||
return (new SealedExportVerifier(config, storage).verify(row, "USER"));
|
||||
}
|
||||
|
||||
|
||||
String findings(QRecord row) throws Exception
|
||||
{
|
||||
return (verify(row).result().getBreaks().toString());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise canonicalGoldenFilesAndRoundTrip.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void canonicalGoldenFilesAndRoundTrip() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
QRecord row = seal();
|
||||
String prefix = "worm-audit/2026/10/14/1-1/";
|
||||
assertThat(row.getValueString("object_prefix")).isEqualTo(prefix);
|
||||
String content = "{\"audit\":{\"clientChannel\":null,\"id\":1,\"message\":\"Record was Edited\",\"recordId\":\"17\",\"table\":\"cms_page\",\"timestamp\":\"2020-01-02T03:04:06.123456Z\",\"user\":\"james\"},\"details\":[{\"fieldName\":\"title\",\"id\":1,\"message\":\"Changed Title\",\"newValue\":\"new\",\"oldValue\":\"old\"}],\"v\":1}";
|
||||
String hash = Hashes.link(Hashes.GENESIS, content.getBytes(StandardCharsets.UTF_8));
|
||||
String golden = "{\"content\":" + content + ",\"link\":{\"auditId\":1,\"detailIds\":[1],\"id\":1,\"kind\":\"AUDIT\",\"linkHash\":\"" + hash
|
||||
+ "\",\"prevHash\":\"" + Hashes.GENESIS + "\"}}\n";
|
||||
assertThat(new String(storage.objects.get(prefix + "links.jsonl"), StandardCharsets.UTF_8)).isEqualTo(golden);
|
||||
var manifest = ExportFiles.parse(storage.objects.get(prefix + "manifest.json"));
|
||||
assertThat(manifest.keySet()).containsExactly("checkpointCount", "createdAt", "files", "format", "fromLinkId", "headHash", "linkCount",
|
||||
"prevManifestSha256", "qbitVersion", "toLinkId");
|
||||
assertThat(manifest).containsEntry("format", "worm-audit-export/1").containsEntry("linkCount", 1L).containsEntry("checkpointCount", 1L)
|
||||
.containsEntry("prevManifestSha256", null).containsEntry("headHash", hash).containsEntry("createdAt", NOW.toString());
|
||||
assertThat(storage.objects.get(prefix + "manifest.json")).isEqualTo(Jcs.bytes(manifest));
|
||||
assertThat(row.getValues()).containsKeys("from_checkpoint_id", "to_checkpoint_id", "bucket_alias", "manifest_key", "manifest_sha256",
|
||||
"prev_manifest_sha256", "object_count", "byte_count", "lock_mode", "retain_until", "sealed_at");
|
||||
assertThat(row.getValueLong("byte_count")).isEqualTo(storage.objects.values().stream().mapToLong(b -> b.length).sum());
|
||||
assertThat(row.getValueInstant("retain_until")).isEqualTo(NOW.plusSeconds(30 * 86400L));
|
||||
assertThat(verify(row).result().passed()).isTrue();
|
||||
QRecord verification = STORE.last(WormAuditTables.VERIFICATION);
|
||||
assertThat(verification.getValueString("kind")).isEqualTo("SEALED_EXPORT");
|
||||
assertThat(verification.getValueLong("export_id")).isEqualTo(row.getValueLong("id"));
|
||||
assertThat(verification.getValueString("outcome")).isEqualTo("PASS");
|
||||
assertThat(ProcessesAndAlertsTest.Sink.resolved).hasSize(1);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every overwrite is a break while verification still uses the sealed bytes.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void detectsOverwritesAndReadsSealedVersions() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
QRecord row = seal();
|
||||
Map<String, Object> versions = ExportFiles.parse(row.getValueString("object_versions").getBytes(StandardCharsets.UTF_8));
|
||||
assertThat(versions.keySet()).containsExactly("checkpoints.jsonl", "links.jsonl", "manifest.json");
|
||||
assertThat(row.getValueString("object_versions")).isEqualTo(Jcs.string(versions));
|
||||
assertThat(STORE.get(WormAuditTables.EXPORT, row.getValueLong("id")).getValueString("object_versions"))
|
||||
.isEqualTo(row.getValueString("object_versions"));
|
||||
try(var files = new ExportFiles())
|
||||
{
|
||||
String linksKey = row.getValueString("object_prefix") + "links.jsonl";
|
||||
Files.write(files.file("links.jsonl"), storage.objects.get(linksKey));
|
||||
storage.put(linksKey, files.file("links.jsonl"), com.kingsrook.qbits.objectstorage.api.LockMode.valueOf(row.getValueString("lock_mode")), row.getValueInstant("retain_until"));
|
||||
var identical = verify(row);
|
||||
assertThat(identical.result().passed()).isFalse();
|
||||
assertThat(identical.result().getLinksChecked()).isEqualTo(1);
|
||||
assertThat(identical.result().getBreaks()).singleElement().satisfies(b ->
|
||||
assertThat(b.reason()).contains("object links.jsonl was overwritten after sealing"));
|
||||
for(String name : ExportFiles.NAMES)
|
||||
{
|
||||
String key = row.getValueString("object_prefix") + name;
|
||||
Files.writeString(files.file(name), "different bytes");
|
||||
String current = storage.put(key, files.file(name), com.kingsrook.qbits.objectstorage.api.LockMode.valueOf(row.getValueString("lock_mode")), row.getValueInstant("retain_until"));
|
||||
var report = verify(row);
|
||||
assertThat(report.result().passed()).isFalse();
|
||||
assertThat(report.result().getLinksChecked()).isEqualTo(1);
|
||||
assertThat(report.result().getBreaks()).allSatisfy(b -> assertThat(b.reason()).contains("was overwritten after sealing"));
|
||||
assertThat(report.result().getBreaks().toString()).contains("object " + name + " was overwritten after sealing (current version "
|
||||
+ current + ", sealed version " + versions.get(name) + " is still retained under Object Lock)");
|
||||
}
|
||||
}
|
||||
var load = new ExportProcesses.VerifyLoad(storage);
|
||||
load.runOnePage(new RunBackendStepInput().withRecords(List.of(row)), new RunBackendStepOutput());
|
||||
assertThat(load.getProcessSummary(new RunBackendStepOutput(), true).toString()).contains("ERROR", "was overwritten after sealing");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unversioned uploads and upload/HEAD races cannot create a sealed catalog row.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void refusesMissingVersionsAndUploadRaces() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
new WormAuditChainer(config).chainOnce(NOW);
|
||||
storage.omitVersion = true;
|
||||
for(String version : java.util.Arrays.asList(null, "", " ", "null"))
|
||||
{
|
||||
storage.missingVersion = version;
|
||||
var result = new SealedExporter(config, storage).run("USER", NOW);
|
||||
assertThat(result.failed()).isTrue();
|
||||
assertThat(result.message()).contains("returned no version id");
|
||||
assertThat(STORE.all(WormAuditTables.EXPORT)).isEmpty();
|
||||
}
|
||||
storage.omitVersion = false;
|
||||
storage.racePut = true;
|
||||
assertThat(new SealedExporter(config, storage).run("USER", NOW).message()).contains("version changed after upload");
|
||||
assertThat(STORE.all(WormAuditTables.EXPORT)).isEmpty();
|
||||
assertThat(ProcessesAndAlertsTest.Sink.raised).hasSize(5);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Missing catalog versions must never silently fall back to current content.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void refusesMissingRecordedVersion() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
QRecord row = seal();
|
||||
row.setValue("object_versions", "{}");
|
||||
assertThat(findings(row)).contains("object links.jsonl has no recorded sealed version id");
|
||||
row.setValue("object_versions", null);
|
||||
assertThat(verify(row).result().passed()).isFalse();
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise rangeSelectionNoCheckpointContiguousAndPurgeAnchor.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void rangeSelectionNoCheckpointContiguousAndPurgeAnchor() throws Exception
|
||||
{
|
||||
assertThat(new SealedExporter(config, storage).run("SCHEDULE", NOW).message()).isEqualTo("Nothing to seal: no new checkpoint");
|
||||
assertThat(ProcessesAndAlertsTest.Sink.beats).containsExactly("wormaudit.export.daily");
|
||||
audit(1);
|
||||
QRecord first = seal();
|
||||
assertThat(new SealedExporter(config, storage).run("USER", NOW).row()).isNull();
|
||||
audit(2);
|
||||
QRecord second = seal();
|
||||
assertThat(second.getValueLong("from_link_id")).isEqualTo(2L);
|
||||
assertThat(second.getValueString("prev_manifest_sha256")).isEqualTo(first.getValueString("manifest_sha256"));
|
||||
assertThat(verify(second).result().passed()).isTrue();
|
||||
|
||||
init(config);
|
||||
STORE.insert(WormAuditTables.PURGE, List.of(new QRecord().withValue("outcome", "PURGED").withValue("anchor_link_id", 10L).withValue("anchor_hash", Hashes.GENESIS)));
|
||||
audit(11);
|
||||
QRecord anchored = seal();
|
||||
assertThat(anchored.getValueLong("from_link_id")).isEqualTo(11L);
|
||||
assertThat(anchored.getValueString("prev_manifest_sha256")).isNull();
|
||||
assertThat(verify(anchored).result().passed()).isTrue();
|
||||
STORE.insert(WormAuditTables.PURGE, List.of(new QRecord().withValue("outcome", "PURGED").withValue("anchor_link_id", 20L).withValue("anchor_hash", Hashes.GENESIS)));
|
||||
assertThatThrownBy(() -> SealedExporter.scope(STORE)).hasMessage("Unexported links were purged");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise refusesUnsupportedAndIncorrectLockEchoAndPartialUpload.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void refusesUnsupportedAndIncorrectLockEchoAndPartialUpload() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
new WormAuditChainer(config).chainOnce(NOW);
|
||||
storage.supported = false;
|
||||
assertThat(new SealedExporter(config, storage).run("SCHEDULE", NOW).message()).contains("does not support Object Lock");
|
||||
assertThat(storage.objects).isEmpty();
|
||||
storage.supported = true;
|
||||
storage.echoMode = "COMPLIANCE";
|
||||
assertThat(new SealedExporter(config, storage).run("USER", NOW).message()).contains("did not echo GOVERNANCE");
|
||||
storage.echoMode = null;
|
||||
storage.shortenSeconds = 1;
|
||||
assertThat(new SealedExporter(config, storage).run("USER", NOW).failed()).isTrue();
|
||||
storage.shortenSeconds = 0;
|
||||
storage.objects.clear();
|
||||
storage.failPut = 2;
|
||||
assertThat(new SealedExporter(config, storage).run("SCHEDULE", NOW).message()).doesNotContain("CANARY").contains("storage or database");
|
||||
assertThat(STORE.all(WormAuditTables.EXPORT)).isEmpty();
|
||||
assertThat(ProcessesAndAlertsTest.Sink.raised).hasSize(4);
|
||||
assertThat(ProcessesAndAlertsTest.Sink.beats).hasSize(2);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise tamperedObjectsBrokenManifestChainAndLockRemoval.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void tamperedObjectsBrokenManifestChainAndLockRemoval() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
QRecord first = seal();
|
||||
String key = first.getValueString("object_prefix") + "links.jsonl";
|
||||
byte[] original = storage.objects.get(key);
|
||||
storage.objects.put(key, "tampered".getBytes(StandardCharsets.UTF_8));
|
||||
assertThat(findings(first)).contains("object links.jsonl hash does not match the manifest");
|
||||
storage.objects.put(key, original);
|
||||
storage.heads.get(key).withLockMode("COMPLIANCE");
|
||||
assertThat(findings(first)).contains("lacks the recorded GOVERNANCE retention");
|
||||
storage.heads.get(key).withLockMode(first.getValueString("lock_mode"));
|
||||
audit(2);
|
||||
QRecord second = seal();
|
||||
tamper(WormAuditTables.EXPORT, first.getValueLong("id"), "manifest_sha256", "f".repeat(64));
|
||||
assertThat(findings(second)).contains("prevManifestSha256 does not match");
|
||||
assertThat(findings(STORE.get(WormAuditTables.EXPORT, first.getValueLong("id")))).contains("manifest.json hash does not match");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise exactLiveDifferencesMissingRowsAndPurgedCopies.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void exactLiveDifferencesMissingRowsAndPurgedCopies() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
audit(2);
|
||||
QRecord row = seal();
|
||||
tamper("auditDetail", 1, "newValue", "CANARY-PRIVATE");
|
||||
assertThat(findings(row)).contains("link 1 (audit 1): live content differs from the sealed copy (sealed 2026-10-14)").doesNotContain("CANARY");
|
||||
remove(WormAuditTables.CHAIN, 2);
|
||||
assertThat(findings(row)).contains("link 2 is missing from the live chain");
|
||||
tamper(WormAuditTables.CHAIN, 1, "kind", "LATE_DETAIL");
|
||||
assertThat(findings(row)).contains("live chain kind differs");
|
||||
remove("audit", 1);
|
||||
assertThat(findings(row)).contains("live audit header is missing");
|
||||
String head = (String) ExportFiles.parse(storage.objects.get(row.getValueString("manifest_key"))).get("headHash");
|
||||
STORE.insert(WormAuditTables.PURGE, List.of(new QRecord().withValue("outcome", "PURGED").withValue("anchor_link_id", 2L).withValue("anchor_hash", head)));
|
||||
remove(WormAuditTables.CHAIN, 1);
|
||||
remove("audit", 2);
|
||||
var report = verify(row);
|
||||
assertThat(report.result().passed()).isTrue();
|
||||
assertThat(report.purged()).isEqualTo(2);
|
||||
var load = new ExportProcesses.VerifyLoad(storage);
|
||||
load.runOnePage(new RunBackendStepInput().withRecords(List.of(row)), new RunBackendStepOutput());
|
||||
assertThat(load.getProcessSummary(new RunBackendStepOutput(), true).toString()).contains("INFO", "2 links verified from the sealed copy only");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise refusesBadSourceAndDetectsMissingLiveDetails.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void refusesBadSourceAndDetectsMissingLiveDetails() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
new WormAuditChainer(config).chainOnce(NOW);
|
||||
tamper("auditDetail", 1, "newValue", "changed");
|
||||
assertThat(new SealedExporter(config, storage).run("USER", NOW).message()).contains("content differs at link 1");
|
||||
assertThat(storage.objects).isEmpty();
|
||||
tamper("auditDetail", 1, "newValue", "new");
|
||||
QRecord row = seal();
|
||||
remove("auditDetail", 1);
|
||||
assertThat(findings(row)).contains("link 1 (audit 1): live content is missing or invalid");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise spansPagesLateDetailsAndCheckpointBoundaries.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void spansPagesLateDetailsAndCheckpointBoundaries() throws Exception
|
||||
{
|
||||
config.withBatchSize(1100).withCheckpointEveryLinks(500);
|
||||
for(long id = 1; id <= 1100; id++)
|
||||
{
|
||||
audit(id);
|
||||
}
|
||||
new WormAuditChainer(config).chainOnce(NOW);
|
||||
// The final 100 links need the hourly checkpoint.
|
||||
new WormAuditChainer(config).chainOnce(NOW.plusSeconds(3601));
|
||||
QRecord row = seal();
|
||||
assertThat(row.getValueLong("to_link_id")).isEqualTo(1100);
|
||||
assertThat(verify(row).result().getLinksChecked()).isEqualTo(1100);
|
||||
detail(1, 1);
|
||||
new WormAuditChainer(config).chainOnce(NOW.plusSeconds(7202));
|
||||
new WormAuditChainer(config).chainOnce(NOW.plusSeconds(10803));
|
||||
QRecord late = seal();
|
||||
assertThat(late.getValueLong("from_link_id")).isEqualTo(1101);
|
||||
assertThat(verify(late).result().passed()).isTrue();
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise reviewLoadSelectionAndSchedulesUseStandardEtl.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void reviewLoadSelectionAndSchedulesUseStandardEtl() throws Exception
|
||||
{
|
||||
var instance = com.kingsrook.qqq.backend.core.context.QContext.getQInstance();
|
||||
assertThat(config.getExportBucketAlias()).isEqualTo("worm_audit");
|
||||
assertThat(config.getExportPrefix()).isEqualTo("worm-audit");
|
||||
assertThat(instance.getProcess(ExportProcesses.EXPORT).getTableName()).isEqualTo(WormAuditTables.EXPORT);
|
||||
assertThat(instance.getProcess(ExportProcesses.VERIFY).getMinInputRecords()).isEqualTo(1);
|
||||
assertThat(instance.getProcess(ExportProcesses.DAILY).getSchedule().getCronTimeZoneId()).isEqualTo("UTC");
|
||||
assertThat(instance.getProcess(ExportProcesses.DAILY).getSchedule().getCronExpression()).isEqualTo("0 0 2 * * ?");
|
||||
var output = new RunBackendStepOutput();
|
||||
var input = new RunBackendStepInput().withRecords(List.of(new QRecord().withValue("id", 0L)));
|
||||
var extract = new ExportProcesses.ExportExtract();
|
||||
assertThat(extract.doCount(input)).isEqualTo(1);
|
||||
extract.setRecordPipe(new com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe());
|
||||
extract.run(input, output);
|
||||
var review = new ExportProcesses.ExportReview();
|
||||
review.runOnePage(input, output);
|
||||
assertThat(review.getProcessSummary(output, false).toString()).contains("Nothing to seal");
|
||||
audit(1);
|
||||
new WormAuditChainer(config).chainOnce(NOW);
|
||||
review.runOnePage(input, output);
|
||||
assertThat(review.getProcessSummary(output, false).toString()).contains("1 chain links will be sealed", "1 checkpoints");
|
||||
assertThat(STORE.all(WormAuditTables.EXPORT)).isEmpty();
|
||||
var load = new ExportProcesses.ExportLoad(storage);
|
||||
assertThat(load.getProcessSummary(output, true)).isEmpty();
|
||||
load.runOnePage(input, output);
|
||||
load.postRun(new BackendStepPostRunInput(input), new BackendStepPostRunOutput(output));
|
||||
assertThat(load.getProcessSummary(output, true).toString()).contains("1 links sealed in export");
|
||||
var rows = STORE.all(WormAuditTables.EXPORT);
|
||||
var verifyInput = new RunBackendStepInput().withRecords(rows).addValue("recordIds", rows.getFirst().getValueString("id"));
|
||||
assertThat(new ExportProcesses.VerifyExtract().getQueryFilter(verifyInput).getCriteria()).isNotEmpty();
|
||||
var verifyReview = new ExportProcesses.VerifyReview();
|
||||
verifyReview.runOnePage(verifyInput, output);
|
||||
assertThat(verifyReview.getProcessSummary(output, false).toString()).contains("1 sealed exports");
|
||||
var verifyLoad = new ExportProcesses.VerifyLoad(storage);
|
||||
verifyLoad.runOnePage(verifyInput, output);
|
||||
assertThat(verifyLoad.getProcessSummary(output, true).toString()).contains("1 links verified");
|
||||
new ExportProcesses.Daily().run(input, output);
|
||||
assertThat(output.getValueString("outcome")).isEqualTo("PASS");
|
||||
audit(2);
|
||||
new WormAuditChainer(config).chainOnce(NOW);
|
||||
new ExportProcesses.Daily().run(input, output);
|
||||
assertThat(output.getValueString("outcome")).isEqualTo("FAIL");
|
||||
assertThat(ProcessesAndAlertsTest.Sink.beats).contains("wormaudit.export.daily");
|
||||
new ExportProcesses.ExportLoad();
|
||||
new ExportProcesses.VerifyLoad();
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise malformedJsonIsValueFreeAndConfigRejectsUnsafePrefixes.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void malformedJsonIsValueFreeAndConfigRejectsUnsafePrefixes() throws Exception
|
||||
{
|
||||
assertThatThrownBy(() -> ExportFiles.parse("{\"x\":\"CANARY\",\"x\":2}".getBytes(StandardCharsets.UTF_8)))
|
||||
.hasMessage("Invalid sealed JSON").hasNoCause();
|
||||
assertThatThrownBy(() -> config.withExportPrefix("../escape").validate()).isInstanceOf(IllegalArgumentException.class);
|
||||
config.withExportPrefix("evidence/audit").withExportBucketAlias("backup").withExportCron("0 5 2 * * ?").validate();
|
||||
assertThat(config.getExportBucketAlias()).isEqualTo("backup");
|
||||
try(var files = new ExportFiles())
|
||||
{
|
||||
assertThatThrownBy(() -> files.file("../escape")).isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Re-hash a tampered object and catalog to reach the independent signature checks.
|
||||
*******************************************************************************/
|
||||
QRecord rewrite(QRecord row, String name, byte[] bytes) throws Exception
|
||||
{
|
||||
String prefix = row.getValueString("object_prefix");
|
||||
storage.objects.put(prefix + name, bytes);
|
||||
Map<String, Object> manifest = ExportFiles.parse(storage.objects.get(prefix + "manifest.json"));
|
||||
Map<String, Object> hashes = new LinkedHashMap<>((Map<String, Object>) manifest.get("files"));
|
||||
hashes.put(name, java.util.HexFormat.of().formatHex(Hashes.digest(bytes)));
|
||||
manifest.put("files", hashes);
|
||||
byte[] encoded = Jcs.bytes(manifest);
|
||||
storage.objects.put(prefix + "manifest.json", encoded);
|
||||
tamper(WormAuditTables.EXPORT, row.getValueLong("id"), "manifest_sha256", java.util.HexFormat.of().formatHex(Hashes.digest(encoded)));
|
||||
return (STORE.get(WormAuditTables.EXPORT, row.getValueLong("id")));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise independentSignaturesMerkleAndSealedContentChecks.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void independentSignaturesMerkleAndSealedContentChecks() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
QRecord row = seal();
|
||||
String prefix = row.getValueString("object_prefix");
|
||||
byte[] originalCheckpoint = storage.objects.get(prefix + "checkpoints.jsonl");
|
||||
Map<String, Object> checkpoint = ExportFiles.parse(originalCheckpoint);
|
||||
checkpoint.put("signature", java.util.Base64.getEncoder().encodeToString(new byte[64]));
|
||||
row = rewrite(row, "checkpoints.jsonl", (Jcs.string(checkpoint) + "\n").getBytes(StandardCharsets.UTF_8));
|
||||
assertThat(findings(row)).contains("checkpoint 1: signature does not verify");
|
||||
checkpoint = ExportFiles.parse(originalCheckpoint);
|
||||
checkpoint.put("merkle_root", "a".repeat(64));
|
||||
row = rewrite(row, "checkpoints.jsonl", (Jcs.string(checkpoint) + "\n").getBytes(StandardCharsets.UTF_8));
|
||||
assertThat(findings(row)).contains("Merkle root or head hash differs");
|
||||
row = rewrite(row, "checkpoints.jsonl", originalCheckpoint);
|
||||
Map<String, Object> document = ExportFiles.parse(storage.objects.get(prefix + "links.jsonl"));
|
||||
Map<String, Object> content = (Map<String, Object>) document.get("content");
|
||||
((Map<String, Object>) content.get("audit")).put("message", "CANARY-SEALED");
|
||||
row = rewrite(row, "links.jsonl", (Jcs.string(document) + "\n").getBytes(StandardCharsets.UTF_8));
|
||||
assertThat(findings(row)).contains("sealed content differs from the chained hash").doesNotContain("CANARY");
|
||||
storage.objects.remove(prefix + "links.jsonl");
|
||||
assertThat(findings(row)).contains("cannot be read or has invalid structure").doesNotContain("CANARY");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise concurrentExportersDoNotOverlapAndMultipleSelectionsPersistSeparately.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void concurrentExportersDoNotOverlapAndMultipleSelectionsPersistSeparately() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
new WormAuditChainer(config).chainOnce(NOW);
|
||||
var instance = com.kingsrook.qqq.backend.core.context.QContext.getQInstance();
|
||||
var workers = java.util.concurrent.Executors.newFixedThreadPool(2);
|
||||
try
|
||||
{
|
||||
var jobs = new java.util.ArrayList<java.util.concurrent.Future<SealedExporter.Result>>();
|
||||
for(int i = 0; i < 2; i++)
|
||||
{
|
||||
jobs.add(workers.submit(() ->
|
||||
{
|
||||
com.kingsrook.qqq.backend.core.context.QContext.init(instance, new com.kingsrook.qqq.backend.core.model.session.QSession());
|
||||
try
|
||||
{
|
||||
return (new SealedExporter(config, storage).run("USER", NOW));
|
||||
}
|
||||
finally
|
||||
{
|
||||
com.kingsrook.qqq.backend.core.context.QContext.clear();
|
||||
}
|
||||
}));
|
||||
}
|
||||
int sealed = 0;
|
||||
for(var job : jobs)
|
||||
{
|
||||
var result = job.get(10, java.util.concurrent.TimeUnit.SECONDS);
|
||||
assertThat(result.failed()).as(result.message()).isFalse();
|
||||
sealed += result.row() == null ? 0 : 1;
|
||||
}
|
||||
assertThat(sealed).isEqualTo(1);
|
||||
}
|
||||
finally
|
||||
{
|
||||
workers.shutdownNow();
|
||||
}
|
||||
audit(2);
|
||||
seal();
|
||||
var load = new ExportProcesses.VerifyLoad(storage);
|
||||
load.runOnePage(new RunBackendStepInput().withRecords(STORE.all(WormAuditTables.EXPORT)), new RunBackendStepOutput());
|
||||
assertThat(STORE.all(WormAuditTables.VERIFICATION)).hasSize(2)
|
||||
.allMatch(r -> "PASS".equals(r.getValueString("outcome")));
|
||||
assertThat(STORE.all(WormAuditTables.VERIFICATION)).extracting(r -> r.getValueLong("export_id")).doesNotHaveDuplicates();
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise h2TransactionRoundTripAndFailedPutHasNoCatalogRow.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void h2TransactionRoundTripAndFailedPutHasNoCatalogRow() throws Exception
|
||||
{
|
||||
String url = "jdbc:h2:mem:exports_" + java.util.UUID.randomUUID() + ";DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE";
|
||||
var instance = RdbmsTestSupport.instance(url, "sa", "", config, false);
|
||||
try(var connection = java.sql.DriverManager.getConnection(url, "sa", ""))
|
||||
{
|
||||
RdbmsTestSupport.createTables(connection, instance, false);
|
||||
com.kingsrook.qqq.backend.core.context.QContext.init(instance, new com.kingsrook.qqq.backend.core.model.session.QSession());
|
||||
WormAuditTestSupport.lookups();
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
QRecord row = seal();
|
||||
assertThat(verify(row).result().passed()).isTrue();
|
||||
assertThat(verify(STORE.get(WormAuditTables.EXPORT, row.getValueLong("id"))).result().passed()).isTrue();
|
||||
connection.createStatement().execute("UPDATE audit_detail SET new_value='changed' WHERE id=1");
|
||||
assertThat(findings(row)).contains("link 1 (audit 1): live content differs");
|
||||
audit(2);
|
||||
new WormAuditChainer(config).chainOnce(NOW);
|
||||
storage.failPut = storage.objects.size() + 1;
|
||||
assertThat(new SealedExporter(config, storage).run("USER", NOW).failed()).isTrue();
|
||||
assertThat(STORE.all(WormAuditTables.EXPORT)).hasSize(1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise realAdapterRequiresBackupRoleAndPositiveCapability.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void realAdapterRequiresBackupRoleAndPositiveCapability() throws Exception
|
||||
{
|
||||
var bucket = new com.kingsrook.qbits.objectstorage.config.StorageBucketConfig("worm_audit")
|
||||
.withConnection("test").withBucketName("test-bucket").withIndexed(false)
|
||||
.withRole(com.kingsrook.qbits.objectstorage.config.BucketRole.BACKUP);
|
||||
System.setProperty("worm.unit.access", "unit-access");
|
||||
System.setProperty("worm.unit.secret", "unit-secret");
|
||||
var server = com.sun.net.httpserver.HttpServer.create(new java.net.InetSocketAddress("127.0.0.1", 0), 0);
|
||||
var responseVersion = new java.util.concurrent.atomic.AtomicReference<>("sealed-version");
|
||||
var requested = new java.util.concurrent.CopyOnWriteArrayList<String>();
|
||||
server.createContext("/test-bucket/links.jsonl", exchange ->
|
||||
{
|
||||
requested.add(exchange.getRequestURI().getRawQuery());
|
||||
exchange.getResponseHeaders().add("x-amz-version-id", responseVersion.get());
|
||||
byte[] bytes = "sealed fixture".getBytes(StandardCharsets.UTF_8);
|
||||
exchange.sendResponseHeaders(200, bytes.length);
|
||||
try(var output = exchange.getResponseBody())
|
||||
{
|
||||
output.write(bytes);
|
||||
}
|
||||
});
|
||||
server.start();
|
||||
try
|
||||
{
|
||||
com.kingsrook.qbits.objectstorage.api.ObjectStorage.configureStandalone(
|
||||
new com.kingsrook.qbits.objectstorage.ObjectStorageQBitConfig().withBackendName("memory")
|
||||
.withAllowedSecretRefPatterns(List.of("prop.worm.unit.*"))
|
||||
.withConnection(new com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig("test")
|
||||
.withEndpoint("http://127.0.0.1:" + server.getAddress().getPort()).withRegion("us-east-1")
|
||||
.withAddressingStyle(com.kingsrook.qbits.objectstorage.config.AddressingStyle.PATH)
|
||||
.withProviderType(com.kingsrook.qbits.objectstorage.config.ProviderType.MINIO)
|
||||
.withAccessKeyRef("${prop.worm.unit.access}").withSecretKeyRef("${prop.worm.unit.secret}"))
|
||||
.withBucket(bucket));
|
||||
var adapter = new com.kingsrook.qbits.wormaudit.export.ObjectStorageSealedStore("worm_audit", "unit");
|
||||
assertThat(adapter.supportsLock()).isFalse();
|
||||
com.kingsrook.qbits.objectstorage.internal.CapabilityStore.record(bucket,
|
||||
com.kingsrook.qbits.objectstorage.api.Capability.OBJECT_LOCK,
|
||||
com.kingsrook.qbits.objectstorage.api.CapabilityStatus.SUPPORTED, "unit fixture");
|
||||
assertThat(adapter.supportsLock()).isTrue();
|
||||
try(var input = adapter.getVersion("links.jsonl", "sealed-version"))
|
||||
{
|
||||
assertThat(new String(input.readAllBytes(), StandardCharsets.UTF_8)).isEqualTo("sealed fixture");
|
||||
}
|
||||
responseVersion.set("different-version");
|
||||
assertThatThrownBy(() -> adapter.getVersion("links.jsonl", "sealed-version"))
|
||||
.hasMessage("Storage returned a different object version");
|
||||
assertThatThrownBy(() -> adapter.getVersion("links.jsonl", null)).hasMessage("Missing sealed object version");
|
||||
assertThat(requested).containsExactly("versionId=sealed-version", "versionId=sealed-version");
|
||||
bucket.withRole(com.kingsrook.qbits.objectstorage.config.BucketRole.PRIVATE);
|
||||
assertThat(adapter.supportsLock()).isFalse();
|
||||
}
|
||||
finally
|
||||
{
|
||||
server.stop(0);
|
||||
com.kingsrook.qbits.objectstorage.internal.ObjectStorageRuntime.reset();
|
||||
System.clearProperty("worm.unit.access");
|
||||
System.clearProperty("worm.unit.secret");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QAuthenticationException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
|
||||
import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleInterface;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test authentication module that behaves like the token-based core modules
|
||||
** (OAuth2, Auth0): a QSystemUserSession is always valid, a session it created
|
||||
** is valid, and any other session has no token and fails validation with the
|
||||
** same message the OAuth2 module produces (JWTDecodeException: The token is
|
||||
** null).
|
||||
*******************************************************************************/
|
||||
public class TokenRequiredAuthenticationModule implements QAuthenticationModuleInterface
|
||||
{
|
||||
private static final Set<String> ISSUED = ConcurrentHashMap.newKeySet();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Issue a session that carries a token.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QSession createSession(QInstance qInstance, Map<String, String> context) throws QAuthenticationException
|
||||
{
|
||||
QSession session = new QSession();
|
||||
ISSUED.add(session.getUuid());
|
||||
return (session);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Accept the system user and issued sessions; reject tokenless sessions.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean isSessionValid(QInstance instance, QSession session)
|
||||
{
|
||||
if(session instanceof QSystemUserSession || ISSUED.contains(session.getUuid()))
|
||||
{
|
||||
return (true);
|
||||
}
|
||||
throw (new IllegalStateException("The token is null"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.ExportProcesses;
|
||||
import com.kingsrook.qbits.wormaudit.processes.WormAuditProcesses;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Regression: production must work on a host whose default authentication
|
||||
** rejects tokenless sessions (kof22 uses OAuth2). produce() reads the settings
|
||||
** singleton for the startup schedules; under a plain new QSession() that read
|
||||
** failed with "The token is null" and the host could not start.
|
||||
*******************************************************************************/
|
||||
class TokenRequiredHostTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Clear the context and the memory store.
|
||||
*******************************************************************************/
|
||||
@AfterEach
|
||||
void tearDown()
|
||||
{
|
||||
MemoryRecordStore.fullReset();
|
||||
QContext.clear();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The test module really rejects a tokenless session, so the tests below
|
||||
** prove the qbit does not use one.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void moduleRejectsTokenlessSession() throws Exception
|
||||
{
|
||||
MemoryRecordStore.fullReset();
|
||||
QInstance instance = tokenRequiredHost(new WormAuditQBitConfig());
|
||||
QContext.init(instance, new QSession());
|
||||
assertThatThrownBy(() -> new AuditStore(null).get(WormAuditTables.SETTINGS, 1))
|
||||
.hasStackTraceContaining("The token is null");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** produce() succeeds and still honors the persisted settings row for the
|
||||
** startup schedules, read as the system user.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void produceReadsPersistedSettingsAsSystemUser() throws Exception
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// first start: an anonymous host stores an edited settings row //
|
||||
////////////////////////////////////////////////////////////////////
|
||||
MemoryRecordStore.fullReset();
|
||||
WormAuditQBitConfig seedConfig = new WormAuditQBitConfig();
|
||||
QInstance seedHost = WormAuditTestSupport.core(memory());
|
||||
new WormAuditQBitProducer().withQBitConfig(seedConfig.withBackendName("memory")).produce(seedHost);
|
||||
QContext.init(seedHost, new QSession());
|
||||
QRecord row = WormAuditSettings.defaults(seedConfig)
|
||||
.withValue("chain_interval_seconds", 45)
|
||||
.withValue("verify_cron", "0 15 4 * * ?")
|
||||
.withValue("export_cron", "0 45 1 * * ?");
|
||||
new AuditStore(null).insert(WormAuditTables.SETTINGS, List.of(row));
|
||||
QContext.clear();
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// restart on a token-required host, with no session in context, as //
|
||||
// when a host builds its instance before any request arrives //
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
QInstance instance = tokenRequiredHost(new WormAuditQBitConfig());
|
||||
|
||||
assertThat(instance.getProcess(WormAuditProcesses.CHAIN).getSchedule().getRepeatSeconds()).isEqualTo(45);
|
||||
assertThat(instance.getProcess(WormAuditProcesses.DAILY).getSchedule().getCronExpression()).isEqualTo("0 15 4 * * ?");
|
||||
assertThat(instance.getProcess(ExportProcesses.DAILY).getSchedule().getCronExpression()).isEqualTo("0 45 1 * * ?");
|
||||
assertThat(QContext.getQSession()).isNull();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** With no settings row, production falls back to the configured values.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void produceFallsBackToConfigWithoutRow() throws Exception
|
||||
{
|
||||
MemoryRecordStore.fullReset();
|
||||
QInstance instance = tokenRequiredHost(new WormAuditQBitConfig().withChainIntervalSeconds(17));
|
||||
assertThat(instance.getProcess(WormAuditProcesses.CHAIN).getSchedule().getRepeatSeconds()).isEqualTo(17);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build a host whose default authentication requires a token, and produce.
|
||||
*******************************************************************************/
|
||||
private static QInstance tokenRequiredHost(WormAuditQBitConfig config) throws Exception
|
||||
{
|
||||
QInstance instance = WormAuditTestSupport.core(memory());
|
||||
instance.withInstanceDefaultAuthentication(new QAuthenticationMetaData()
|
||||
.withName("token-required")
|
||||
.withType(QAuthenticationType.MOCK)
|
||||
.withModuleCodeReference(new QCodeReference(TokenRequiredAuthenticationModule.class)));
|
||||
new WormAuditQBitProducer().withQBitConfig(config.withBackendName("memory")).produce(instance);
|
||||
return (instance);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The memory backend.
|
||||
*******************************************************************************/
|
||||
private static QBackendMetaData memory()
|
||||
{
|
||||
return (new QBackendMetaData().withName("memory").withBackendType(MemoryBackendModule.class));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,366 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import java.sql.DriverManager;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
|
||||
import com.kingsrook.qbits.wormaudit.app.WormAuditApp;
|
||||
import com.kingsrook.qbits.wormaudit.app.WormAuditHomeData;
|
||||
import com.kingsrook.qbits.wormaudit.app.WormAuditHomeView;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.demo.MemorySealedStore;
|
||||
import com.kingsrook.qbits.wormaudit.demo.WormAuditDemoServer;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedExportVerifier;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedExporter;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.VerificationRuns;
|
||||
import com.kingsrook.qbits.wormaudit.verify.VerificationResult;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
|
||||
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
|
||||
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.OLD;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.STORE;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.detail;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.lookups;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.tamper;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Home acceptance: truthful summaries, permissions, privacy and real demo data.
|
||||
*******************************************************************************/
|
||||
class WormAuditHomeTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Empty tables do not imply successful verification or export coverage.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void emptyStatesAndWarnings() throws Exception
|
||||
{
|
||||
var config = new WormAuditQBitConfig();
|
||||
init(config);
|
||||
var data = WormAuditHomeData.load(config, new MemorySealedStore(), OLD);
|
||||
assertThat(data.links()).isZero();
|
||||
assertThat(data.backlog()).isZero();
|
||||
assertThat(data.settling()).isZero();
|
||||
assertThat(data.anchor()).isNull();
|
||||
assertThat(data.oldestAudit()).isNull();
|
||||
assertThat(data.nextPurge()).isNull();
|
||||
assertThat(data.exportAgeHours()).isNull();
|
||||
assertThat(data.warnings()).contains("No signing key configured", "No trusted public keys configured", "Daily verification has never run");
|
||||
String html = WormAuditHomeView.render(data, "/app");
|
||||
assertThat(html).contains("Chain status", "Export status", "Retention", "Recent purges", "Setup warnings",
|
||||
"No links chained yet", "No verification has run yet", "Unknown until verification runs", "No sealed exports yet",
|
||||
"No sealed ranges to check", "No live audits", "No purge requests yet");
|
||||
assertThat(WormAuditHomeView.renderHealthCard(data, "/app/worm_audit")).contains("Not verified yet");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Strict settle boundary and AUDIT anti-join, including late-detail links.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void chainBacklogRetentionAndRealFailureRecovery() throws Exception
|
||||
{
|
||||
var config = ChainVerificationTest.signed();
|
||||
init(config);
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
Instant now = OLD.plusSeconds(120);
|
||||
new WormAuditChainer(config).chainOnce(now);
|
||||
audit(2);
|
||||
audit(3);
|
||||
tamper("audit", 3, "timestamp", now.minusSeconds(60));
|
||||
var data = WormAuditHomeData.load(config, new MemorySealedStore(), now);
|
||||
assertThat(data.links()).isEqualTo(1);
|
||||
assertThat(data.lastLinkAt()).isEqualTo(now);
|
||||
assertThat(data.backlog()).isEqualTo(1);
|
||||
assertThat(data.settling()).isEqualTo(1);
|
||||
assertThat(data.oldestAudit()).isEqualTo(OLD.plusSeconds(1));
|
||||
assertThat(data.nextPurge()).isEqualTo(OLD.plusSeconds(1).atZone(ZoneOffset.UTC).plusYears(6).toInstant());
|
||||
new WormAuditChainer(config).chainOnce(now.plusSeconds(61));
|
||||
tamper("auditDetail", 1, "newValue", "forced mismatch");
|
||||
var failure = new WormAuditVerifier(config).verify(null, now.plusSeconds(61));
|
||||
QRecord failed = VerificationRuns.save(failure, "SCHEDULE", null, now.plusSeconds(62));
|
||||
assertThat(failure.passed()).isFalse();
|
||||
data = WormAuditHomeData.load(config, new MemorySealedStore(), now.plusSeconds(63));
|
||||
assertThat(data.openBreak().getValueLong("id")).isEqualTo(failed.getValueLong("id"));
|
||||
String html = WormAuditHomeView.render(data, "/app/admin/worm");
|
||||
assertThat(html).contains("/app/admin/worm/worm_audit_verification/1", "/app/admin/worm/worm_audit_chain/1", "content differs from the chained hash");
|
||||
assertThat(WormAuditHomeView.renderHealthCard(data, "/app/worm_audit")).contains("Chain breaks reported");
|
||||
// A passing selected run must not clear a full-chain failure.
|
||||
VerificationRuns.save(new WormAuditVerifier(config).verify(Set.of(2L), now.plusSeconds(64)), "USER", null, now.plusSeconds(65));
|
||||
assertThat(WormAuditHomeData.load(config, new MemorySealedStore(), now.plusSeconds(66)).openBreak()).isNotNull();
|
||||
tamper("auditDetail", 1, "newValue", "new");
|
||||
VerificationRuns.save(new WormAuditVerifier(config).verify(null, now.plusSeconds(67)), "SCHEDULE", null, now.plusSeconds(68));
|
||||
data = WormAuditHomeData.load(config, new MemorySealedStore(), now.plusSeconds(69));
|
||||
assertThat(data.openBreak()).isNull();
|
||||
assertThat(data.warnings()).hasSize(2);
|
||||
assertThat(WormAuditHomeView.render(data, "/app")).contains("Setup warnings", "GOVERNANCE", "30 days");
|
||||
assertThat(WormAuditHomeView.renderHealthCard(data, "/app/worm_audit")).contains("Setup needs attention");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Genuine exports, trailing lag and a corruption fixture for a coverage hole.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void sealedRangesLagGapsAndPurges() throws Exception
|
||||
{
|
||||
var storage = new MemorySealedStore();
|
||||
var config = ChainVerificationTest.signed().withSealedStoreSupplier(() -> storage);
|
||||
init(config);
|
||||
for(long id = 1; id <= 4; id++)
|
||||
{
|
||||
audit(id);
|
||||
}
|
||||
Instant now = OLD.plusSeconds(180);
|
||||
new WormAuditChainer(config).chainOnce(now);
|
||||
var export = new SealedExporter(config).run("USER", now);
|
||||
assertThat(export.failed()).isFalse();
|
||||
assertThat(new SealedExportVerifier(config).verify(export.row(), "USER").result().passed()).isTrue();
|
||||
audit(5);
|
||||
new WormAuditChainer(config).chainOnce(now.plusSeconds(120));
|
||||
var data = WormAuditHomeData.load(config, null, now.plus(Duration.ofHours(3)));
|
||||
assertThat(data.exportLag()).isEqualTo(1);
|
||||
assertThat(data.exportAgeHours()).isEqualTo(3);
|
||||
assertThat(data.exportGaps()).isEmpty();
|
||||
assertThat(data.lockCapability()).isEqualTo("SUPPORTED");
|
||||
assertThat(data.bucketAlias()).isEqualTo("worm_audit");
|
||||
assertThat(data.lastExport().getValueString("lock_mode")).isEqualTo("GOVERNANCE");
|
||||
assertThat(WormAuditHomeView.render(data, "/app")).contains("/app/worm_audit_export/1", "GOVERNANCE", "Retain until");
|
||||
// Corruption fixture only: the real exporter never creates this hole.
|
||||
tamper(WormAuditTables.EXPORT, 1, "from_link_id", 3L);
|
||||
assertThat(WormAuditHomeData.load(config, storage, now).exportGaps()).containsExactly("1–2");
|
||||
for(long id = 1; id <= 12; id++)
|
||||
{
|
||||
STORE.insert(WormAuditTables.PURGE, List.of(new QRecord().withValue("id", id).withValue("requested_at", now.plusSeconds(id))
|
||||
.withValue("requested_by", "<operator>").withValue("cutoff", OLD).withValue("outcome", id == 1 ? "PURGED" : "REFUSED")
|
||||
.withValue("reason", "Retention not elapsed").withValue("anchor_link_id", 2L).withValue("anchor_hash", "abcd".repeat(16))
|
||||
.withValue("audits_deleted", 0L).withValue("details_deleted", 0L).withValue("links_deleted", 0L)));
|
||||
}
|
||||
data = WormAuditHomeData.load(config, storage, now.plusSeconds(20));
|
||||
assertThat(data.anchor().getValueLong("anchor_link_id")).isEqualTo(2);
|
||||
assertThat(data.recentPurges()).hasSize(10).extracting(r -> r.getValueLong("id")).containsExactly(12L, 11L, 10L, 9L, 8L, 7L, 6L, 5L, 4L, 3L);
|
||||
assertThat(data.exportGaps()).isEmpty();
|
||||
assertThat(WormAuditHomeView.render(data, "/app")).contains("abcdabcdabcd…", "<operator>", "REFUSED").doesNotContain("abcd".repeat(16));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unknown/unsupported capability and stale schedule diagnostics stay value-free.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void capabilityAndDailyWarnings() throws Exception
|
||||
{
|
||||
var config = ChainVerificationTest.signed();
|
||||
init(config);
|
||||
var unsupported = new MemorySealedStore()
|
||||
{
|
||||
@Override
|
||||
public boolean supportsLock()
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
};
|
||||
var unknown = new MemorySealedStore()
|
||||
{
|
||||
@Override
|
||||
public CapabilityStatus lockCapability()
|
||||
{
|
||||
return (CapabilityStatus.UNKNOWN);
|
||||
}
|
||||
};
|
||||
var error = new MemorySealedStore()
|
||||
{
|
||||
@Override
|
||||
public boolean supportsLock()
|
||||
{
|
||||
throw (new IllegalStateException("CANARY-provider-secret"));
|
||||
}
|
||||
};
|
||||
assertThat(WormAuditHomeData.load(config, unsupported, OLD).lockCapability()).isEqualTo("UNSUPPORTED");
|
||||
assertThat(WormAuditHomeData.load(config, unknown, OLD).lockCapability()).isEqualTo("UNKNOWN");
|
||||
String html = WormAuditHomeView.render(WormAuditHomeData.load(config, error, OLD), "/app");
|
||||
assertThat(html).contains("UNKNOWN", "Object Lock capability check failed").doesNotContain("CANARY", "provider-secret");
|
||||
VerificationRuns.save(new VerificationResult(OLD), "SCHEDULE", null, OLD);
|
||||
assertThat(WormAuditHomeData.load(config, unsupported, OLD.plus(Duration.ofHours(26))).warnings()).doesNotContain("Daily verification last ran more than 26 hours ago");
|
||||
var stale = WormAuditHomeData.load(config, unsupported, OLD.plus(Duration.ofHours(26)).plusSeconds(1));
|
||||
assertThat(stale.warnings()).contains("Daily verification last ran more than 26 hours ago");
|
||||
assertThat(WormAuditHomeView.renderHealthCard(stale, "/app/worm_audit")).contains("Setup needs attention");
|
||||
config.withTrustedPublicKeys(Map.of());
|
||||
assertThat(WormAuditHomeData.load(config, unsupported, OLD).warnings()).contains("No trusted public keys configured");
|
||||
// USER and SEALED_EXPORT runs cannot refresh the daily CHAIN heartbeat.
|
||||
VerificationRuns.save(new VerificationResult(OLD.plus(Duration.ofDays(2))), "USER", null, OLD.plus(Duration.ofDays(2)));
|
||||
VerificationRuns.save(new VerificationResult(OLD.plus(Duration.ofDays(2))), "SCHEDULE", 1L, OLD.plus(Duration.ofDays(2)));
|
||||
assertThat(WormAuditHomeData.load(config, unsupported, OLD.plus(Duration.ofDays(2))).warnings()).contains("Daily verification last ran more than 26 hours ago");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** No portion of raw audit content belongs in either HTML widget.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void canaryNeverAppearsAndWidgetsRequireReadPermission() throws Exception
|
||||
{
|
||||
var config = ChainVerificationTest.signed().withAppName("evidence").withAppLabel("Evidence").withTablePathPrefix("/app");
|
||||
var instance = init(config);
|
||||
instance.addApp(new QAppMetaData().withName("admin").withLabel("Admin"));
|
||||
config.withParentAppName("admin");
|
||||
instance.getApp("evidence").setParentAppName("admin");
|
||||
String canary = "CANARY-PHI-" + UUID.randomUUID();
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
tamper("audit", 1, "message", canary);
|
||||
tamper("auditDetail", 1, "message", canary);
|
||||
tamper("auditDetail", 1, "oldValue", canary);
|
||||
tamper("auditDetail", 1, "newValue", canary);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
var input = new RenderWidgetInput().withWidgetMetaData(instance.getWidget(WormAuditApp.HOME_WIDGET));
|
||||
assertThatThrownBy(() -> new WormAuditApp.HomeRenderer().render(input)).isInstanceOf(QPermissionDeniedException.class);
|
||||
assertThatThrownBy(() -> new WormAuditApp.HealthCardRenderer().render(input)).isInstanceOf(QPermissionDeniedException.class);
|
||||
QContext.getQSession().withPermissions(WormAuditTables.PURGE + ".read");
|
||||
String home = ((RawHTML) new WormAuditApp.HomeRenderer().render(input).getWidgetData()).getHtml();
|
||||
String card = ((RawHTML) new WormAuditApp.HealthCardRenderer().render(input).getWidgetData()).getHtml();
|
||||
for(int start = 0; start <= canary.length() - 5; start++)
|
||||
{
|
||||
assertThat(home + card).doesNotContain(canary.substring(start, start + 5));
|
||||
}
|
||||
assertThat(home + card).contains("overflow-wrap:anywhere", "color:inherit").doesNotContain("overflow-x", "<table", "#fff");
|
||||
assertThat(card).contains("href=\"/app/admin/evidence\"");
|
||||
assertThat(instance.getApp("evidence").getLabel()).isEqualTo("Evidence");
|
||||
assertThat(instance.getApp("evidence").getParentAppName()).isEqualTo("admin");
|
||||
assertThat(instance.getApp("evidence").getWidgets()).containsExactly(WormAuditApp.HOME_WIDGET);
|
||||
assertThat(instance.getApp("evidence").getChildren()).extracting(c -> c.getName()).containsExactlyElementsOf(WormAuditTables.NAV);
|
||||
assertThat(instance.getApp("evidence").getChildren()).extracting(c -> c.getLabel())
|
||||
.containsExactly("Verifications", "Sealed Exports", "Checkpoints", "Audit Chain", "Detail Coverage", "Purges", "Settings");
|
||||
assertThat(instance.getWidget(WormAuditApp.HOME_WIDGET).getGridColumns()).isEqualTo(12);
|
||||
assertThat(instance.getWidget(WormAuditApp.HOME_WIDGET).getIsCard()).isFalse();
|
||||
assertThat(instance.getWidget(WormAuditApp.HEALTH_CARD_WIDGET).getIsCard()).isTrue();
|
||||
new QInstanceValidator().validate(instance);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Embedded widgets retain direct authorization without a standalone app.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void embeddedWidgetsRequireReadPermission() throws Exception
|
||||
{
|
||||
var config = ChainVerificationTest.signed().withIncludeApp(false).withAppName("audit").withAppLabel("Audit");
|
||||
var instance = init(config);
|
||||
assertThat(instance.getApp("worm_audit")).isNull();
|
||||
assertThat(instance.getApp("audit")).isNull();
|
||||
var input = new RenderWidgetInput().withWidgetMetaData(instance.getWidget(WormAuditApp.HEALTH_CARD_WIDGET));
|
||||
QContext.getQSession().withPermissions("admin.hasAccess");
|
||||
assertThatThrownBy(() -> new WormAuditApp.HealthCardRenderer().render(input)).isInstanceOf(QPermissionDeniedException.class);
|
||||
assertThatThrownBy(() -> new WormAuditApp.HomeRenderer().render(input)).isInstanceOf(QPermissionDeniedException.class);
|
||||
QContext.getQSession().withPermissions(WormAuditTables.CHAIN + ".read");
|
||||
String card = ((RawHTML) new WormAuditApp.HealthCardRenderer().render(input).getWidgetData()).getHtml();
|
||||
assertThat(card).contains("href=\"/app/audit\"");
|
||||
assertThat(new WormAuditApp.HomeRenderer().render(input).getWidgetData()).isInstanceOf(RawHTML.class);
|
||||
QContext.setQSession(new QSession());
|
||||
assertThatThrownBy(() -> new WormAuditApp.HealthCardRenderer().render(input)).isInstanceOf(QPermissionDeniedException.class);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise the real SQL anti-join without Docker, including its exact boundary.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void sqlSummaryUsesTheWriterAntiJoin() throws Exception
|
||||
{
|
||||
String url = "jdbc:h2:mem:home_" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE;INIT=SET TIME ZONE 'UTC'";
|
||||
var config = ChainVerificationTest.signed();
|
||||
var instance = RdbmsTestSupport.instance(url, "sa", "", config, false);
|
||||
try(var connection = DriverManager.getConnection(url, "sa", ""))
|
||||
{
|
||||
RdbmsTestSupport.createTables(connection, instance, false);
|
||||
QContext.init(instance, new QSession());
|
||||
lookups();
|
||||
audit(1);
|
||||
new WormAuditChainer(config).chainOnce(OLD.plusSeconds(100));
|
||||
audit(2);
|
||||
audit(3);
|
||||
var data = WormAuditHomeData.load(config, new MemorySealedStore(), OLD.plusSeconds(63));
|
||||
assertThat(data.links()).isEqualTo(1);
|
||||
assertThat(data.backlog()).isEqualTo(1);
|
||||
assertThat(data.settling()).isEqualTo(1);
|
||||
assertThat(data.warnings()).contains("Daily verification has never run");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The demo's history comes from real engines, including FAIL, recovery and PHI tokens.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void demoSeedsRealHistory() throws Exception
|
||||
{
|
||||
MemoryRecordStore.fullReset();
|
||||
var application = new WormAuditDemoServer();
|
||||
var instance = application.defineQInstance();
|
||||
new QInstanceValidator().validate(instance);
|
||||
application.seed(instance);
|
||||
QContext.init(instance, new QSession());
|
||||
var data = WormAuditHomeData.load();
|
||||
assertThat(data.links()).isGreaterThanOrEqualTo(350);
|
||||
assertThat(data.lastVerification().getValueString("outcome")).isEqualTo("PASS");
|
||||
assertThat(data.openBreak()).isNull();
|
||||
assertThat(data.exportGaps()).isEmpty();
|
||||
assertThat(data.exportLag()).isEqualTo(5);
|
||||
assertThat(data.backlog()).isEqualTo(2);
|
||||
assertThat(data.settling()).isEqualTo(1);
|
||||
assertThat(data.warnings()).hasSize(2);
|
||||
assertThat(data.recentPurges()).extracting(r -> r.getValueString("outcome")).containsExactly("REFUSED", "PURGED");
|
||||
assertThat(STORE.where(WormAuditTables.VERIFICATION, "outcome", "FAIL")).hasSize(1).allSatisfy(row ->
|
||||
{
|
||||
assertThat(row.getValueLong("first_break_link_id")).isPositive();
|
||||
assertThat(row.getValueLong("first_break_audit_id")).isPositive();
|
||||
});
|
||||
assertThat(STORE.all("auditDetail")).filteredOn(r -> "diagnosis".equals(r.getValueString("fieldName")))
|
||||
.isNotEmpty().allSatisfy(row -> assertThat(row.getValueString("newValue")).startsWith("hmac:v1:test:"));
|
||||
var config = VerificationRuns.config();
|
||||
// Settle and chain the final backlog before checking the entire live suffix.
|
||||
new WormAuditChainer(config).chainOnce(Instant.now().plusSeconds(120));
|
||||
assertThat(new WormAuditVerifier(config).verify().passed()).isTrue();
|
||||
assertThat(new SealedExportVerifier(config).verify(STORE.last(WormAuditTables.EXPORT), "USER").result().passed()).isTrue();
|
||||
}
|
||||
}
|
||||
@@ -20,71 +20,182 @@
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.wormaudit.app.WormAuditApp;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditHelp;
|
||||
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.WormAuditProcesses;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify the scaffold's registration and configuration contract.
|
||||
** Tests and fixtures for worm audit qbit producer test.
|
||||
*******************************************************************************/
|
||||
class WormAuditQBitProducerTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Produce the identity and exact config into an otherwise empty instance.
|
||||
** Verify metadata and defaults.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void producesQBitWithoutTables()
|
||||
void metadataAndDefaults() throws Exception
|
||||
{
|
||||
QInstance instance = new QInstance();
|
||||
WormAuditQBitConfig config = new WormAuditQBitConfig();
|
||||
WormAuditQBitProducer producer = new WormAuditQBitProducer().withQBitConfig(config);
|
||||
MetaDataProducerMultiOutput output = producer.produce(instance);
|
||||
QBitMetaData qBit = instance.getQBits().get("com.kingsrook.qbits:qbit-worm-audit");
|
||||
|
||||
assertThat(qBit).isNotNull();
|
||||
assertThat(qBit.getGroupId()).isEqualTo("com.kingsrook.qbits");
|
||||
assertThat(qBit.getArtifactId()).isEqualTo("qbit-worm-audit");
|
||||
assertThat(qBit.getVersion()).isEqualTo("0.1.0");
|
||||
assertThat(qBit.getConfig()).isSameAs(config);
|
||||
assertThat(producer.getQBitConfig()).isSameAs(config);
|
||||
assertThat(producer.isEnabled()).isTrue();
|
||||
assertThat(producer.getNamespace()).isNull();
|
||||
assertThat(output.getEach(QBitMetaData.class)).containsExactly(qBit);
|
||||
assertThat(output.getEach(QTableMetaData.class)).isEmpty();
|
||||
assertThat(instance.getTables()).isEmpty();
|
||||
var config = new WormAuditQBitConfig();
|
||||
QInstance instance = WormAuditTestSupport.init(config);
|
||||
assertThat(config.getIncludeApp()).isTrue();
|
||||
assertThat(config.getRetentionYears()).isEqualTo(6);
|
||||
assertThat(config.getTableMetaDataCustomizer()).isNull();
|
||||
assertThat(config.getSigningKey()).isNull();
|
||||
assertThat(config.getTrustedPublicKeys()).isEmpty();
|
||||
assertThat(config.getChainIntervalSeconds()).isEqualTo(30);
|
||||
assertThat(config.getSettleHorizonSeconds()).isEqualTo(60);
|
||||
assertThat(config.getCheckpointEveryLinks()).isEqualTo(10000);
|
||||
assertThat(config.getCheckpointEveryMinutes()).isEqualTo(60);
|
||||
assertThat(config.getBatchSize()).isEqualTo(1000);
|
||||
assertThat(config.getLateDetailLookbackHours()).isEqualTo(24);
|
||||
assertThat(config.getVerifyCron()).isEqualTo("0 0 3 * * ?");
|
||||
assertThat(config.getAppRole()).isEqualTo("app");
|
||||
assertThat(instance.getQBits().get("com.kingsrook.qbits:qbit-worm-audit").getConfig())
|
||||
.isSameAs(config);
|
||||
assertThat(instance.getApp("worm_audit").getSections())
|
||||
.extracting(s -> s.getName())
|
||||
.containsExactlyElementsOf(WormAuditTables.NAV);
|
||||
assertThat(instance.getApp("worm_audit").getChildren())
|
||||
.allMatch(
|
||||
c ->
|
||||
c.getName().startsWith("worm_audit_")
|
||||
&& !c.getName().contains("verify"));
|
||||
assertThat(instance.getProcess(WormAuditProcesses.VERIFY).getTableName())
|
||||
.isEqualTo(WormAuditTables.CHAIN);
|
||||
assertThat(instance.getProcess(WormAuditProcesses.CHAIN).getSchedule().getRepeatSeconds())
|
||||
.isEqualTo(30);
|
||||
assertThat(instance.getProcess(WormAuditProcesses.DAILY).getSchedule().getCronExpression())
|
||||
.isEqualTo(config.getVerifyCron());
|
||||
for(String name :
|
||||
List.of(
|
||||
"audit",
|
||||
"auditDetail",
|
||||
"auditUser",
|
||||
"auditTable",
|
||||
WormAuditTables.CHAIN,
|
||||
WormAuditTables.CHECKPOINT,
|
||||
WormAuditTables.EXPORT,
|
||||
WormAuditTables.VERIFICATION,
|
||||
WormAuditTables.PURGE))
|
||||
{
|
||||
var table = instance.getTable(name);
|
||||
assertThat(
|
||||
table.isCapabilityEnabled(
|
||||
instance.getBackendForTable(name), Capability.TABLE_UPDATE))
|
||||
.isFalse();
|
||||
assertThat(
|
||||
table.isCapabilityEnabled(
|
||||
instance.getBackendForTable(name), Capability.TABLE_DELETE))
|
||||
.isFalse();
|
||||
if(name.startsWith("worm_audit_"))
|
||||
{
|
||||
assertThat(table.getPermissionRules().getPermissionBaseName()).isEqualTo(name);
|
||||
assertThat(
|
||||
table.isCapabilityEnabled(
|
||||
instance.getBackendForTable(name), Capability.TABLE_INSERT))
|
||||
.isFalse();
|
||||
assertThat(table.getSections()).isNotEmpty();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify customizations and validation.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void customizationsAndValidation() throws Exception
|
||||
{
|
||||
assertThat(new WormAuditQBitProducer().isEnabled()).isFalse();
|
||||
var config =
|
||||
new WormAuditQBitConfig()
|
||||
.withRetentionYears(7)
|
||||
.withChainIntervalSeconds(45)
|
||||
.withSettleHorizonSeconds(0)
|
||||
.withBatchSize(13)
|
||||
.withCheckpointEveryLinks(5)
|
||||
.withCheckpointEveryMinutes(2)
|
||||
.withVerifyCron("0 1 3 * * ?")
|
||||
.withAppRole("runtime")
|
||||
.withSchedulerName("jobs")
|
||||
.withCoreAuditTableNames(java.util.Map.of("audit", "custom_audit"))
|
||||
.withTableMetaDataCustomizer(
|
||||
(instance, table) ->
|
||||
table.withLabel("custom")
|
||||
.withCapabilities(Capability.TABLE_UPDATE));
|
||||
var instance =
|
||||
WormAuditTestSupport.core(
|
||||
new com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData()
|
||||
.withName("memory")
|
||||
.withBackendType(
|
||||
com.kingsrook.qqq.backend.core.modules.backend
|
||||
.implementations.memory.MemoryBackendModule.class));
|
||||
var producer =
|
||||
new WormAuditQBitProducer()
|
||||
.withNamespace("test")
|
||||
.withQBitConfig(config.withBackendName("memory"));
|
||||
producer.produce(instance);
|
||||
assertThat(producer.getNamespace()).isEqualTo("test");
|
||||
assertThat(producer.getQBitConfig()).isSameAs(config);
|
||||
assertThat(producer.isEnabled()).isTrue();
|
||||
assertThat(config.getCoreAuditTableNames()).containsEntry("audit", "custom_audit");
|
||||
assertThat(instance.getTable(WormAuditTables.CHAIN).getLabel()).isEqualTo("custom");
|
||||
assertThat(
|
||||
instance.getTable(WormAuditTables.CHAIN)
|
||||
.isCapabilityEnabled(
|
||||
instance.getBackend("memory"), Capability.TABLE_UPDATE))
|
||||
.isFalse();
|
||||
assertThatThrownBy(() -> new WormAuditQBitConfig().validate())
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
assertThat(config.withLateDetailLookbackHours(48).getLateDetailLookbackHours()).isEqualTo(48);
|
||||
assertThatThrownBy(() -> config.withLateDetailLookbackHours(0).validate()).isInstanceOf(IllegalArgumentException.class);
|
||||
config.withLateDetailLookbackHours(24);
|
||||
assertThatThrownBy(() -> config.withRetentionYears(0).validate())
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatThrownBy(
|
||||
() -> WormAuditDdl.resolveNames(new WormAuditQBitConfig(), new QInstance()))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configured instances may be namespaced; unconfigured scanning is disabled.
|
||||
** Embedded production keeps the host app intact and registers all evidence metadata.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void supportsNamespacedConfiguration()
|
||||
void embeddedMetadataPreservesHostApp() throws Exception
|
||||
{
|
||||
assertThat(new WormAuditQBitProducer().isEnabled()).isFalse();
|
||||
MetaDataCustomizerInterface<QTableMetaData> customizer = (instance, table) -> table;
|
||||
WormAuditQBitConfig config = new WormAuditQBitConfig()
|
||||
.withTableMetaDataCustomizer(customizer)
|
||||
.withRetentionYears(7);
|
||||
WormAuditQBitProducer producer = new WormAuditQBitProducer()
|
||||
.withQBitConfig(config)
|
||||
.withNamespace("test");
|
||||
QInstance instance = new QInstance();
|
||||
producer.produce(instance);
|
||||
|
||||
assertThat(producer.getNamespace()).isEqualTo("test");
|
||||
assertThat(instance.getQBits().get("com.kingsrook.qbits:qbit-worm-audit:test").getConfig()).isSameAs(config);
|
||||
assertThat(config.getTableMetaDataCustomizer()).isSameAs(customizer);
|
||||
assertThat(config.getRetentionYears()).isEqualTo(7);
|
||||
assertThat(instance.getTables()).isEmpty();
|
||||
var config = new WormAuditQBitConfig().withBackendName("memory").withIncludeApp(false).withAppName("audit");
|
||||
var instance = WormAuditTestSupport.core(new QBackendMetaData().withName("memory").withBackendType(MemoryBackendModule.class));
|
||||
var host = new QAppMetaData().withName("audit").withLabel("Combined Audit").withWidgets(List.of("hostSummary"));
|
||||
instance.addApp(host);
|
||||
new WormAuditQBitProducer().withQBitConfig(config).produce(instance);
|
||||
assertThat(config.getIncludeApp()).isFalse();
|
||||
assertThat(instance.getApp("worm_audit")).isNull();
|
||||
assertThat(instance.getApp("audit")).isSameAs(host);
|
||||
assertThat(host.getLabel()).isEqualTo("Combined Audit");
|
||||
assertThat(host.getWidgets()).containsExactly("hostSummary");
|
||||
assertThat(host.getHelpContent()).isNullOrEmpty();
|
||||
assertThat(instance.getWidget(WormAuditApp.HEALTH_CARD_WIDGET)).isNotNull();
|
||||
assertThat(instance.getWidget(WormAuditApp.HOME_WIDGET)).isNotNull();
|
||||
for(String name : WormAuditTables.ALL)
|
||||
{
|
||||
assertThat(instance.getTable(name).getHelpContent()).containsKey(WormAuditHelp.SLOT);
|
||||
assertThat(instance.getTable(name).getPermissionRules().getPermissionBaseName()).isEqualTo(name);
|
||||
}
|
||||
assertThat(instance.getTable(WormAuditTables.CHAIN).getDisabledCapabilities()).contains(Capability.TABLE_UPDATE, Capability.TABLE_DELETE, Capability.TABLE_INSERT);
|
||||
assertThat(instance.getTable("audit").getDisabledCapabilities()).contains(Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
|
||||
assertThat(instance.getProcess(WormAuditProcesses.VERIFY)).isNotNull();
|
||||
assertThat(instance.getProcess(WormAuditProcesses.CHAIN).getSchedule().getRepeatSeconds()).isEqualTo(config.getChainIntervalSeconds());
|
||||
assertThat(instance.getProcess(WormAuditProcesses.DAILY).getSchedule().getCronExpression()).isEqualTo(config.getVerifyCron());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.Statement;
|
||||
import java.util.UUID;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The settings row the shipped database script seeds (ddl/postgres/worm-audit.sql,
|
||||
** changeset worm-audit:10, rendered with the host's configured values).
|
||||
** The whole script is PostgreSQL-only (triggers, roles, plpgsql), so it cannot
|
||||
** run on H2; this runs the settings table's CREATE and its seed statement as
|
||||
** the script renders them, on H2 in PostgreSQL mode: the row appears with the
|
||||
** host's values, a second run inserts nothing, and an edit survives.
|
||||
** WormAuditSettings.seed() is deprecated (nothing in src/main calls it) but still
|
||||
** works for memory and H2 hosts; WormAuditSettings.read() falls back to the
|
||||
** configured values while the row is absent.
|
||||
*******************************************************************************/
|
||||
class WormAuditSettingsSeedTest
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
** Fresh table, seed twice, edit survives, values follow the host config.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void seedStatementIsInsertOnlyAndUsesHostValues() throws Exception
|
||||
{
|
||||
var config = new WormAuditQBitConfig().withRetentionYears(8).withExportRetentionDays(45).withCheckpointEveryLinks(12);
|
||||
String ddl = WormAuditDdl.postgres(config);
|
||||
String create = statement(ddl, "CREATE TABLE IF NOT EXISTS public.worm_audit_settings");
|
||||
String seed = statement(ddl, "INSERT INTO public.worm_audit_settings");
|
||||
|
||||
try(Connection connection = DriverManager.getConnection("jdbc:h2:mem:" + UUID.randomUUID() + ";MODE=PostgreSQL;DB_CLOSE_DELAY=-1");
|
||||
Statement sql = connection.createStatement())
|
||||
{
|
||||
sql.execute(create.replace("timestamptz", "timestamp with time zone").replace("now()", "CURRENT_TIMESTAMP").replace("current_user", "'app'"));
|
||||
sql.execute(seed);
|
||||
assertThat(scalar(sql, "SELECT COUNT(*) FROM public.worm_audit_settings")).isEqualTo("1");
|
||||
assertThat(scalar(sql, "SELECT audit_retention_years FROM public.worm_audit_settings")).isEqualTo("8");
|
||||
assertThat(scalar(sql, "SELECT export_retention_days FROM public.worm_audit_settings")).isEqualTo("45");
|
||||
assertThat(scalar(sql, "SELECT checkpoint_every_links FROM public.worm_audit_settings")).isEqualTo("12");
|
||||
assertThat(scalar(sql, "SELECT export_lock_mode FROM public.worm_audit_settings")).isEqualTo(config.getExportLockMode());
|
||||
assertThat(scalar(sql, "SELECT export_cron FROM public.worm_audit_settings")).isEqualTo(config.getExportCron());
|
||||
|
||||
sql.execute("UPDATE public.worm_audit_settings SET export_retention_days = 90 WHERE id = 1");
|
||||
sql.execute(seed);
|
||||
assertThat(scalar(sql, "SELECT COUNT(*) FROM public.worm_audit_settings")).isEqualTo("1");
|
||||
assertThat(scalar(sql, "SELECT export_retention_days FROM public.worm_audit_settings")).isEqualTo("90");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The deprecated public seed() still creates row 1 once and keeps edits.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
@SuppressWarnings("deprecation")
|
||||
void deprecatedSeedStillWorksForMemoryHosts() throws Exception
|
||||
{
|
||||
var config = new WormAuditQBitConfig().withRetentionYears(8);
|
||||
WormAuditTestSupport.init(config);
|
||||
WormAuditSettings.seed(config);
|
||||
assertThat(WormAuditTestSupport.STORE.get("worm_audit_settings", 1).getValueInteger("audit_retention_years")).isEqualTo(8);
|
||||
WormAuditSettings.seed(config);
|
||||
assertThat(WormAuditTestSupport.STORE.get("worm_audit_settings", 1)).isNotNull();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The script seeds nothing but the settings table: its only top-level INSERT is the
|
||||
** settings seed (the indented purge INSERTs are the purge function writing its log).
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void onlySettingsAreSeeded()
|
||||
{
|
||||
String ddl = WormAuditDdl.postgres(new WormAuditQBitConfig());
|
||||
Matcher matcher = Pattern.compile("(?m)^INSERT INTO\\s+(\\S+)").matcher(ddl);
|
||||
while(matcher.find())
|
||||
{
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// worm-audit:13 backfills detail coverage from links that already exist: //
|
||||
// derived from the chain, never seeded content. //
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
assertThat(matcher.group(1)).as("every top-level INSERT in the script targets the settings table, not a log")
|
||||
.isIn("public.worm_audit_settings", "public.worm_audit_chain_detail(detail_id,");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** From the start text up to the statement's closing semicolon.
|
||||
*******************************************************************************/
|
||||
private static String statement(String ddl, String start)
|
||||
{
|
||||
int from = ddl.indexOf(start);
|
||||
assertThat(from).as(start).isGreaterThanOrEqualTo(0);
|
||||
return (ddl.substring(from, ddl.indexOf(";", from)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static String scalar(Statement sql, String query) throws Exception
|
||||
{
|
||||
try(ResultSet result = sql.executeQuery(query))
|
||||
{
|
||||
assertThat(result.next()).isTrue();
|
||||
return (result.getString(1));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,307 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneId;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.wormaudit.app.WormAuditHomeData;
|
||||
import com.kingsrook.qbits.wormaudit.app.WormAuditHomeView;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedExporter;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedExportVerifier;
|
||||
import com.kingsrook.qbits.wormaudit.model.SettingsCustomizer;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.WormAuditProcesses;
|
||||
import com.kingsrook.qbits.wormaudit.processes.ExportProcesses;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.*;
|
||||
import static org.assertj.core.api.Assertions.*;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Editable settings, bounded cache, audited changes and historical lock modes.
|
||||
*******************************************************************************/
|
||||
class WormAuditSettingsTest
|
||||
{
|
||||
static final String TABLE = WormAuditTables.SETTINGS;
|
||||
|
||||
/*******************************************************************************
|
||||
** Deterministic clock for the exact 30-second cache boundary.
|
||||
*******************************************************************************/
|
||||
static class MutableClock extends Clock
|
||||
{
|
||||
Instant now = Instant.parse("2026-10-14T02:00:00Z");
|
||||
/*******************************************************************************
|
||||
** Verify get zone.
|
||||
*******************************************************************************/
|
||||
public ZoneId getZone()
|
||||
{
|
||||
return (ZoneOffset.UTC);
|
||||
}
|
||||
/*******************************************************************************
|
||||
** Verify with zone.
|
||||
*******************************************************************************/
|
||||
public Clock withZone(ZoneId zone)
|
||||
{
|
||||
return (this);
|
||||
}
|
||||
/*******************************************************************************
|
||||
** Verify instant.
|
||||
*******************************************************************************/
|
||||
public Instant instant()
|
||||
{
|
||||
return (now);
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify editor.
|
||||
*******************************************************************************/
|
||||
static void editor()
|
||||
{
|
||||
QContext.getQInstance().withEnforcePermissionsInCoreActions(true);
|
||||
QContext.getQSession().withPermissions(Set.of(TABLE + ".read", TABLE + ".edit", "worm_audit.settings"));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify update.
|
||||
*******************************************************************************/
|
||||
static void update(String field, java.io.Serializable value) throws Exception
|
||||
{
|
||||
var result = new UpdateAction().execute(new UpdateInput(TABLE).withRecords(List.of(new QRecord().withValue("id", 1).withValue(field, value))));
|
||||
assertThat(result.getRecords()).allSatisfy(row -> assertThat(row.getErrors()).isNullOrEmpty());
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify fallback overrides cache and insert only seed.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void fallbackOverridesCacheAndInsertOnlySeed() throws Exception
|
||||
{
|
||||
var config = new WormAuditQBitConfig().withRetentionYears(8).withExportRetentionDays(45).withCheckpointEveryLinks(12);
|
||||
var instance = init(config);
|
||||
var clock = new MutableClock();
|
||||
var reader = new WormAuditSettings(config, clock);
|
||||
assertThat(reader.read().audit_retention_years()).isEqualTo(8);
|
||||
assertThat(reader.read().export_retention_days()).isEqualTo(45);
|
||||
WormAuditTestSupport.seedSettings(config);
|
||||
editor();
|
||||
update("export_retention_days", 90);
|
||||
assertThat(reader.read().export_retention_days()).isEqualTo(90);
|
||||
update("export_retention_days", 120);
|
||||
clock.now = clock.now.plusSeconds(29);
|
||||
assertThat(reader.read().export_retention_days()).isEqualTo(90);
|
||||
clock.now = clock.now.plusSeconds(1);
|
||||
assertThat(reader.read().export_retention_days()).isEqualTo(120);
|
||||
WormAuditTestSupport.seedSettings(config);
|
||||
assertThat(STORE.get(TABLE, 1).getValueInteger("export_retention_days")).isEqualTo(120);
|
||||
assertThat(reader.read().checkpoint_every_links()).isEqualTo(12);
|
||||
instance.getTables().remove(TABLE);
|
||||
reader.invalidate();
|
||||
assertThat(reader.read().export_retention_days()).isEqualTo(45);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify partial null row uses config and invalid persisted settings fail closed.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void partialNullRowUsesConfigAndInvalidPersistedSettingsFailClosed() throws Exception
|
||||
{
|
||||
var config = new WormAuditQBitConfig().withExportRetentionDays(42);
|
||||
init(config);
|
||||
com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore.getInstance().insert(
|
||||
new InsertInput(TABLE).withRecord(new QRecord().withValue("id", 1).withValue("export_lock_mode", "COMPLIANCE")), false);
|
||||
var reader = new WormAuditSettings(config);
|
||||
assertThat(reader.read().export_retention_days()).isEqualTo(42);
|
||||
assertThat(reader.read().export_lock_mode()).isEqualTo("COMPLIANCE");
|
||||
com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore.getInstance().update(
|
||||
new UpdateInput(TABLE).withRecords(List.of(new QRecord().withValue("id", 1).withValue("export_retention_days", 0))), false);
|
||||
reader.invalidate();
|
||||
assertThatThrownBy(reader::read).hasMessageContaining("Export retention days");
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify missing physical table falls back.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void missingPhysicalTableFallsBack() throws Exception
|
||||
{
|
||||
var config = new WormAuditQBitConfig().withExportRetentionDays(81);
|
||||
var instance = RdbmsTestSupport.instance("jdbc:h2:mem:settings_missing;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE", "sa", "", config, false);
|
||||
QContext.init(instance, new QSession());
|
||||
assertThat(config.effectiveSettings().export_retention_days()).isEqualTo(81);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify validation and capabilities.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void validationAndCapabilities() throws Exception
|
||||
{
|
||||
var config = new WormAuditQBitConfig();
|
||||
var instance = init(config);
|
||||
var table = instance.getTable(TABLE);
|
||||
assertThat(table.getLabel()).isEqualTo("WORM Audit Settings");
|
||||
assertThat(table.getAuditRules().getAuditLevel()).isEqualTo(AuditLevel.FIELD);
|
||||
assertThat(table.isCapabilityEnabled(instance.getBackendForTable(TABLE), Capability.TABLE_UPDATE)).isTrue();
|
||||
assertThat(table.isCapabilityEnabled(instance.getBackendForTable(TABLE), Capability.TABLE_INSERT)).isFalse();
|
||||
assertThat(table.isCapabilityEnabled(instance.getBackendForTable(TABLE), Capability.TABLE_DELETE)).isFalse();
|
||||
WormAuditTestSupport.seedSettings(config);
|
||||
editor();
|
||||
assertThatThrownBy(() -> new DeleteAction().execute(new DeleteInput(TABLE).withPrimaryKeys(List.of(1)))).isInstanceOf(Exception.class);
|
||||
assertThatThrownBy(() -> new InsertAction().execute(new InsertInput(TABLE).withRecord(new QRecord().withValue("id", 2)))).isInstanceOf(Exception.class);
|
||||
assertThatThrownBy(() -> new SettingsCustomizer().preInsert(new InsertInput(TABLE), List.of(new QRecord().withValue("id", 2)), false)).hasMessageContaining("one row only (ID 1)");
|
||||
assertThatThrownBy(() -> new SettingsCustomizer().preInsert(new InsertInput(TABLE), List.of(WormAuditSettings.defaults(config)), false))
|
||||
.hasMessageContaining("already exists");
|
||||
assertThatThrownBy(() -> new SettingsCustomizer().preDelete(new DeleteInput(TABLE), List.of(STORE.get(TABLE, 1)), false))
|
||||
.hasMessageContaining("cannot be deleted");
|
||||
for(String field : List.of("audit_retention_years", "export_retention_days", "checkpoint_every_links", "checkpoint_every_minutes", "chain_interval_seconds", "late_detail_lookback_hours"))
|
||||
{
|
||||
assertThatThrownBy(() -> WormAuditSettings.validate(new QRecord().withValue(field, 0))).hasMessageContaining("whole number");
|
||||
}
|
||||
assertThatThrownBy(() -> WormAuditSettings.validate(new QRecord().withValue("settle_horizon_seconds", -1))).hasMessageContaining("Settle horizon seconds");
|
||||
assertThatThrownBy(() -> WormAuditSettings.validate(new QRecord().withValue("audit_retention_years", 101))).hasMessageContaining("100");
|
||||
assertThatThrownBy(() -> WormAuditSettings.validate(new QRecord().withValue("export_retention_days", 36501))).hasMessageContaining("36500");
|
||||
assertThatThrownBy(() -> WormAuditSettings.validate(new QRecord().withValue("export_retention_days", "1.5"))).hasMessageContaining("whole number");
|
||||
assertThatThrownBy(() -> WormAuditSettings.validate(new QRecord().withValue("export_lock_mode", "NONE"))).hasMessageContaining("GOVERNANCE or COMPLIANCE");
|
||||
assertThatThrownBy(() -> WormAuditSettings.validate(new QRecord().withValue("export_lock_mode", null))).hasMessageContaining("GOVERNANCE or COMPLIANCE");
|
||||
for(String field : List.of("export_cron", "verify_cron"))
|
||||
{
|
||||
assertThatThrownBy(() -> new SettingsCustomizer().preUpdate(new UpdateInput(TABLE), List.of(new QRecord().withValue("id", 1).withValue(field, "bad")), false, java.util.Optional.empty()))
|
||||
.hasMessageContaining("valid Quartz cron");
|
||||
}
|
||||
assertThatThrownBy(() -> new UpdateAction().execute(new UpdateInput(TABLE).withRecords(List.of(new QRecord().withValue("id", 1).withValue("export_cron", "bad")))))
|
||||
.hasMessageContaining("valid Quartz cron");
|
||||
assertThat(STORE.get(TABLE, 1).getValueString("export_cron")).isEqualTo(config.getExportCron());
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify edit requires both permissions and produces field audit that chains.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void editRequiresBothPermissionsAndProducesFieldAuditThatChains() throws Exception
|
||||
{
|
||||
var config = ChainVerificationTest.signed().withSettleHorizonSeconds(0);
|
||||
init(config);
|
||||
WormAuditTestSupport.seedSettings(config);
|
||||
QContext.getQInstance().withEnforcePermissionsInCoreActions(true);
|
||||
QContext.getQSession().withPermissions(Set.of(TABLE + ".read", TABLE + ".edit"));
|
||||
assertThatThrownBy(() -> update("export_lock_mode", "COMPLIANCE")).hasMessageContaining("worm_audit.settings");
|
||||
QContext.getQSession().setPermissions(new java.util.HashSet<>(Set.of("worm_audit.settings")));
|
||||
assertThatThrownBy(() -> update("export_lock_mode", "COMPLIANCE")).hasMessageContaining("worm_audit_settings.edit");
|
||||
editor();
|
||||
update("export_lock_mode", "COMPLIANCE");
|
||||
assertThat(STORE.all("audit")).hasSize(1);
|
||||
assertThat(STORE.where("auditDetail", "fieldName", "export_lock_mode")).hasSize(1);
|
||||
assertThat(STORE.get(TABLE, 1).getValueInstant("modify_date")).isNotNull();
|
||||
assertThat(STORE.get(TABLE, 1).getValueString("modified_by")).isEqualTo("system");
|
||||
new WormAuditChainer(config).chainOnce(Instant.now().plusSeconds(1));
|
||||
assertThat(STORE.all(WormAuditTables.CHAIN)).hasSize(1);
|
||||
assertThat(new WormAuditVerifier(config).verify(null, Instant.now().plusSeconds(1)).passed()).isTrue();
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify startup schedules and home use effective settings.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void startupSchedulesAndHomeUseEffectiveSettings() throws Exception
|
||||
{
|
||||
var config = ChainVerificationTest.signed();
|
||||
var instance = init(config);
|
||||
WormAuditTestSupport.seedSettings(config);
|
||||
editor();
|
||||
update("chain_interval_seconds", 45);
|
||||
update("export_cron", "0 15 4 * * ?");
|
||||
update("verify_cron", "0 30 5 * * ?");
|
||||
update("audit_retention_years", 7);
|
||||
assertThat(instance.getProcess(WormAuditProcesses.CHAIN).getSchedule().getRepeatSeconds()).isEqualTo(30);
|
||||
var restarted = WormAuditTestSupport.core(new com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData().withName("memory")
|
||||
.withBackendType(com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule.class));
|
||||
new WormAuditQBitProducer().withQBitConfig(config).produce(restarted);
|
||||
assertThat(QContext.getQInstance()).isSameAs(instance);
|
||||
instance = restarted;
|
||||
QContext.init(instance, new QSession());
|
||||
assertThat(instance.getProcess(WormAuditProcesses.CHAIN).getSchedule().getRepeatSeconds()).isEqualTo(45);
|
||||
assertThat(instance.getProcess(ExportProcesses.DAILY).getSchedule().getCronExpression()).isEqualTo("0 15 4 * * ?");
|
||||
assertThat(instance.getProcess(WormAuditProcesses.DAILY).getSchedule().getCronExpression()).isEqualTo("0 30 5 * * ?");
|
||||
assertThat(instance.getProcess(WormAuditProcesses.DAILY).getSchedule().getCronTimeZoneId()).isEqualTo("UTC");
|
||||
audit(50);
|
||||
var home = WormAuditHomeData.load(config, new SealedExportTest.MemorySealedStore(), Instant.now());
|
||||
assertThat(home.nextPurge()).isEqualTo(OLD.plusSeconds(50).atZone(ZoneOffset.UTC).plusYears(7).toInstant());
|
||||
assertThat(home.warnings()).anyMatch(w -> w.contains("differs from") && w.contains("purge interval"));
|
||||
assertThat(WormAuditHomeView.render(home, "/app")).contains("GOVERNANCE", "s3:BypassGovernanceRetention", "Switch to COMPLIANCE", "30 days", "Export retention is shorter");
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify switch affects new exports only and old checkpoint cadence remains valid.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void switchAffectsNewExportsOnlyAndOldCheckpointCadenceRemainsValid() throws Exception
|
||||
{
|
||||
var config = ChainVerificationTest.signed().withCheckpointEveryLinks(2).withSettleHorizonSeconds(0);
|
||||
init(config);
|
||||
WormAuditTestSupport.seedSettings(config);
|
||||
var storage = new SealedExportTest.MemorySealedStore();
|
||||
audit(1);
|
||||
audit(2);
|
||||
Instant now = Instant.now().plusSeconds(1);
|
||||
new WormAuditChainer(config).chainOnce(now);
|
||||
var exporter = new SealedExporter(config, storage);
|
||||
var first = exporter.run("USER", now).row();
|
||||
assertThat(first).isNotNull();
|
||||
assertThat(first.getValueString("lock_mode")).isEqualTo("GOVERNANCE");
|
||||
assertThat(first.getValueInstant("retain_until")).isEqualTo(now.truncatedTo(java.time.temporal.ChronoUnit.MILLIS).plusSeconds(30 * 86400L));
|
||||
assertThat(storage.heads.values()).allSatisfy(h -> assertThat(h.getLockMode()).isEqualTo("GOVERNANCE"));
|
||||
var verifier = new SealedExportVerifier(config, storage);
|
||||
assertThat(verifier.verify(first, "USER").result().passed()).isTrue();
|
||||
editor();
|
||||
update("export_lock_mode", "COMPLIANCE");
|
||||
update("checkpoint_every_links", 1);
|
||||
new WormAuditChainer(config).chainOnce(Instant.now().plusSeconds(2));
|
||||
var second = exporter.run("USER", Instant.now().plusSeconds(2)).row();
|
||||
assertThat(second.getValueString("lock_mode")).isEqualTo("COMPLIANCE");
|
||||
assertThat(verifier.verify(first, "USER").result().passed()).isTrue();
|
||||
assertThat(verifier.verify(second, "USER").result().passed()).isTrue();
|
||||
assertThat(new WormAuditVerifier(config).verify(null, Instant.now().plusSeconds(2)).passed()).isTrue();
|
||||
String key = first.getValueString("object_prefix") + "links.jsonl";
|
||||
storage.heads.get(key).withLockMode("COMPLIANCE");
|
||||
assertThat(verifier.verify(first, "USER").result().getBreaks()).anyMatch(b -> b.reason().contains("lacks the recorded GOVERNANCE retention"));
|
||||
storage.heads.remove(key);
|
||||
assertThat(verifier.verify(first, "USER").result().getBreaks()).anyMatch(b -> b.reason().contains(key) && b.reason().contains("GOVERNANCE bypass removal"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Stream;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditHelp;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Guardrail (owner requirement 2026-10-14: "all tables should be populated and
|
||||
** the qbit should be fully using them"). For every worm_audit_* table:
|
||||
** - runtime code (not just its model, metadata or home screen) reads or writes
|
||||
** it;
|
||||
** - it is either seeded by the shipped database script, or listed here with
|
||||
** why it starts empty;
|
||||
** - it has bundle help that says what it is for and how its rows get created
|
||||
** (shown on the table screen and when it is empty).
|
||||
** A new table fails this test until it is wired, seeded or explained.
|
||||
*******************************************************************************/
|
||||
class WormAuditTablesGuardrailTest
|
||||
{
|
||||
private static final Path MAIN = Path.of("src", "main", "java", "com", "kingsrook", "qbits", "wormaudit");
|
||||
|
||||
/*******************************************************************************
|
||||
** Tables the shipped database script (ddl/postgres/worm-audit.sql) seeds.
|
||||
*******************************************************************************/
|
||||
static final List<String> SEEDED = List.of(WormAuditTables.SETTINGS);
|
||||
|
||||
/*******************************************************************************
|
||||
** Tables that legitimately start empty, and where their rows come from.
|
||||
*******************************************************************************/
|
||||
static final Map<String, String> STARTS_EMPTY = Map.of(
|
||||
WormAuditTables.CHAIN, "log: the chain job links each settled audit record",
|
||||
WormAuditTables.CHAIN_DETAIL, "log: the chain trigger (or writer) records each detail a new link covers",
|
||||
WormAuditTables.CHECKPOINT, "log: the chain job signs a range every N links or minutes",
|
||||
WormAuditTables.EXPORT, "log: the daily export or Export now seals a range",
|
||||
WormAuditTables.VERIFICATION, "log: every verification run, manual or daily",
|
||||
WormAuditTables.PURGE, "log: the database owner's purge function, after retention");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every table is used by runtime code.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void everyTableIsUsedAtRuntime() throws Exception
|
||||
{
|
||||
List<String> runtimeSource;
|
||||
try(Stream<Path> files = Files.walk(MAIN))
|
||||
{
|
||||
runtimeSource = files.filter(p -> p.toString().endsWith(".java"))
|
||||
.filter(p -> !p.startsWith(MAIN.resolve("model")) && !p.startsWith(MAIN.resolve("app")) && !p.startsWith(MAIN.resolve("ddl")))
|
||||
.filter(p -> !p.endsWith("WormAuditQBitProducer.java"))
|
||||
.map(WormAuditTablesGuardrailTest::read).toList();
|
||||
}
|
||||
for(String table : WormAuditTables.ALL)
|
||||
{
|
||||
String constant = "WormAuditTables." + table.substring("worm_audit_".length()).toUpperCase();
|
||||
assertThat(runtimeSource.stream().anyMatch(source -> source.contains(constant)))
|
||||
.as(table + " (" + constant + ") must be read or written by runtime code, not only declared").isTrue();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every table is seeded by the script or explained.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void everyTableIsSeededOrExplained() throws Exception
|
||||
{
|
||||
String ddl = WormAuditDdl.postgres(new WormAuditQBitConfig());
|
||||
for(String table : WormAuditTables.ALL)
|
||||
{
|
||||
boolean seeded = SEEDED.contains(table);
|
||||
assertThat(seeded || STARTS_EMPTY.containsKey(table)).as(table + " must be seeded by the database script or listed in STARTS_EMPTY").isTrue();
|
||||
assertThat(seeded && STARTS_EMPTY.containsKey(table)).as(table + " cannot be both seeded and starting empty").isFalse();
|
||||
assertThat(ddl.contains("INSERT INTO public." + table + "\n") || ddl.contains("INSERT INTO public." + table + " ")).as(table + " seeded iff the script inserts it").isEqualTo(seeded);
|
||||
}
|
||||
assertThat(STARTS_EMPTY.keySet()).allMatch(WormAuditTables.ALL::contains, "STARTS_EMPTY names only real tables");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every table and the app have bundle help that says how rows get created.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void everyTableHasEmptyStateHelp() throws Exception
|
||||
{
|
||||
QInstance instance = WormAuditTestSupport.init(new WormAuditQBitConfig());
|
||||
assertThat(WormAuditHelp.entries()).allSatisfy((key, entry) -> assertThat(entry.path("summary").asText().length()).as(key).isLessThanOrEqualTo(120));
|
||||
for(String table : WormAuditTables.ALL)
|
||||
{
|
||||
List<QHelpContent> help = instance.getTable(table).getHelpContent() == null ? null : instance.getTable(table).getHelpContent().get(WormAuditHelp.SLOT);
|
||||
assertThat(help).as(table + " table help").isNotEmpty();
|
||||
assertThat(help.get(0).getContent()).as(table + " help says how rows get created").contains("How rows get created");
|
||||
}
|
||||
assertThat(instance.getApp("worm_audit").getHelpContent()).containsKey(WormAuditHelp.SLOT);
|
||||
for(Map.Entry<String, JsonNode> entry : WormAuditHelp.entries().entrySet())
|
||||
{
|
||||
String key = entry.getKey();
|
||||
assertThat(key.startsWith("app:") || WormAuditTables.ALL.contains(key.substring(6, key.indexOf(';')))).as("help names only real tables: " + key).isTrue();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A host's own help is never replaced.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void hostHelpIsKept() throws Exception
|
||||
{
|
||||
QInstance instance = WormAuditTestSupport.init(new WormAuditQBitConfig());
|
||||
instance.getTable(WormAuditTables.CHAIN).getHelpContent().put(WormAuditHelp.SLOT, List.of(new QHelpContent().withContentAsMarkdown("host text")));
|
||||
WormAuditHelp.apply(instance, new WormAuditQBitConfig());
|
||||
assertThat(instance.getTable(WormAuditTables.CHAIN).getHelpContent().get(WormAuditHelp.SLOT)).hasSize(1);
|
||||
assertThat(instance.getTable(WormAuditTables.CHAIN).getHelpContent().get(WormAuditHelp.SLOT).get(0).getContent()).isEqualTo("host text");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static String read(Path path)
|
||||
{
|
||||
try
|
||||
{
|
||||
return (Files.readString(path, StandardCharsets.UTF_8));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw (new IllegalStateException(e));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.audits.AuditsMetaDataProvider;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
|
||||
public class WormAuditTestSupport {
|
||||
static final Instant OLD = Instant.parse("2020-01-02T03:04:05.123456Z");
|
||||
static final AuditStore STORE = new AuditStore(null);
|
||||
|
||||
static QInstance init(WormAuditQBitConfig config) throws Exception {
|
||||
MemoryRecordStore.fullReset();
|
||||
QInstance instance =
|
||||
core(
|
||||
new QBackendMetaData()
|
||||
.withName("memory")
|
||||
.withBackendType(MemoryBackendModule.class));
|
||||
new WormAuditQBitProducer()
|
||||
.withQBitConfig(config.withBackendName("memory"))
|
||||
.produce(instance);
|
||||
QContext.init(instance, new QSession());
|
||||
lookups();
|
||||
return (instance);
|
||||
}
|
||||
|
||||
static QInstance core(QBackendMetaData backend) throws Exception {
|
||||
QInstance instance = new QInstance();
|
||||
instance.withInstanceDefaultAuthentication(
|
||||
new com.kingsrook.qqq.backend.core.model.metadata.authentication
|
||||
.QAuthenticationMetaData()
|
||||
.withType(
|
||||
com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType
|
||||
.FULLY_ANONYMOUS)
|
||||
.withName("anon"));
|
||||
instance.addBackend(backend);
|
||||
new AuditsMetaDataProvider().defineAll(instance, backend.getName(), null);
|
||||
instance.getTable("audit")
|
||||
.withField(new QFieldMetaData("clientChannel", QFieldType.STRING));
|
||||
return (instance);
|
||||
}
|
||||
|
||||
static void lookups() throws Exception {
|
||||
STORE.insert(
|
||||
"auditTable",
|
||||
List.of(
|
||||
new QRecord()
|
||||
.withValue("id", 1)
|
||||
.withValue("name", "cms_page")
|
||||
.withValue("label", "Page")));
|
||||
STORE.insert(
|
||||
"auditUser", List.of(new QRecord().withValue("id", 1).withValue("name", "james")));
|
||||
}
|
||||
|
||||
static QRecord audit(long id) throws Exception {
|
||||
QRecord row =
|
||||
new QRecord()
|
||||
.withValue("id", id)
|
||||
.withValue("auditTableId", 1)
|
||||
.withValue("auditUserId", 1)
|
||||
.withValue("recordId", 17)
|
||||
.withValue("message", "Record was Edited")
|
||||
.withValue("timestamp", OLD.plusSeconds(id));
|
||||
return (STORE.insert("audit", List.of(row)).getFirst());
|
||||
}
|
||||
|
||||
static QRecord detail(long id, long auditId) throws Exception {
|
||||
return (STORE.insert(
|
||||
"auditDetail",
|
||||
List.of(
|
||||
new QRecord()
|
||||
.withValue("id", id)
|
||||
.withValue("auditId", auditId)
|
||||
.withValue("fieldName", "title")
|
||||
.withValue("message", "Changed Title")
|
||||
.withValue("oldValue", "old")
|
||||
.withValue("newValue", "new")))
|
||||
.getFirst());
|
||||
}
|
||||
|
||||
static void tamper(String table, long id, String field, java.io.Serializable value)
|
||||
throws Exception {
|
||||
MemoryRecordStore.getInstance()
|
||||
.update(
|
||||
new UpdateInput(table)
|
||||
.withRecords(
|
||||
List.of(
|
||||
new QRecord()
|
||||
.withValue(
|
||||
"id",
|
||||
table.equals("auditTable")
|
||||
|| table.equals(
|
||||
"auditUser")
|
||||
? (java.io.Serializable)
|
||||
(int) id
|
||||
: id)
|
||||
.withValue(field, value))),
|
||||
false);
|
||||
}
|
||||
|
||||
/** What the shipped database script does for the singleton: insert row 1 from the host config, only when absent. */
|
||||
public static void seedSettings(WormAuditQBitConfig config) throws Exception {
|
||||
if (STORE.get(com.kingsrook.qbits.wormaudit.model.WormAuditTables.SETTINGS, 1) == null) {
|
||||
STORE.insert(
|
||||
com.kingsrook.qbits.wormaudit.model.WormAuditTables.SETTINGS,
|
||||
List.of(WormAuditSettings.defaults(config)));
|
||||
}
|
||||
config.invalidateSettings();
|
||||
}
|
||||
|
||||
static void remove(String table, long id) throws Exception {
|
||||
MemoryRecordStore.getInstance().delete(new DeleteInput(table).withPrimaryKeys(List.of(id)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.chain;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Floors move only after complete sweeps, only to ids observed before the
|
||||
** horizon, and reset when the tables shrink below them.
|
||||
*******************************************************************************/
|
||||
class SweepFloorsTest
|
||||
{
|
||||
private static final Instant T0 = Instant.parse("2026-10-15T00:00:00Z");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Floors follow the newest observation at or before the horizon.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void floorsAdvanceOnlyPastSettledObservationsOfCompleteSweeps()
|
||||
{
|
||||
SweepFloors floors = SweepFloors.of(new WormAuditQBitConfig());
|
||||
var first = floors.observe(T0, 10, 100);
|
||||
floors.swept(first, T0.minusSeconds(1), true, true, null);
|
||||
assertThat(floors.audit()).as("nothing observed before the horizon yet").isZero();
|
||||
|
||||
var second = floors.observe(T0.plusSeconds(30), 12, 130);
|
||||
floors.swept(second, T0.plusSeconds(10), false, true, null);
|
||||
assertThat(floors.audit()).as("the header sweep hit its limit").isZero();
|
||||
assertThat(floors.detail()).isEqualTo(100);
|
||||
|
||||
var third = floors.observe(T0.plusSeconds(60), 15, 160);
|
||||
floors.swept(third, T0.plusSeconds(45), true, false, null);
|
||||
assertThat(floors.audit()).isEqualTo(12);
|
||||
assertThat(floors.detail()).as("the late-detail sweep hit its limit").isEqualTo(100);
|
||||
|
||||
var fourth = floors.observe(T0.plusSeconds(90), 15, 160);
|
||||
floors.swept(fourth, T0.plusSeconds(90), true, true, null);
|
||||
assertThat(floors.audit()).isEqualTo(15);
|
||||
assertThat(floors.detail()).isEqualTo(160);
|
||||
|
||||
var shrunk = floors.observe(T0.plusSeconds(120), 3, 7);
|
||||
assertThat(floors.audit()).as("tables emptied below the floor: start over").isZero();
|
||||
assertThat(floors.detail()).isZero();
|
||||
floors.swept(shrunk, T0.plusSeconds(100), true, true, null);
|
||||
assertThat(floors.audit()).isZero();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A full sweep is due first, then hourly after each complete one.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void fullSweepIsDueFirstAndHourly()
|
||||
{
|
||||
SweepFloors floors = SweepFloors.of(new WormAuditQBitConfig());
|
||||
assertThat(floors.fullSweepDue(T0)).isTrue();
|
||||
var first = floors.observe(T0, 1, 1);
|
||||
floors.swept(first, T0, true, false, T0);
|
||||
assertThat(floors.fullSweepDue(T0.plusSeconds(30))).as("an incomplete full sweep repeats").isTrue();
|
||||
var second = floors.observe(T0.plusSeconds(30), 1, 1);
|
||||
floors.swept(second, T0, true, true, T0.plusSeconds(30));
|
||||
assertThat(floors.fullSweepDue(T0.plusSeconds(60))).isFalse();
|
||||
assertThat(floors.fullSweepDue(T0.plusSeconds(30).plus(SweepFloors.FULL_SWEEP_EVERY))).isTrue();
|
||||
assertThat(floors.fullSweepDue(T0)).as("clock moved backwards").isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Each installed config has its own floors.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void floorsArePerConfig()
|
||||
{
|
||||
var config = new WormAuditQBitConfig();
|
||||
assertThat(SweepFloors.of(config)).isSameAs(SweepFloors.of(config)).isNotSameAs(SweepFloors.of(new WormAuditQBitConfig()));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.demo;
|
||||
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Instant;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.objectstorage.api.ListRequest;
|
||||
import com.kingsrook.qbits.objectstorage.api.ObjectSummary;
|
||||
import com.kingsrook.qbits.objectstorage.api.Page;
|
||||
import com.kingsrook.qbits.objectstorage.api.StoredObject;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedStore;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Tiny test-scope immutable store with version and COMPLIANCE retention echoes.
|
||||
*******************************************************************************/
|
||||
public class MemorySealedStore implements SealedStore
|
||||
{
|
||||
private final Map<String, byte[]> objects = new LinkedHashMap<>();
|
||||
private final Map<String, StoredObject> heads = new LinkedHashMap<>();
|
||||
|
||||
/*******************************************************************************
|
||||
** Demo capability.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean supportsLock()
|
||||
{
|
||||
return (true);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Refuse overwrites and retain the exact bytes.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String put(String key, Path file, com.kingsrook.qbits.objectstorage.api.LockMode mode, Instant until) throws Exception
|
||||
{
|
||||
if(objects.containsKey(key))
|
||||
{
|
||||
throw (new IllegalStateException("Demo sealed object already exists"));
|
||||
}
|
||||
String version = "v" + (objects.size() + 1);
|
||||
objects.put(key, Files.readAllBytes(file));
|
||||
heads.put(key, new StoredObject().withKey(key).withVersionId(version).withLockMode(mode.name()).withLockRetainUntil(until));
|
||||
return (version);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exact retention metadata.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public StoredObject head(String key)
|
||||
{
|
||||
return (heads.get(key));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exact version read.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public InputStream getVersion(String key, String versionId)
|
||||
{
|
||||
if(!heads.containsKey(key) || !heads.get(key).getVersionId().equals(versionId))
|
||||
{
|
||||
throw (new IllegalArgumentException("Demo sealed version is missing"));
|
||||
}
|
||||
return (new ByteArrayInputStream(objects.get(key)));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The small demo fits one page.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public Page<ObjectSummary> list(ListRequest request)
|
||||
{
|
||||
return (new Page<ObjectSummary>().withItems(objects.keySet().stream().filter(k -> k.startsWith(request.getPrefix()))
|
||||
.map(k -> new ObjectSummary().withKey(k)).toList()));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.demo;
|
||||
|
||||
|
||||
import java.security.KeyPairGenerator;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.wormaudit.PhiCanaryTest;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitProducer;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedExporter;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.VerificationRuns;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.actions.audits.AuditActionCustomizerInterface;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.instances.AbstractQQQApplication;
|
||||
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.audits.AuditsMetaDataProvider;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
|
||||
import com.kingsrook.qqq.middleware.javalin.QApplicationJavalinServer;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Local demo: synthetic notes, test HMAC tokens, real chain, checkpoints,
|
||||
** verifications and sealed exports over a moving clock. Memory storage and its
|
||||
** simulated purge are not production enforcement.
|
||||
**
|
||||
** <pre>
|
||||
** ./mvnw -o test-compile exec:java -Dexec.classpathScope=test -Dexec.mainClass=com.kingsrook.qbits.wormaudit.demo.WormAuditDemoServer -Dexec.args=8099
|
||||
** </pre>
|
||||
*******************************************************************************/
|
||||
public class WormAuditDemoServer extends AbstractQQQApplication
|
||||
{
|
||||
private static Clock clock = Clock.systemUTC();
|
||||
private final MemorySealedStore storage = new MemorySealedStore();
|
||||
private WormAuditQBitConfig config;
|
||||
|
||||
/*******************************************************************************
|
||||
** The PhiCanaryTest stand-in, with a moving timestamp before audit insertion.
|
||||
*******************************************************************************/
|
||||
public static class DemoTokenHook extends PhiCanaryTest.TokenHook
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Tokenize through the same core customization point as the canary test.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void customizeInput(AuditSingleInput input)
|
||||
{
|
||||
input.withTimestamp(clock.instant());
|
||||
super.customizeInput(input);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fully anonymous demo with a FIELD-audited synthetic note table.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QInstance defineQInstance() throws QException
|
||||
{
|
||||
try
|
||||
{
|
||||
var keys = KeyPairGenerator.getInstance("Ed25519").generateKeyPair();
|
||||
config = new WormAuditQBitConfig().withBackendName("memory").withCheckpointEveryLinks(20).withCheckpointEveryMinutes(1)
|
||||
.withSigningKey(() -> new WormAuditQBitConfig.SigningKey("demo", keys.getPrivate())).withTrustedPublicKeys(Map.of("demo", keys.getPublic()))
|
||||
.withSealedStoreSupplier(() -> storage)
|
||||
// Synthetic local demo only: anonymous readers can view its tables.
|
||||
.withTableMetaDataCustomizer((instance, table) -> table.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED)));
|
||||
QInstance instance = new QInstance().withInstanceDefaultAuthentication(new QAuthenticationMetaData()
|
||||
.withName("anonymous").withType(QAuthenticationType.FULLY_ANONYMOUS));
|
||||
instance.addBackend(new QBackendMetaData().withName("memory").withBackendType(MemoryBackendModule.class));
|
||||
new AuditsMetaDataProvider().defineAll(instance, "memory", null);
|
||||
instance.addTable(new QTableMetaData().withName("patient_note").withLabel("Synthetic patient note").withBackendName("memory")
|
||||
.withPrimaryKeyField("id").withRecordLabelFields("id").withRecordLabelFormat("Note #%s")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("diagnosis", QFieldType.STRING).withLabel("Diagnosis").withSupplementalMetaData(new PhiCanaryTest.Phi()))
|
||||
.withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD)));
|
||||
instance.addSupplementalCustomizer(AuditActionCustomizerInterface.CUSTOMIZER_TYPE, new QCodeReference(DemoTokenHook.class));
|
||||
new WormAuditQBitProducer().withQBitConfig(config).produce(instance);
|
||||
instance.setHomeAppName(config.getAppName());
|
||||
return (instance);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw (new QException("Cannot define WORM Audit demo", e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Write through core DML and the test token hook at the current demo time.
|
||||
*******************************************************************************/
|
||||
private void notes(int count) throws QException
|
||||
{
|
||||
for(int i = 0; i < count; i++)
|
||||
{
|
||||
new InsertAction().execute(new InsertInput("patient_note").withRecord(new QRecord().withValue("diagnosis", "SYNTHETIC-DEMO-NOTE-" + i)));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Persist a real verifier result with the historical run time.
|
||||
*******************************************************************************/
|
||||
private void verify() throws QException
|
||||
{
|
||||
var result = new WormAuditVerifier(config).verify(null, clock.instant());
|
||||
VerificationRuns.save(result, "SCHEDULE", null, clock.instant());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Use the production exporter and configured storage boundary.
|
||||
*******************************************************************************/
|
||||
private void seal() throws QException
|
||||
{
|
||||
var result = new SealedExporter(config).run("SCHEDULE", clock.instant());
|
||||
if(result.failed() || result.row() == null)
|
||||
{
|
||||
throw (new QException("Demo seeding failed to seal its checkpointed range"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A fortnight of activity and a genuinely expired, sealed prefix.
|
||||
*******************************************************************************/
|
||||
public void seed(QInstance instance) throws Exception
|
||||
{
|
||||
QContext.init(instance, new QSession());
|
||||
com.kingsrook.qbits.wormaudit.WormAuditTestSupport.seedSettings(config);
|
||||
AuditStore store = new AuditStore(null);
|
||||
Instant today = Instant.now().minus(Duration.ofHours(2));
|
||||
try
|
||||
{
|
||||
Instant expired = today.atZone(ZoneOffset.UTC).minusYears(7).toInstant();
|
||||
clock = Clock.fixed(expired, ZoneOffset.UTC);
|
||||
notes(3);
|
||||
clock = Clock.fixed(expired.plusSeconds(120), ZoneOffset.UTC);
|
||||
new WormAuditChainer(config).chainOnce(clock.instant());
|
||||
clock = Clock.fixed(expired.plusSeconds(240), ZoneOffset.UTC);
|
||||
new WormAuditChainer(config).chainOnce(clock.instant());
|
||||
verify();
|
||||
seal();
|
||||
QRecord anchor = store.last(WormAuditTables.CHAIN);
|
||||
long auditCount = store.count("audit", new QQueryFilter());
|
||||
long detailCount = store.count("auditDetail", new QQueryFilter());
|
||||
// Demo data: memory has no purge function. Remove the expired, sealed
|
||||
// prefix and append the receipt the PostgreSQL function would write.
|
||||
for(String table : List.of("auditDetail", "audit", WormAuditTables.CHAIN))
|
||||
{
|
||||
for(QRecord row : store.all(table))
|
||||
{
|
||||
MemoryRecordStore.getInstance().delete(new DeleteInput(table).withPrimaryKeys(List.of(row.getValue("id"))));
|
||||
}
|
||||
}
|
||||
store.insert(WormAuditTables.PURGE, List.of(new QRecord().withValue("requested_at", today.minus(Duration.ofDays(15)))
|
||||
.withValue("requested_by", "demo-purger").withValue("cutoff", expired.plusSeconds(1)).withValue("retention_interval", "6 years")
|
||||
.withValue("outcome", "PURGED").withValue("reason", "Expired sealed prefix removed (demo memory purge)")
|
||||
.withValue("audits_deleted", auditCount).withValue("details_deleted", detailCount).withValue("links_deleted", anchor.getValueLong("id"))
|
||||
.withValue("anchor_link_id", anchor.getValueLong("id")).withValue("anchor_hash", anchor.getValueString("link_hash"))));
|
||||
for(int day = 13; day >= 0; day--)
|
||||
{
|
||||
Instant time = today.minus(Duration.ofDays(day));
|
||||
clock = Clock.fixed(time, ZoneOffset.UTC);
|
||||
notes(25);
|
||||
clock = Clock.fixed(time.plusSeconds(120), ZoneOffset.UTC);
|
||||
new WormAuditChainer(config).chainOnce(clock.instant());
|
||||
clock = Clock.fixed(time.plusSeconds(240), ZoneOffset.UTC);
|
||||
new WormAuditChainer(config).chainOnce(clock.instant());
|
||||
if(day == 7)
|
||||
{
|
||||
// Demo seeding only: temporarily tamper a memory detail, persist a
|
||||
// genuine FAIL naming that link, then restore before later PASSes.
|
||||
QRecord detail = store.all("auditDetail").getFirst();
|
||||
var original = detail.getValue("newValue");
|
||||
memoryValue(detail, "hmac:v1:demo:tampered");
|
||||
verify();
|
||||
memoryValue(detail, original);
|
||||
}
|
||||
else
|
||||
{
|
||||
verify();
|
||||
}
|
||||
seal();
|
||||
}
|
||||
// Demo REFUSED receipt: too-recent cutoff; no rows removed.
|
||||
store.insert(WormAuditTables.PURGE, List.of(new QRecord().withValue("requested_at", today.plusSeconds(300))
|
||||
.withValue("requested_by", "demo-purger").withValue("cutoff", today).withValue("retention_interval", "6 years")
|
||||
.withValue("outcome", "REFUSED").withValue("reason", "Cutoff is inside the retention interval")
|
||||
.withValue("audits_deleted", 0L).withValue("details_deleted", 0L).withValue("links_deleted", 0L)));
|
||||
// Real unsealed suffix and unchained backlog; no invented export gap.
|
||||
clock = Clock.fixed(today.plusSeconds(600), ZoneOffset.UTC);
|
||||
notes(5);
|
||||
new WormAuditChainer(config).chainOnce(today.plusSeconds(720));
|
||||
clock = Clock.fixed(Instant.now().minusSeconds(120), ZoneOffset.UTC);
|
||||
notes(2);
|
||||
clock = Clock.systemUTC();
|
||||
notes(1);
|
||||
}
|
||||
finally
|
||||
{
|
||||
clock = Clock.systemUTC();
|
||||
QContext.clear();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Deliberate memory-only tamper used solely while seeding the FAIL receipt.
|
||||
*******************************************************************************/
|
||||
private static void memoryValue(QRecord detail, java.io.Serializable value) throws QException
|
||||
{
|
||||
MemoryRecordStore.getInstance().update(new UpdateInput("auditDetail").withRecords(List.of(new QRecord()
|
||||
.withValue("id", detail.getValue("id")).withValue("newValue", value))), false);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Serve the standard widget API and packaged Next UI, reusing the seeded keys.
|
||||
*******************************************************************************/
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
int port = args.length == 0 ? 8099 : Integer.parseInt(args[0]);
|
||||
WormAuditDemoServer application = new WormAuditDemoServer();
|
||||
QInstance instance = application.defineQInstance();
|
||||
new QInstanceValidator().validate(instance);
|
||||
application.seed(instance);
|
||||
new QApplicationJavalinServer(new AbstractQQQApplication()
|
||||
{
|
||||
@Override
|
||||
public QInstance defineQInstance()
|
||||
{
|
||||
return (instance);
|
||||
}
|
||||
}).withPort(port).withServeFrontendNext(true).start();
|
||||
System.out.println("WORM Audit demo on http://localhost:" + port + "/app/worm_audit");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.verify;
|
||||
|
||||
|
||||
import java.security.KeyPairGenerator;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.canonical.AuditCanonical;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.chain.Hashes;
|
||||
import com.kingsrook.qbits.wormaudit.checkpoint.Checkpoints;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Instrument the storage boundary independently of the memory backend: no
|
||||
** unbounded reads are allowed, including portable completeness checks.
|
||||
*******************************************************************************/
|
||||
class PagedVerificationTest
|
||||
{
|
||||
private static final int LINKS = 25000;
|
||||
private static final int PAGE = 137;
|
||||
private static final int RANGE = 997;
|
||||
private static final Instant NOW = Instant.parse("2026-10-13T00:00:00Z");
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Cross many non-aligned page/checkpoint boundaries with genuine signatures.
|
||||
*******************************************************************************/
|
||||
@ParameterizedTest
|
||||
@ValueSource(booleans = { true, false })
|
||||
void fullRunBoundsPagesAndCheckpointRanges(boolean sql) throws Exception
|
||||
{
|
||||
var keys = KeyPairGenerator.getInstance("Ed25519").generateKeyPair();
|
||||
var config = new WormAuditQBitConfig().withCheckpointEveryLinks(RANGE)
|
||||
.withSigningKey(() -> new WormAuditQBitConfig.SigningKey("test", keys.getPrivate()))
|
||||
.withTrustedPublicKeys(Map.of("test", keys.getPublic()));
|
||||
var store = new InstrumentedStore(config, sql);
|
||||
var result = new WormAuditVerifier(config, PAGE).verify(store, null, NOW);
|
||||
assertThat(result.getBreaks()).isEmpty();
|
||||
assertThat(result.getLinksChecked()).isEqualTo(LINKS);
|
||||
assertThat(result.getFromLinkId()).isEqualTo(1);
|
||||
assertThat(result.getToLinkId()).isEqualTo(LINKS);
|
||||
assertThat(store.maxPage).isEqualTo(PAGE);
|
||||
assertThat(store.maxRange).isEqualTo(RANGE);
|
||||
assertThat(store.checkpointsRead).isEqualTo(26);
|
||||
assertThat(store.completenessVisited).isTrue();
|
||||
if(!sql)
|
||||
{
|
||||
store.links.get(0).withValue("detail_ids", "1,1,2");
|
||||
store.links.get(1).withValue("detail_ids", "2");
|
||||
store.links.get(2).withValue("detail_ids", "malformed");
|
||||
store.links.get(3).withValue("audit_id", 1L);
|
||||
store.audits.get(2).withValue("timestamp", null);
|
||||
store.audits.get(4).withValue("timestamp", NOW);
|
||||
store.details.addAll(List.of(new QRecord().withValue("id", 1L).withValue("auditId", 1L),
|
||||
new QRecord().withValue("id", 2L).withValue("auditId", 2L),
|
||||
new QRecord().withValue("id", 3L).withValue("auditId", 99999L),
|
||||
new QRecord().withValue("id", 4L).withValue("auditId", 3L)));
|
||||
var broken = new WormAuditVerifier(config, PAGE).verify(store, null, NOW);
|
||||
assertThat(broken.getBreaks()).extracting(finding -> finding.reason()).contains(
|
||||
"audit 1 is covered twice or more by the chain", "audit 4 is not covered by the chain",
|
||||
"audit 3 has no timestamp", "audit_detail 1 is covered twice or more by the chain",
|
||||
"audit_detail 2 is covered twice or more by the chain", "audit_detail 3 is not covered by the chain",
|
||||
"audit_detail 4 is not covered by the chain");
|
||||
}
|
||||
assertThatThrownBy(() -> new WormAuditVerifier(config, 0)).isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatThrownBy(() -> new WormAuditVerifier(config, 1001)).isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fixture storage may retain its data; the verifier can only fetch bounded
|
||||
** pages. Track the total hashes read between successive checkpoint requests.
|
||||
*******************************************************************************/
|
||||
private static class InstrumentedStore extends AuditStore
|
||||
{
|
||||
private final List<QRecord> audits = new ArrayList<>();
|
||||
private final List<QRecord> details = new ArrayList<>();
|
||||
private final List<QRecord> links = new ArrayList<>();
|
||||
private final List<QRecord> checkpoints = new ArrayList<>();
|
||||
private final boolean sql;
|
||||
private int maxPage;
|
||||
private int maxRange;
|
||||
private int rangeRead;
|
||||
private int checkpointsRead;
|
||||
private boolean inCheckpoints;
|
||||
private boolean completenessVisited;
|
||||
private boolean mainScanFinished;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Generate signed, valid content without involving the backend under test.
|
||||
*******************************************************************************/
|
||||
InstrumentedStore(WormAuditQBitConfig config, boolean sql) throws Exception
|
||||
{
|
||||
super(null);
|
||||
this.sql = sql;
|
||||
String previous = Hashes.GENESIS;
|
||||
for(long id = 1; id <= LINKS; id++)
|
||||
{
|
||||
QRecord audit = new QRecord().withValue("id", id).withValue("timestamp", NOW.minusSeconds(120))
|
||||
.withValue("auditTableId", 1).withValue("auditUserId", 1).withValue("recordId", id);
|
||||
audits.add(audit);
|
||||
String hash = Hashes.link(previous, AuditCanonical.bytes(audit, "fixture", "fixture", List.of()));
|
||||
links.add(new QRecord().withValue("id", id).withValue("audit_id", id).withValue("kind", "AUDIT")
|
||||
.withValue("detail_ids", "").withValue("prev_hash", previous).withValue("link_hash", hash).withValue("chained_at", NOW));
|
||||
previous = hash;
|
||||
}
|
||||
for(int start = 0; start < LINKS; start += RANGE)
|
||||
{
|
||||
checkpoints.add(Checkpoints.sign(links.subList(start, Math.min(start + RANGE, LINKS)), config, NOW)
|
||||
.withValue("id", (long) checkpoints.size() + 1));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** No full-table read is ever permitted through this boundary.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> all(String table)
|
||||
{
|
||||
throw (new AssertionError("Unbounded verifier read: " + table));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A fresh copied page prevents subList from retaining the fixture chain.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> page(String table, long after, int limit)
|
||||
{
|
||||
assertThat(limit).isBetween(1, PAGE);
|
||||
maxPage = Math.max(maxPage, limit);
|
||||
List<QRecord> source;
|
||||
if(table.equals(WormAuditTables.CHECKPOINT))
|
||||
{
|
||||
assertThat(mainScanFinished).isTrue();
|
||||
assertThat(limit).isEqualTo(1);
|
||||
maxRange = Math.max(maxRange, rangeRead);
|
||||
assertThat(rangeRead).isLessThanOrEqualTo(RANGE);
|
||||
rangeRead = 0;
|
||||
inCheckpoints = after < checkpoints.size();
|
||||
if(inCheckpoints)
|
||||
{
|
||||
checkpointsRead++;
|
||||
}
|
||||
source = checkpoints;
|
||||
}
|
||||
else if(table.equals(WormAuditTables.CHAIN))
|
||||
{
|
||||
source = links;
|
||||
if(after == LINKS && !inCheckpoints)
|
||||
{
|
||||
mainScanFinished = true;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
completenessVisited = true;
|
||||
source = table.equals("audit") ? audits : details;
|
||||
}
|
||||
int start = (int) Math.min(after, source.size());
|
||||
var page = new ArrayList<>(source.subList(start, Math.min(start + limit, source.size())));
|
||||
if(inCheckpoints && table.equals(WormAuditTables.CHAIN))
|
||||
{
|
||||
rangeRead += page.size();
|
||||
}
|
||||
return (page);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Batch only the audit headers belonging to the current chain page.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> in(String table, String field, Collection<Long> ids)
|
||||
{
|
||||
assertThat(ids.size()).isLessThanOrEqualTo(PAGE);
|
||||
return (ids.stream().distinct().map(id -> (table.equals("audit") ? audits : details).get(id.intValue() - 1)).toList());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve detail parents without collecting audit history.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QRecord get(String table, long id)
|
||||
{
|
||||
return (id > audits.size() ? null : audits.get((int) id - 1));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stable lookup values match the fixture canonical documents.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String name(String table, Long id)
|
||||
{
|
||||
return ("fixture");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Start at genesis.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QRecord anchor()
|
||||
{
|
||||
return (new QRecord().withValue("anchor_link_id", 0L).withValue("anchor_hash", Hashes.GENESIS));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise both SQL dispatch and portable non-memory completeness.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean isRdbms()
|
||||
{
|
||||
return (sql);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Never permit the memory fallback in this test.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean isMemory()
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** SQL completeness has separate acceptance tests; verify dispatch here.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void completeness(Instant cutoff, VerificationResult result)
|
||||
{
|
||||
completenessVisited = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Configuration status="WARN">
|
||||
<Appenders>
|
||||
<Console name="Console" target="SYSTEM_OUT">
|
||||
<!-- Core can emit diagnostic events at OFF; exclude those as well. -->
|
||||
<LevelRangeFilter minLevel="FATAL" maxLevel="WARN" onMatch="NEUTRAL" onMismatch="DENY"/>
|
||||
<PatternLayout pattern="%d{HH:mm:ss} %-5level %logger{36} - %msg%n"/>
|
||||
</Console>
|
||||
</Appenders>
|
||||
<Loggers>
|
||||
<Logger name="org.testcontainers" level="WARN"/>
|
||||
<Logger name="com.github.dockerjava" level="WARN"/>
|
||||
<Logger name="org.apache.hc" level="WARN"/>
|
||||
<Logger name="tc" level="WARN"/>
|
||||
<Root level="WARN"><AppenderRef ref="Console"/></Root>
|
||||
</Loggers>
|
||||
</Configuration>
|
||||