Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b879a1ac1a
|
||
|
|
fc4375adb5
|
||
|
|
e89ea02b22
|
||
|
|
b904359bea
|
||
|
|
bb015dcc1d
|
||
|
|
54bef4a36b | ||
|
|
db2c2b6755
|
||
|
|
b2d9b536e7
|
||
|
|
e157ff5038
|
||
|
|
29fc15a10a
|
||
|
|
dc6361b4c2
|
||
|
|
2eaf49eccb
|
||
|
|
e1d3986acc
|
||
|
|
be0d49d9db | ||
|
|
d93b279629 | ||
|
|
2c470560a3 | ||
|
|
c8ea4b627b | ||
|
|
0ce28313f9 | ||
|
|
a8980eec0f | ||
|
|
082f3b95f0 | ||
|
|
c681b2c0e7 | ||
|
|
558effe85f
|
||
|
|
93f6f86f2d | ||
|
|
7e15fb92e0 | ||
|
|
a5d7192b95
|
||
|
|
05fcbdd8b8
|
||
|
|
d78ecc1bc4 | ||
|
|
e81c8ba37e
|
||
|
|
a9e985a86c | ||
|
|
bde8ff416f
|
||
|
|
92c5fc9ca4 | ||
|
|
86f9929cd2
|
||
|
|
4842cb174d | ||
|
|
18ae80740b
|
||
|
|
3c083c996e | ||
|
|
6c3acb7e53
|
||
|
|
ff0e5cbe73 | ||
|
|
2b66492571
|
||
|
|
062a95e30a | ||
|
|
4d2a735992
|
||
|
|
a2fad5103d
|
||
|
|
4194837610
|
||
|
|
5736bf3e1f | ||
|
|
f14a24fdbf
|
||
|
|
0fb9e8912c | ||
|
|
626e153841 | ||
|
|
85de1daf79 | ||
|
|
be26a370ac | ||
|
|
3c6e7d57a2 | ||
|
|
4b6717ebdd
|
||
|
|
9c2766e324
|
No files matched your search
@@ -1,65 +1,14 @@
|
||||
version: 2.1
|
||||
|
||||
setup: true
|
||||
orbs:
|
||||
qqq-orb: kingsrook/qqq-orb@0.6.8
|
||||
|
||||
commands:
|
||||
configure-package-install:
|
||||
steps:
|
||||
- run:
|
||||
name: Keep package installation unattended
|
||||
command: |
|
||||
# These build machines are discarded after the job.
|
||||
sudo install -d /etc/needrestart/conf.d
|
||||
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
|
||||
|
||||
qqq-orb: kingsrook/qqq-orb@0.8
|
||||
workflows:
|
||||
|
||||
test_only:
|
||||
setup:
|
||||
jobs:
|
||||
- qqq-orb/mvn_test_only:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [ qqq-maven-registry-credentials ]
|
||||
filters:
|
||||
branches:
|
||||
ignore: /(develop|release/.*|integration.*)/
|
||||
tags:
|
||||
only: []
|
||||
|
||||
publish_snapshot:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [ qqq-maven-registry-credentials ]
|
||||
branch_type: snapshot
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- develop
|
||||
|
||||
publish_release_candidate:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [ qqq-maven-registry-credentials ]
|
||||
branch_type: release_candidate
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- /release\/.*/
|
||||
|
||||
publish_release:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [ qqq-maven-registry-credentials ]
|
||||
branch_type: release
|
||||
filters:
|
||||
branches:
|
||||
ignore: /.*/
|
||||
tags:
|
||||
only: [ /^v\d+\.\d+\.\d+$/ ]
|
||||
- qqq-orb/generate_pipeline:
|
||||
orb_version: '0.8'
|
||||
filters:
|
||||
branches:
|
||||
ignore: automation/qbit-rc-pending
|
||||
tags:
|
||||
only: /^v\d+\.\d+\.\d+$/
|
||||
@@ -0,0 +1,7 @@
|
||||
name: build
|
||||
on:
|
||||
push:
|
||||
jobs:
|
||||
maven:
|
||||
uses: self:KofTwentyTwo/ci-workflows/.gitea/workflows/maven.yaml@v1
|
||||
secrets: inherit
|
||||
@@ -0,0 +1,23 @@
|
||||
name: CodeQL
|
||||
'on':
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- develop
|
||||
- release/**
|
||||
- hotfix/**
|
||||
pull_request: {}
|
||||
schedule:
|
||||
- cron: 41 16 * * 0
|
||||
workflow_dispatch: {}
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
security-events: write
|
||||
packages: read
|
||||
concurrency:
|
||||
group: codeql-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
jobs:
|
||||
analyze:
|
||||
uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22
|
||||
@@ -1,25 +1,74 @@
|
||||
# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
|
||||
# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
|
||||
# the release line and the release/* push runs CircleCI publish_release_candidate.
|
||||
name: Release candidate
|
||||
|
||||
on:
|
||||
'on':
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: Print the plan without merging or creating branches
|
||||
description: Plan a cut without changing release state
|
||||
type: boolean
|
||||
default: false
|
||||
finalize:
|
||||
description: Retry finalization of an existing cut; do not cut another RC
|
||||
type: boolean
|
||||
default: false
|
||||
recover:
|
||||
description: Recover the same published RC after a deployment timeout; never
|
||||
reupload
|
||||
type: boolean
|
||||
default: false
|
||||
circle_build:
|
||||
description: Original Circle publisher build number (recovery only)
|
||||
type: string
|
||||
default: ''
|
||||
merge_sha:
|
||||
description: Existing cut merge_sha (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
bump_sha:
|
||||
description: Existing cut bump_sha (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
branch:
|
||||
description: Existing cut branch (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
version:
|
||||
description: Existing cut version (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
pull_request:
|
||||
types: [closed]
|
||||
branches: [develop]
|
||||
|
||||
types:
|
||||
- closed
|
||||
branches:
|
||||
- develop
|
||||
repository_dispatch:
|
||||
types:
|
||||
- qbit-rc-published
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
actions: read
|
||||
checks: read
|
||||
jobs:
|
||||
release-rc:
|
||||
if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
|
||||
uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
|
||||
if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
|
||||
|| (github.event_name == 'pull_request' && github.event.pull_request.merged
|
||||
== true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
|
||||
'release:rc'))
|
||||
uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
|
||||
with:
|
||||
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
|
||||
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
|
||||
}}
|
||||
finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
|
||||
== 'workflow_dispatch' && inputs.finalize) }}
|
||||
recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
|
||||
}}
|
||||
circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
|
||||
|| '' }}
|
||||
merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
|
||||
|| inputs.merge_sha || '' }}
|
||||
bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
|
||||
|| inputs.bump_sha || '' }}
|
||||
branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
|
||||
|| inputs.branch || '' }}
|
||||
version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
|
||||
|| inputs.version || '' }}
|
||||
secrets: inherit
|
||||
@@ -0,0 +1,16 @@
|
||||
profile: qbit
|
||||
orb_version: '0.8'
|
||||
default_branch: develop
|
||||
java_version: '21'
|
||||
browsers: true
|
||||
api_checks: false
|
||||
docs_only: false
|
||||
release_on_main: false
|
||||
release_tag_policy: stable
|
||||
test_branch_policy: non-release-no-integration
|
||||
publish:
|
||||
snapshot: true
|
||||
release_candidate: true
|
||||
release: true
|
||||
asciidoc: false
|
||||
release_completion: true
|
||||
@@ -29,8 +29,8 @@ Package: `com.kingsrook.qbits.wormaudit`. Entry point: `WormAuditQBitProducer im
|
||||
|
||||
PostgreSQL uses a transaction advisory lock; other backends require a single writer JVM.
|
||||
Use `./mvnw.cmd -o verify` offline. PostgreSQL tests skip without Docker locally, but fail whenever `CI` is set.
|
||||
Core RC.5 has no assistant deny API (reflective registration) and disallows underscores in declared alert-type metadata;
|
||||
signals retain the required `worm_audit.chain.break` name and dispatch through QAlerts without that declaration.
|
||||
Core RC.5 has no assistant deny API (reflective registration). Alert types use the
|
||||
`wormaudit.*` namespace and are declared through core alert-type metadata.
|
||||
|
||||
## Invariants and naming
|
||||
|
||||
|
||||
@@ -4,11 +4,41 @@ All notable changes are documented here, following [Keep a Changelog](https://ke
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Fixed
|
||||
|
||||
- `worm-audit:13` failed with `permission denied for function worm_audit_purge` when the migrating role owns the function but had revoked its own EXECUTE (a runtime role that owns the schema, as on CNPG; PostgreSQL refuses `CREATE OR REPLACE` then). The changeset now grants EXECUTE back to the migrating role only if it lacks it, replaces the function, and revokes it again in the same transaction. Found by Website-Backend's CNPG-shaped changelog test.
|
||||
- `worm-audit:13` had a comment line starting with `-- changeset`, which Liquibase formatted-SQL parsing rejects as a malformed directive. The comment is reworded, and `CanonicalAndCryptoTest` now requires every `-- changeset` line to be a real directive.
|
||||
- Chain writer database load. Each 30-second tick re-read the whole late-detail lookback (24 h of `audit_detail`) and probed the GIN array index once per detail (18.4 s and 1.78M buffer hits per tick on a 120k-detail production database), and re-scanned every settled audit header for the unchained anti-join. Coverage now lives in `worm_audit_chain_detail` (`worm-audit:13`): one insert-only row per covered detail, written by the chain trigger in the same statement as its link and backfilled from the existing chain, so the late-detail check is a primary-key probe. Each tick also reads only ids above per-JVM sweep floors (rows settled since the last complete sweep); after a restart and once an hour a tick sweeps everything, as before. Link hash inputs, `detail_ids`, the verifier and exports are unchanged.
|
||||
- The chain trigger's overlap check (`EXISTS ... && ...`) ran as a cached generic plan that sequentially scanned all of `worm_audit_chain` for every inserted link (55k sequential scans, 1.2B tuples read in production, about 80 ms per link and growing). The trigger now inserts the coverage rows and lets their primary key reject a detail covered twice.
|
||||
- Verification counts detail coverage by aggregating the chain's own detail lists once instead of one index probe per detail, and breaks when the coverage table disagrees with the chain.
|
||||
|
||||
- Startup on hosts whose default authentication requires a token (OAuth2, Auth0): `produce()` read the settings singleton under a plain tokenless `QSession`, which failed session validation (`JWTDecodeException: The token is null`) and stopped the host. The startup read now runs as the core `QSystemUserSession`, still honoring the persisted schedules. `TokenRequiredHostTest` runs production under a token-required test authentication module.
|
||||
|
||||
### Added
|
||||
|
||||
- Foreign keys (`worm-audit:11-12`), `ON DELETE RESTRICT` only: chain link -> core audit header, sealed export -> its first and last checkpoints, export verification -> its export, with their indexes. Link references (ranges, break link, purge anchor) stay unconstrained because the retention purge deletes the links they outlive. Pre-existing orphans are copied to `qqq_fk_quarantine` (`KEPT`), never modified, and leave that constraint `NOT VALID`. Re-apply the rendered DDL (idempotent).
|
||||
- Guided empty states: `help/qbit-worm-audit.yaml` (QQQ help-bundle format) gives the app and all six tables a summary and a "How rows get created" section, applied by the `WormAuditHelp` stand-in (`STAND-IN(core=QRun-IO/qqq#1198, removeBy=4.1.0-RC.7)`) without replacing host help. `WormAuditTablesGuardrailTest` pins runtime use, seeding or a documented reason, and help for every table; `WormAuditSettingsSeedTest` runs the script's settings seed on H2.
|
||||
- RC.2 settings: editable, FIELD-audited singleton with insert-only defaults, validation, separate edit permission and a 30-second effective-settings cache. Schedules load at startup.
|
||||
- GOVERNANCE exports with 30-day default retention; per-export retention verification, administrator-bypass diagnostics, home warnings and documented COMPLIANCE transition.
|
||||
- New `worm-audit:10` settings migration; historical changesets remain unchanged.
|
||||
|
||||
- Phase (c): configurable WORM Audit app home, full-width summary widget and optional host health card; permission-aware navigation and content-free, responsive light/dark summaries.
|
||||
- Chain backlog/settling counts, verification and export record links, sealed-range gaps/lag, retention eligibility and recent purge/refusal history.
|
||||
- PostgreSQL guard/ownership/purge-function diagnostics, signing/trust warnings, tri-state Object Lock capability and stale daily-verification warnings.
|
||||
- Test-scope Next UI demo with synthetic tokenized patient notes, genuine verification failure/recovery, signed checkpoints, sealed memory exports and documented simulated purges.
|
||||
- Home privacy/permission/empty-state tests, memory/H2 summary tests and Docker-gated PostgreSQL setup-warning acceptance.
|
||||
|
||||
- Phase (a): PostgreSQL append-only tables/triggers, safe DDL renderer, retention purge with immutable refusal/anchor records, and runtime grants.
|
||||
- Canonical audit v1 documents, contiguous hash chain with late-detail coverage, pinned Ed25519 checkpoints and binary Merkle roots.
|
||||
- Snapshot verification, streamed ETL Review/Result action, scheduled chaining/daily verification, and core QAlerts raise/resolve/heartbeat integration.
|
||||
- Read-only metadata, WORM Audit table navigation, record range references and child lists; reflective RC.6 assistant deny registration.
|
||||
- Offline memory/H2 acceptance, real pre-insert PHI canary hook, and PostgreSQL Testcontainers acceptance with CI-strict Docker availability.
|
||||
- Adoption and operations documentation. Export execution and app-home widgets remain later phases.
|
||||
- Adoption, app-home, demo and operations documentation.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Keep runtime signing/trust objects and storage suppliers out of serialized metadata so signed-chain record child lists render successfully.
|
||||
|
||||
### Deprecated
|
||||
|
||||
- `WormAuditSettings.seed(config)` is deprecated for PostgreSQL hosts, whose database script (`worm-audit:10`) already seeds row 1; it remains supported for memory and H2 hosts. `read()` falls back to the configured values while the row is absent.
|
||||
@@ -29,8 +29,8 @@ Package: `com.kingsrook.qbits.wormaudit`. Entry point: `WormAuditQBitProducer im
|
||||
|
||||
PostgreSQL uses a transaction advisory lock; other backends require a single writer JVM.
|
||||
Use `./mvnw.cmd -o verify` offline. PostgreSQL tests skip without Docker locally, but fail whenever `CI` is set.
|
||||
Core RC.5 has no assistant deny API (reflective registration) and disallows underscores in declared alert-type metadata;
|
||||
signals retain the required `worm_audit.chain.break` name and dispatch through QAlerts without that declaration.
|
||||
Core RC.5 has no assistant deny API (reflective registration). Alert types use the
|
||||
`wormaudit.*` namespace and are declared through core alert-type metadata.
|
||||
|
||||
## Invariants and naming
|
||||
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# QBit WORM Audit
|
||||
|
||||
Phase (a) provides PostgreSQL append-only enforcement, an asynchronous audit hash chain,
|
||||
Ed25519 checkpoints, a verification action, and daily verification with core QAlerts signals.
|
||||
Phases (a)–(c) provide PostgreSQL append-only enforcement, an asynchronous audit hash chain,
|
||||
Ed25519 checkpoints, live and sealed-export verification actions, and daily jobs with core QAlerts signals.
|
||||
It hashes exactly the audit values core stored; it does not read application records or
|
||||
secured-data keys. Sealed exports and the app home are later phases. The export table and
|
||||
range references are present as integration hooks; no export job or home widgets are installed.
|
||||
secured-data keys. Sealed JSONL exports use qbit-object-storage 0.1.0-RC.2 and configurable
|
||||
Object Lock (GOVERNANCE for 30 days by default). The WORM Audit app home summarizes chain health, exports, retention and setup.
|
||||
|
||||
Requires Java 21, Maven wrapper 3.9.11 and QQQ 4.1.0-RC.5. PostgreSQL provides enforcement;
|
||||
memory/H2 support tamper-evidence tests without database enforcement.
|
||||
@@ -12,27 +12,31 @@ memory/H2 support tamper-evidence tests without database enforcement.
|
||||
## Tables and permissions
|
||||
|
||||
All sidecar tables use the configured host backend, snake_case fields, BIGINT identifiers,
|
||||
record sections, and full table-name permissions. They do not audit themselves. The app lists
|
||||
Verifications, Sealed Exports, Checkpoints, Audit Chain, and Purges in that order.
|
||||
record sections, and full table-name permissions. Immutable sidecars do not audit themselves.
|
||||
The editable Settings singleton uses an INTEGER ID and FIELD-level core audit. The app lists
|
||||
Verifications, Sealed Exports, Checkpoints, Audit Chain, Purges, and Settings in that order.
|
||||
|
||||
| Table | Purpose | Permission |
|
||||
|---|---|---|
|
||||
| `worm_audit_verification` | Immutable USER/SCHEDULE results | `worm_audit_verification.read` |
|
||||
| `worm_audit_export` | Reserved sealed-export catalog | `worm_audit_export.read` |
|
||||
| `worm_audit_export` | Successfully sealed export catalog | `worm_audit_export.read` |
|
||||
| `worm_audit_checkpoint` | Signed ranges and Merkle roots | `worm_audit_checkpoint.read` |
|
||||
| `worm_audit_chain` | Contiguous links and explicit detail coverage | `worm_audit_chain.read` |
|
||||
| `worm_audit_purge` | Refusals, removal counts and retention anchors | `worm_audit_purge.read` |
|
||||
| `worm_audit_settings` | Editable singleton; every edit is audited and chained | `worm_audit_settings.read`; editing needs `worm_audit_settings.edit` **and** `worm_audit.settings` |
|
||||
|
||||
`worm_audit.verify` authorizes the standard streamed ETL **Verify audit chain** action.
|
||||
Processes are attached to the chain table or scheduled; none is a navigation entry.
|
||||
Insert/update/delete capabilities are absent on sidecar tables; update/delete capabilities
|
||||
It also authorizes **Verify sealed export** on selected export records. `worm_audit.export`
|
||||
authorizes **Export now** on the export table. Processes are table/record actions or schedules;
|
||||
none is a navigation entry.
|
||||
Insert/update/delete capabilities are absent on immutable sidecar tables; Settings allows updates only; update/delete capabilities
|
||||
are removed from all four core audit tables. Internal appends use an explicit permission bypass.
|
||||
Checkpoint, export and verification record screens have a bounded chain-link child list;
|
||||
range endpoints and export/checkpoint references link to their records.
|
||||
|
||||
The table metadata customizer can configure host ESB publications. Quick Search registration
|
||||
is host-owned through `QuickSearchQBitConfig.searchableTables`; add all five tables.
|
||||
Verification inserts use QQQ's record-change listener SPI (as will phase-(b) export inserts),
|
||||
is host-owned through `QuickSearchQBitConfig.searchableTables`; add the desired tables.
|
||||
Verification and export inserts use QQQ's record-change listener SPI,
|
||||
so an installed ESB publisher receives those events when the host configures its destinations
|
||||
on these two tables. The qbit defines no destination and configures no chain-link publication.
|
||||
|
||||
@@ -41,11 +45,18 @@ on these two tables. The qbit defines no destination and configures no chain-lin
|
||||
| Field | Default / contract |
|
||||
|---|---|
|
||||
| `backendName` | Required; core and sidecar audit tables must use this backend |
|
||||
| `appName` / `appLabel` | `worm_audit` / `WORM Audit` |
|
||||
| `parentAppName` | Optional host app, for example `admin` |
|
||||
| `tablePathPrefix` | `/app`; standard QQQ table/record route prefix |
|
||||
| `sealedStoreSupplier` | Optional host/test storage boundary; defaults to qbit-object-storage |
|
||||
| `retentionYears` | 6; compiled into the migration, not changed by a runtime flag |
|
||||
| `signingKey` | Optional `Supplier<SigningKey>`; `SigningKey(keyId, PrivateKey)` |
|
||||
| `trustedPublicKeys` | `Map<String, PublicKey>`; must contain the signing key's public key |
|
||||
| `chainIntervalSeconds` | 30 |
|
||||
| `settleHorizonSeconds` | 60 (zero allowed) |
|
||||
| `exportBucketAlias` | `worm_audit`; qbit-object-storage BACKUP bucket alias |
|
||||
| `exportPrefix` | `worm-audit`; slash-separated letters, digits, underscores or hyphens |
|
||||
| `exportCron` | `0 0 2 * * ?`; 02:00 UTC on the configured scheduler |
|
||||
| `batchSize` | 1,000 headers per tick / links per insert batch |
|
||||
| `lateDetailLookbackHours` | 24; positive hours, older omissions checked by full verification |
|
||||
| `checkpointEveryLinks` | 10,000; maximum links in each checkpoint |
|
||||
@@ -61,6 +72,115 @@ to verification. It does not itself break the chain. Signing requires a matching
|
||||
Rotate by adding a new pin and changing the supplier; retain old pins for old checkpoints.
|
||||
Never store the private key in a database row or checked-in configuration.
|
||||
|
||||
## Settings
|
||||
|
||||
Open **WORM Audit → Settings**, record **1** (`worm_audit_settings`, labelled WORM Audit Settings).
|
||||
This is a standard editable QQQ table. New records and deletion are disabled; a pre-insert
|
||||
check and database constraint require ID 1. Every edit uses core FIELD-level auditing, so
|
||||
the change enters the audit chain. `modify_date` and `modified_by` are stamped by the server.
|
||||
Keep notes free of sensitive data; notes are audited too.
|
||||
|
||||
### Starter defaults and empty screens
|
||||
|
||||
The only seeded table is `worm_audit_settings`, by the database script (`WormAuditDdl.postgres(config)`, changeset `worm-audit:10`): `INSERT ... SELECT ... WHERE NOT EXISTS`, so it adds row 1 from the host's configured values only when absent and never overwrites an edit. There is no separate defaults changelog and no extra include line. If the row is absent the qbit runs on the configured values (`WormAuditSettings.read()` falls back), so nothing stops, but run the script to make the values visible and editable. The chain, checkpoint, export, verification and purge tables are logs written by the qbit's jobs and the database owner's purge function, and start empty.
|
||||
|
||||
Every table, and the app, ships help (`help/qbit-worm-audit.yaml`, the QQQ help-bundle format) saying what it is for and how its rows get created; the Next UI shows it as the empty state. A host's own help is never replaced. `WormAuditTablesGuardrailTest` and `WormAuditSettingsSeedTest` pin this.
|
||||
|
||||
| Field | Default | Validation / effect |
|
||||
|---|---|---|
|
||||
| `audit_retention_years` | 6 | 1–100; home next eligible purge date |
|
||||
| `export_lock_mode` | GOVERNANCE | GOVERNANCE or COMPLIANCE; new exports |
|
||||
| `export_retention_days` | 30 | 1–36500; new exports |
|
||||
| `checkpoint_every_links` | 10000 | Positive integer; new checkpoint ranges |
|
||||
| `checkpoint_every_minutes` | 60 | Positive integer; checkpoint cadence |
|
||||
| `chain_interval_seconds` | 30 | Positive integer; **takes effect after restart** |
|
||||
| `settle_horizon_seconds` | 60 | Nonnegative integer; chaining and verification |
|
||||
| `late_detail_lookback_hours` | 24 | Positive integer; late-detail polling |
|
||||
| `export_cron` | `0 0 2 * * ?` | Quartz cron, UTC; **takes effect after restart** |
|
||||
| `verify_cron` | `0 0 3 * * ?` | Quartz cron, UTC; **takes effect after restart** |
|
||||
| `notes` | Empty | Administrative notes |
|
||||
| `modify_date`, `modified_by` | Server supplied | Last edit time and user |
|
||||
|
||||
Config values supply defaults for missing rows or null fields; a missing settings table also
|
||||
falls back to config. Other database failures fail closed. `exportLockMode` and
|
||||
`exportRetentionDays` are the corresponding config defaults. Runtime reads cache the row
|
||||
for at most 30 seconds per application instance; local QQQ edits invalidate that cache.
|
||||
Each run reads effective settings. Startup schedules read the existing row, or config when
|
||||
absent. Historical checkpoints remain valid after changing their creation cadence.
|
||||
|
||||
Changeset `worm-audit:10` creates and seeds the table with `INSERT ... SELECT ... WHERE NOT
|
||||
EXISTS`; rerunning it never overwrites an administrator change. The renderer seeds from the
|
||||
host config. Memory and H2 hosts, whose settings table cannot take generic inserts, can still call
|
||||
`WormAuditSettings.seed(config)` for the same insert-only behavior during single-threaded
|
||||
startup (deprecated for PostgreSQL hosts, which get the row from the script).
|
||||
|
||||
`audit_retention_years` does **not** rewrite the database purge function. The home warns if
|
||||
it differs from the compiled interval; changing that interval requires a reviewed migration.
|
||||
The home also warns when export retention is shorter than audit retention and while
|
||||
GOVERNANCE permits administrator bypass deletion. The runtime role receives SELECT, INSERT
|
||||
and UPDATE on Settings, with no DELETE or TRUNCATE grant and no append-only triggers.
|
||||
|
||||
## Switching sealed exports to COMPLIANCE
|
||||
|
||||
Edit `export_lock_mode` to **COMPLIANCE** in Settings. New exports use COMPLIANCE once the
|
||||
settings cache refreshes (within 30 seconds); existing GOVERNANCE objects keep their mode
|
||||
unless an administrator explicitly extends their retention and upgrades their mode. The
|
||||
verifier continues to compare each object with its export row's recorded mode and date, so
|
||||
an out-of-band mode change is reported as a mismatch. The bucket default retention should
|
||||
match the chosen mode and retention days, although the exporter always sends explicit values.
|
||||
The settings change is audited and chained. **COMPLIANCE cannot be undone**: protected
|
||||
versions cannot be deleted or have retention shortened until expiry, including by administrators.
|
||||
|
||||
## App home
|
||||
|
||||
Open `/app/worm_audit` (or `/app/admin/worm_audit` with `parentAppName="admin"`).
|
||||
The full-width `wormAuditHome` HTML widget uses the standard `/qqq/v1/widget` API.
|
||||
Hosts can add `wormAuditHealthCard` to their own dashboard. Both widgets and the app
|
||||
require read permission on at least one WORM table, using the host's table permission rules.
|
||||
To embed WORM in an existing host app, set `withIncludeApp(false)` and configure
|
||||
`appName`, `appLabel` and optional `parentAppName` for that host app. The host owns
|
||||
its navigation, home widget placement and app help; the qbit does not create or replace
|
||||
the app or fill its help. Both widgets and all table help, permissions, guards,
|
||||
processes and schedules remain registered. The health card links to the configured
|
||||
host app. Standalone app creation remains enabled by default.
|
||||
The aggregate home shows only identifiers, counts, times, shortened hashes and reasons;
|
||||
it never renders audit messages or field values. Record links retain the target table's permissions.
|
||||
|
||||
- Chain status: live link count, latest link time, settled unchained backlog and settling
|
||||
count at the configured horizon, latest verification and first open chain break.
|
||||
A subsequent full scheduled CHAIN PASS clears a reported break; a selected USER PASS does not.
|
||||
- Export status: current lock mode and retention days, latest sealed range and its recorded retention, link/time lag, coverage holes
|
||||
between the purge anchor and sealed head, and bucket Object Lock capability. The unsealed
|
||||
trailing suffix is lag, not a gap. Provider failures show UNKNOWN with a value-free reason.
|
||||
- Retention: configured years, purge anchor, oldest live audit and its next eligible purge date.
|
||||
The date indicates retention eligibility; the database purge function still checks the sealed prefix.
|
||||
- Recent purges: the last ten requests, including refusals.
|
||||
- Setup warnings: missing signing/trusted keys, unsupported or unknown Object Lock,
|
||||
daily CHAIN verification never run or older than 26 hours. PostgreSQL also checks every
|
||||
protected table's row/truncate guards, runtime ownership and the purge function.
|
||||
|
||||
Sections and definition-list rows wrap at narrow widths and inherit the Next UI's light/dark colors.
|
||||
Empty states explicitly distinguish missing history from a passing check.
|
||||
|
||||
### Local demo
|
||||
|
||||
The test-scope server serves the API and bundled Next UI with a memory backend:
|
||||
|
||||
```powershell
|
||||
./mvnw.cmd -o test-compile exec:java "-Dexec.classpathScope=test" "-Dexec.mainClass=com.kingsrook.qbits.wormaudit.demo.WormAuditDemoServer" "-Dexec.args=8099"
|
||||
```
|
||||
|
||||
On macOS/Linux use `./mvnw` with the same arguments. Open <http://localhost:8099/app/worm_audit>.
|
||||
No Docker or credentials are required. This anonymous demo uses synthetic `patient_note`
|
||||
records audited at FIELD level through the `PhiCanaryTest` HMAC stand-in. It seeds two weeks
|
||||
of actual chainer ticks, signed checkpoints, daily verifier results and immutable memory exports.
|
||||
One temporary detail tamper produces a genuine FAIL before restoration and later PASSes.
|
||||
The PURGED/REFUSED receipts and memory-only prefix removal are explicitly marked demo data;
|
||||
PostgreSQL enforcement and real provider durability are not simulated production guarantees.
|
||||
|
||||
The demo includes a small unsealed suffix and unchained backlog, with no invented export gap.
|
||||
The final settling count naturally ages into the backlog while the demo is open.
|
||||
|
||||
## Adopting in another app
|
||||
|
||||
Add `com.kingsrook.qbits:qbit-worm-audit:0.1.0-SNAPSHOT`. Define the host's core audit tables
|
||||
@@ -95,10 +215,11 @@ String migration = WormAuditDdl.postgres(config);
|
||||
`sealed_at`, `finished_at`, and `requested_at` respectively (these tables have no `modifyDate`).
|
||||
|
||||
The RC.6 assistant deny API is registered reflectively when present; absence in RC.5 is a no-op.
|
||||
RC.5's alert-type metadata validator disallows underscores, so the qbit sends the specified
|
||||
`worm_audit.chain.break` signal without declaring incompatible alert-type metadata. `QAlerts`
|
||||
still dispatches it to the sink (or its logging fallback). Every scheduled verification sends
|
||||
`worm_audit.verify.daily`; FAIL raises with the first break and verification record reference;
|
||||
The qbit declares `wormaudit.chain.break` (CRITICAL), `wormaudit.export.failed` (ERROR),
|
||||
and the daily verification/export heartbeat types through core alert-type metadata.
|
||||
Daily heartbeat monitors expect a 24-hour interval with 2 hours of grace (overdue at 26 hours).
|
||||
`QAlerts` dispatches to the installed sink or its logging fallback. Every scheduled verification sends
|
||||
`wormaudit.verify.daily`; FAIL raises with the first break and verification record reference;
|
||||
PASS resolves dedup key `chain`.
|
||||
|
||||
## Chain and verification format
|
||||
@@ -119,7 +240,7 @@ PASS resolves dedup key `chain`.
|
||||
an odd node is promoted unchanged. Ed25519 signs JCS of the seven specified range/trust fields.
|
||||
- SQL verification uses a repeatable-read snapshot. Full runs check settled header/detail
|
||||
completeness; selected or filtered runs check their links, predecessor boundaries and
|
||||
intersecting checkpoints. Full verification currently materializes link metadata in memory.
|
||||
intersecting checkpoints.
|
||||
|
||||
## Retention operations
|
||||
|
||||
@@ -143,10 +264,73 @@ rights. The qbit maps those errors to a value-free user-facing message.
|
||||
|
||||
Verification starts from the latest successful purge anchor. Checkpoint signatures still verify
|
||||
after purge, but Merkle recomputation of a purged/partly purged checkpoint requires its sealed
|
||||
export and is reported as a warning. Sealed exports are phase (b); preserve external evidence
|
||||
before exercising retention removal in production. Superusers/owners can disable triggers;
|
||||
export and is reported as a warning. Seal and verify exports before exercising retention
|
||||
removal in production. Superusers/owners can disable triggers;
|
||||
protect signing keys and keep public-key pins outside the database.
|
||||
|
||||
## Sealed exports
|
||||
|
||||
Install qbit-object-storage and configure `exportBucketAlias` (default `worm_audit`) as a
|
||||
`BucketRole.BACKUP` bucket. **Object Lock must be enabled when the bucket is created**.
|
||||
The exporter requires the bucket's `OBJECT_LOCK` capability to be `SUPPORTED`; run the
|
||||
object-storage read-only capability probe during setup. Unknown, partial and unsupported
|
||||
capabilities all refuse sealing. Every object receives the effective `export_lock_mode` and retention until sealing time plus
|
||||
`export_retention_days` (GOVERNANCE and 30 days by default), checked by HEAD after PUT. The
|
||||
export row records both values. Verification checks that historical mode and retain-until,
|
||||
regardless of subsequent settings changes. Missing GOVERNANCE objects are reported as breaks
|
||||
with a possible administrator bypass-removal explanation.
|
||||
|
||||
The runtime IAM policy needs `s3:PutObject`, `s3:GetObject`, `s3:ListBucket`,
|
||||
`s3:GetObjectRetention`, `s3:PutObjectRetention`, and
|
||||
`s3:GetBucketObjectLockConfiguration`. Grant no delete or governance-bypass access.
|
||||
The storage bucket's upload size limit must accommodate the largest daily JSONL file.
|
||||
Retention protects object versions; protect the storage volumes and maintain off-site copies too.
|
||||
|
||||
At 02:00 UTC, or through **Export now**, the exporter takes the chainer's writer lock and
|
||||
seals links after the previous export through the latest checkpoint. The first export starts
|
||||
after the purge anchor, or at genesis. No new checkpoint produces an explicit "Nothing to
|
||||
seal" Result. If retention has removed an unexported gap, or a range cannot be proven by
|
||||
complete signed checkpoints, sealing refuses rather than recording incomplete evidence.
|
||||
Other database backends require a single writer JVM, as in phase (a).
|
||||
|
||||
Keys use `<exportPrefix>/<yyyy>/<MM>/<dd>/<from>-<to>/` with a UTC date:
|
||||
|
||||
- `links.jsonl`: JCS lines containing `link` identifiers/hashes and the embedded canonical
|
||||
`content` document exactly as hashed.
|
||||
- `checkpoints.jsonl`: JCS checkpoint envelopes including signatures and public keys.
|
||||
- `manifest.json`: JCS `worm-audit-export/1` with `fromLinkId`, `toLinkId`, `linkCount`,
|
||||
`checkpointCount`, `files` (filename to SHA-256), `headHash`, `prevManifestSha256`,
|
||||
`qbitVersion`, and `createdAt`.
|
||||
|
||||
Files are streamed through temporary disk storage with 1,000-link pages and batched content
|
||||
fetches, then removed. Only ids, hashes and object metadata enter the export catalog.
|
||||
The `object_versions` text column pins each upload's returned version id in a JCS JSON
|
||||
object keyed by `checkpoints.jsonl`, `links.jsonl`, and `manifest.json`. Sealing refuses
|
||||
missing version ids or a different version in the retention HEAD response, leaving no
|
||||
catalog row and raising the export failure alert.
|
||||
Partial upload failures leave no catalog row; retained orphan objects may remain. Retrying
|
||||
can create new locked versions under the same keys. Failures raise `wormaudit.export.failed`
|
||||
with dedup key `export`; success resolves it. Every scheduled run emits
|
||||
`wormaudit.export.daily`.
|
||||
|
||||
**Verify sealed export** accepts one or several selected records. It downloads and checks
|
||||
the object hashes, retention, manifest chain, link hashes, pinned checkpoint signatures and
|
||||
Merkle roots, then compares the sealed content and link metadata against a live database
|
||||
snapshot. Differences name exact link/audit ids without printing audit values. Each export
|
||||
gets its own `SEALED_EXPORT` verification row. Purged prefixes verify from the sealed copy
|
||||
alone and receive an INFO line. Verification does not write restored audit rows.
|
||||
|
||||
Verification compares every current HEAD version to the catalog and reports an ERROR for
|
||||
any overwrite, even when the replacement bytes are identical. It then reads the recorded
|
||||
sealed versions, so retained evidence remains verifiable after an overwrite; the overall
|
||||
verification still fails because the overwrite is a break. Missing catalog versions fail
|
||||
closed. RC.2's `StorageBucket.get` has no version parameter, so the adapter uses the public
|
||||
`ObjectStorageRuntime.connection(...).client()` accessor for an explicit S3 version GET,
|
||||
reusing its connection, key normalization and rate limiter and checking the response version.
|
||||
No object-storage change is required; this adapter depends on RC.2's public internal runtime
|
||||
accessors. The storage credentials need `s3:GetObjectVersion` in addition to the existing
|
||||
upload, HEAD and listing permissions.
|
||||
|
||||
## Build and acceptance
|
||||
|
||||
```powershell
|
||||
@@ -170,7 +354,21 @@ $env:CI = 'true'
|
||||
./mvnw.cmd -o verify
|
||||
```
|
||||
|
||||
POSIX equivalents: `CI=true ./mvnw -o -Dtest=PostgresAcceptanceTest test` and
|
||||
MinIO export acceptance uses the same `RequiresDockerCondition`. It checks GOVERNANCE
|
||||
with 30-day retention and a COMPLIANCE export, denied version deletion (without bypass for GOVERNANCE), overwrite version preservation, export verification,
|
||||
exact live-tamper findings through the record action, and refusal of a bucket without lock:
|
||||
|
||||
```powershell
|
||||
$env:CI = 'true'
|
||||
./mvnw.cmd -o '-Dtest=MinioExportAcceptanceTest' test
|
||||
```
|
||||
|
||||
The default MinIO image is pinned to the same Chainguard digest as object-storage's tests.
|
||||
Override it with `-Dqqq.test.minioImage=<image>` when necessary. Images must already be
|
||||
available for a completely network-free Docker run.
|
||||
|
||||
POSIX equivalents: `CI=true ./mvnw -o -Dtest=MinioExportAcceptanceTest test`,
|
||||
`CI=true ./mvnw -o -Dtest=PostgresAcceptanceTest test` and
|
||||
`CI=true ./mvnw -o verify`. No commits, publishing or deployment are part of these commands.
|
||||
|
||||
## License
|
||||
@@ -194,6 +392,9 @@ that window. Initial header details are always chained, including overflow
|
||||
chunks. Checkpoints contain at most `checkpointEveryLinks` links; multiple full
|
||||
ranges can be signed per tick. Verification uses 1,000-link keyset pages.
|
||||
|
||||
Full verification holds at most one 1,000-link page and one checkpoint range of hashes.
|
||||
SQL completeness uses bounded offender queries; other persistent backends use repeated bounded
|
||||
scans. Only the memory backend uses a full coverage index.
|
||||
Full verification retains at most 1,000 exact break diagnostics and 100 warnings,
|
||||
while persisting the total break count. SQL completeness counts all offenders
|
||||
and reports the first 100 header ids and first 100 detail ids. This keeps a badly
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
# Upgrading qbit-worm-audit
|
||||
|
||||
## Detail coverage table (0.1.0-RC.4)
|
||||
|
||||
Apply changeset `worm-audit:13` from the rendered DDL (or your copy of it) before or with the new jar. It takes the chain advisory lock, creates `worm_audit_chain_detail` with its append-only guards and grants, backfills it from the existing chain, and replaces `worm_audit_reject()`, `worm_audit_detail_coverage()` and `worm_audit_purge()`. The jar requires the table: its chain writer reads it every tick. The GIN index `worm_audit_chain_detail_coverage` is no longer used but is kept so an RC.3 writer still running during a rolling deploy keeps an indexed probe; a later changeset drops it. Memory and H2 hosts need no migration beyond creating the new table (the writer fills it).
|
||||
|
||||
## Table help bundle (unreleased)
|
||||
|
||||
Nothing to do. The qbit applies `help/qbit-worm-audit.yaml` at the end of `produce()`, filling only help slots the host has not set. No schema or seed changes: the settings row is still seeded by the database script (changeset `worm-audit:10`) and nothing else is seeded.
|
||||
|
||||
`WormAuditSettings.seed(config)` is now `@Deprecated` for PostgreSQL hosts (the script seeds row 1). It still works and remains the way for memory and H2 hosts to create row 1. The settings reader falls back to the configured values while the row is absent.
|
||||
@@ -104,11 +104,24 @@ runtime role owns the audit tables (checked through `pg_tables.tableowner = curr
|
||||
The qbit ships `src/main/resources/com/kingsrook/qbits/wormaudit/ddl/postgres/worm-audit.sql`,
|
||||
a Liquibase formatted-SQL template with placeholders for the four core audit table names, the
|
||||
app role and the retention interval, plus a renderer (`WormAuditDdl.postgres(config)`) and the
|
||||
rendered default (`docs/ddl/worm-audit-postgres.sql`, snake_case names). Hosts copy the rendered
|
||||
rendered default (`docs/ddl/worm-audit-postgres.sql`, snake_case names). Core audit physical
|
||||
columns must use snake_case (for example `audit_table_id`, `audit_id`, `new_value`); hosts map
|
||||
QQQ field names to those columns. Hosts copy the rendered
|
||||
file into their changelog (kof22: `changesets/features/YYYY-MM-DD-worm-audit.sql`). It creates the
|
||||
qbit tables, the functions, the triggers, the purger role (`CREATE ROLE ... NOLOGIN` when it is
|
||||
missing) and the grants. It is idempotent (`CREATE OR REPLACE`, `DROP TRIGGER IF EXISTS`).
|
||||
|
||||
Foreign keys (`worm-audit:11-12`) are `ON DELETE RESTRICT` only, so nothing cascades into or out
|
||||
of evidence: `worm_audit_chain.audit_id` -> the core audit header (the purge deletes details, then
|
||||
links, then headers, in that order), `worm_audit_export.from/to_checkpoint_id` -> checkpoints and
|
||||
`worm_audit_verification.export_id` -> exports (neither is ever deleted). References to chain links
|
||||
(checkpoint, export and verification ranges, `first_break_link_id`, the purge anchor) are not
|
||||
foreign keys: the purge deletes the sealed prefix those rows outlive. On upgrade, rows whose parent
|
||||
is already missing are evidence and are never changed: they are copied to `qqq_fk_quarantine`
|
||||
(action `KEPT`) and that constraint stays `NOT VALID` (new rows are still checked). With the
|
||||
foreign key, a plain `TRUNCATE` of `audit` (or of checkpoints or exports) is refused by PostgreSQL
|
||||
before the guard trigger; `TRUNCATE ... CASCADE` reaches the guards, which refuse it.
|
||||
|
||||
### 4.5 Threat model summary
|
||||
|
||||
| Actor | Can modify audit rows? | Detected by |
|
||||
@@ -179,7 +192,13 @@ chain would need the canonical form in SQL. The design instead uses **one chaine
|
||||
`kind = LATE_DETAIL`, covering exactly those detail ids. Verification flags any detail not
|
||||
covered by exactly one link.
|
||||
- Unique constraints back the lock: `UNIQUE (audit_id) WHERE kind = 'AUDIT'`, and the detail
|
||||
coverage.
|
||||
coverage: `worm_audit_chain_detail (detail_id PRIMARY KEY, chain_id)`, one append-only row per
|
||||
covered detail, written by the chain insert trigger in the same statement as the link. The
|
||||
writer finds uncovered details with a primary-key probe of it; verification still counts
|
||||
coverage from the links' own `detail_ids` and breaks if the table disagrees.
|
||||
- Each tick reads only rows above per-JVM sweep floors: the highest audit and detail ids seen at
|
||||
least (settle horizon + 5 minutes) before the last complete sweep. After a restart and once an
|
||||
hour a tick ignores the floors, so a row whose transaction commits even later is still chained.
|
||||
|
||||
Why not per-partition chains: they make each partition's chain shorter but multiply heads,
|
||||
checkpoints and verification state, and they still need a writer per partition. One chainer
|
||||
@@ -217,15 +236,19 @@ standard streamed ETL:
|
||||
does not verify".
|
||||
|
||||
**Daily verification**: a scheduled job runs the same verifier at 03:00. A failure raises
|
||||
`QAlerts.raise(new AlertSignal("worm_audit.chain.break").withDedupKey("chain")...)` with the first
|
||||
`QAlerts.raise(new AlertSignal("wormaudit.chain.break").withDedupKey("chain")...)` with the first
|
||||
break in the title and the verification record as the record ref; a later pass resolves it. Every
|
||||
run sends `QAlerts.heartbeat("worm_audit.verify.daily")`. `QAlerts` is the core producer SPI
|
||||
run sends `QAlerts.heartbeat("wormaudit.verify.daily")`. `QAlerts` is the core producer SPI
|
||||
(QQQ 4.1.0-RC.5), so the qbit needs no dependency on qbit-alerts. With no alerts implementation
|
||||
installed the alerts go to the log.
|
||||
|
||||
## 6. Sealed exports
|
||||
|
||||
### 6.1 Export job
|
||||
|
||||
**Default superseded by Owner decisions (2026-10-14) below:** lock mode and export retention
|
||||
now come from Settings, defaulting to GOVERNANCE and 30 days. The original decision text is
|
||||
retained for history.
|
||||
Daily at 02:00 (configurable), and by the table action "Export now" on `worm_audit_export`:
|
||||
1. Range: links after the last export's `to_link_id`, up to the latest checkpoint. Exports are
|
||||
contiguous by construction. The app home flags a gap (a range not covered) and lag.
|
||||
@@ -236,14 +259,17 @@ Daily at 02:00 (configurable), and by the table action "Export now" on `worm_aud
|
||||
the previous export's manifest SHA-256 (the exports form their own chain), qbit version.
|
||||
3. Each object is put through qbit-object-storage (`SystemStorage`, bucket alias from config,
|
||||
default `worm_audit`) with `LockMode.COMPLIANCE` and `retainUntil = now + retentionYears`. The
|
||||
export refuses to run, and alerts `worm_audit.export.failed`, when the bucket does not report
|
||||
export refuses to run, and alerts `wormaudit.export.failed`, when the bucket does not report
|
||||
the `OBJECT_LOCK` capability or the put does not echo the lock.
|
||||
4. On success it inserts one `worm_audit_export` row (insert-only; a failed export leaves no row,
|
||||
and its alert carries the reason).
|
||||
and its alert carries the reason). Every scheduled run sends `wormaudit.export.daily`.
|
||||
|
||||
### 6.2 Restore and verify a sealed export
|
||||
"Verify sealed export", a record action on `worm_audit_export` (standard ETL), downloads the
|
||||
objects. It checks the manifest hashes, the lock mode and retain-until of each object, the link
|
||||
exact object versions pinned in the catalog's `object_versions` map. A new version under an
|
||||
existing key is an overwrite even when bytes and retention match: verification checks the latest
|
||||
version against the pin and rejects replacements, then reads and checks the pinned version.
|
||||
It checks the manifest hashes, the lock mode and retain-until of each object, the link
|
||||
hashes recomputed from the exported content, the checkpoint signatures, and the export's link to
|
||||
the previous manifest. Then it compares each exported link and content document with the live
|
||||
chain and the live audit rows. Result lines name each difference ("link 88,412: live content
|
||||
@@ -265,7 +291,7 @@ events integrations care about); chain links are high-volume and publish none.
|
||||
- **`worm_audit_checkpoint`** [Checkpoint]: `id`, `from_link_id`, `to_link_id`, `link_count`,
|
||||
`head_hash`, `merkle_root`, `algorithm`, `key_id`, `public_key`, `signature`, `created_at`.
|
||||
- **`worm_audit_export`** [Sealed Export]: `id`, `from_link_id`, `to_link_id`,
|
||||
`from_checkpoint_id`, `to_checkpoint_id`, `bucket_alias`, `object_prefix`, `manifest_key`,
|
||||
`from_checkpoint_id`, `to_checkpoint_id`, `bucket_alias`, `object_prefix`, `object_versions`, `manifest_key`,
|
||||
`manifest_sha256`, `prev_manifest_sha256`, `object_count`, `byte_count`, `lock_mode`,
|
||||
`retain_until`, `sealed_at`.
|
||||
- **`worm_audit_verification`** [Verification]: `id`, `kind` (CHAIN, SEALED_EXPORT), `trigger`
|
||||
@@ -341,7 +367,7 @@ registers as a `MiddlewareSpecExtension` under `/qqq/v1/qbit/worm-audit/...`
|
||||
qbit's tests until the core hook ships), the audit header, the details (old, new, message),
|
||||
the canonical content and the export JSONL contain neither the canary nor any substring of it
|
||||
longer than 4 characters.
|
||||
6. **Alerts**: a break raises `worm_audit.chain.break` through `QAlerts` (captured by a test
|
||||
6. **Alerts**: a break raises `wormaudit.chain.break` through `QAlerts` (captured by a test
|
||||
sink); a later pass resolves it.
|
||||
7. Coverage gates: 70% instructions, 90% classes.
|
||||
|
||||
@@ -367,7 +393,7 @@ registers as a `MiddlewareSpecExtension` under `/qqq/v1/qbit/worm-audit/...`
|
||||
- **Bucket:** `kof22-worm-audit` on the kof22 MinIO, created by `minio-bucket-init` with
|
||||
`mc mb --with-lock --ignore-existing` and `mc retention set --default COMPLIANCE 6y`. Object
|
||||
Lock can only be enabled at bucket creation.
|
||||
- **Policy:** `kof22-app` gets `s3:PutObject`, `s3:GetObject`, `s3:ListBucket`,
|
||||
- **Policy:** `kof22-app` gets `s3:PutObject`, `s3:GetObject`, `s3:GetObjectVersion`, `s3:ListBucket`,
|
||||
`s3:GetObjectRetention`, `s3:PutObjectRetention` (needed to set the lock on each put),
|
||||
`s3:GetBucketObjectLockConfiguration` on that bucket. No delete, and no
|
||||
`s3:BypassGovernanceRetention` (irrelevant under COMPLIANCE anyway).
|
||||
@@ -381,3 +407,33 @@ registers as a `MiddlewareSpecExtension` under `/qqq/v1/qbit/worm-audit/...`
|
||||
|
||||
These are owner-run (Praetor and cluster-gitops are applied by the owner). The qbit only
|
||||
documents them.
|
||||
|
||||
## Owner decisions (2026-10-14)
|
||||
|
||||
The 2026-10-12 decisions above remain the historical design. These decisions supersede the
|
||||
COMPLIANCE/six-year export default in §6.1 and the corresponding default assumptions in
|
||||
§§2, 4.5, 9, 11, 12 and the §14 infrastructure proposal.
|
||||
|
||||
- Settings live in the application as the standard QQQ `worm_audit_settings` table, record
|
||||
ID 1, labelled WORM Audit Settings and navigated as Settings after Purges. It allows edits,
|
||||
has no append-only triggers, and uses FIELD-level core audit so every change is chained.
|
||||
Reading needs `worm_audit_settings.read`; editing additionally needs
|
||||
`worm_audit_settings.edit` and `worm_audit.settings`. New IDs and deletion are refused.
|
||||
- Defaults are seeded insert-only in new changeset `worm-audit:10`; existing changesets are
|
||||
unchanged. An administrator's edits survive subsequent seeding. Config supplies defaults
|
||||
when a row/field/table is absent. Runtime reads cache at most 30 seconds, scoped to the
|
||||
QInstance. Chain interval and UTC Quartz export/verify schedules take effect after restart.
|
||||
- kof22 starts with **GOVERNANCE, 30 days** for sealed exports. An administrator with
|
||||
`s3:BypassGovernanceRetention` can remove those copies. This is deliberately reversible;
|
||||
it does not provide COMPLIANCE's irreversible retention. Runtime credentials still receive
|
||||
no delete or bypass permission. Bucket defaults should match the selected settings.
|
||||
- Changing Settings to COMPLIANCE affects new exports. Existing GOVERNANCE versions retain
|
||||
their original mode unless an administrator extends/upgrades them. Verification checks
|
||||
the catalog's recorded mode and date, so external mode changes are reported as mismatches.
|
||||
COMPLIANCE cannot be undone. The settings change itself is audited and chained.
|
||||
- Audit retention defaults to six years independently of export retention. The home warns
|
||||
about a shorter export horizon and a mismatch with the purge function's compiled interval.
|
||||
Updating the setting does not modify the purge function; that still requires a migration.
|
||||
|
||||
The README Settings table specifies all fields, defaults, bounds and timing. Signing keys,
|
||||
public-key pins, bucket alias, prefix and backend remain host configuration, never settings.
|
||||
@@ -37,6 +37,7 @@ CREATE TABLE IF NOT EXISTS public.worm_audit_export (
|
||||
to_checkpoint_id bigint,
|
||||
bucket_alias text,
|
||||
object_prefix text,
|
||||
object_versions text,
|
||||
manifest_key text,
|
||||
manifest_sha256 text,
|
||||
prev_manifest_sha256 text,
|
||||
@@ -323,3 +324,233 @@ CREATE INDEX IF NOT EXISTS worm_audit_chain_detail_coverage ON public.worm_audit
|
||||
DROP TRIGGER IF EXISTS worm_audit_detail_coverage_guard ON public.worm_audit_chain;
|
||||
CREATE TRIGGER worm_audit_detail_coverage_guard BEFORE INSERT ON public.worm_audit_chain
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_detail_coverage();
|
||||
|
||||
-- changeset worm-audit:10
|
||||
-- Editable singleton: deliberately excluded from append-only triggers.
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_settings (
|
||||
id integer PRIMARY KEY CHECK (id = 1),
|
||||
audit_retention_years integer NOT NULL DEFAULT 6 CHECK (audit_retention_years BETWEEN 1 AND 100),
|
||||
export_lock_mode text NOT NULL DEFAULT 'GOVERNANCE' CHECK (export_lock_mode IN ('GOVERNANCE', 'COMPLIANCE')),
|
||||
export_retention_days integer NOT NULL DEFAULT 30 CHECK (export_retention_days BETWEEN 1 AND 36500),
|
||||
checkpoint_every_links integer NOT NULL DEFAULT 10000 CHECK (checkpoint_every_links > 0),
|
||||
checkpoint_every_minutes integer NOT NULL DEFAULT 60 CHECK (checkpoint_every_minutes > 0),
|
||||
chain_interval_seconds integer NOT NULL DEFAULT 30 CHECK (chain_interval_seconds > 0),
|
||||
settle_horizon_seconds integer NOT NULL DEFAULT 60 CHECK (settle_horizon_seconds >= 0),
|
||||
late_detail_lookback_hours integer NOT NULL DEFAULT 24 CHECK (late_detail_lookback_hours > 0),
|
||||
export_cron text NOT NULL DEFAULT '0 0 2 * * ?',
|
||||
verify_cron text NOT NULL DEFAULT '0 0 3 * * ?',
|
||||
notes text,
|
||||
modify_date timestamptz NOT NULL DEFAULT now(),
|
||||
modified_by text NOT NULL DEFAULT current_user
|
||||
);
|
||||
REVOKE DELETE, TRUNCATE ON public.worm_audit_settings FROM "app";
|
||||
GRANT SELECT, INSERT, UPDATE ON public.worm_audit_settings TO "app";
|
||||
INSERT INTO public.worm_audit_settings
|
||||
(id, audit_retention_years, export_lock_mode, export_retention_days, checkpoint_every_links,
|
||||
checkpoint_every_minutes, chain_interval_seconds, settle_horizon_seconds, late_detail_lookback_hours,
|
||||
export_cron, verify_cron)
|
||||
SELECT 1, 6, 'GOVERNANCE', 30, 10000, 60, 30, 60, 24, '0 0 2 * * ?', '0 0 3 * * ?'
|
||||
WHERE NOT EXISTS (SELECT 1 FROM public.worm_audit_settings WHERE id = 1);
|
||||
|
||||
-- changeset worm-audit:11
|
||||
-- comment: Index the foreign key columns added by worm-audit:12 (worm_audit_chain.audit_id already leads worm_audit_chain_audit_all).
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_export_from_checkpoint ON public.worm_audit_export(from_checkpoint_id);
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_export_to_checkpoint ON public.worm_audit_export(to_checkpoint_id);
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_verification_export ON public.worm_audit_verification(export_id);
|
||||
|
||||
-- changeset worm-audit:12 splitStatements:false
|
||||
-- comment: Foreign keys, ON DELETE RESTRICT only: nothing cascades into or out of audit evidence. A chain link needs its audit header (the purge deletes links before headers); a sealed export needs its signed checkpoints and an export verification its export (neither is ever deleted). Links referenced by checkpoints, exports, verifications and purges are not foreign keys: the purge deletes the sealed prefix those rows outlive. Rows whose parent is already missing are evidence and are never changed: they are copied to qqq_fk_quarantine (action KEPT) and the constraint is left NOT VALID (enforced for new rows only).
|
||||
DO $$
|
||||
DECLARE
|
||||
fk record;
|
||||
orphans bigint;
|
||||
BEGIN
|
||||
CREATE TABLE IF NOT EXISTS qqq_fk_quarantine (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, table_name VARCHAR(100) NOT NULL, column_name VARCHAR(100) NOT NULL, row_id VARCHAR(100), missing_parent_id VARCHAR(100), action VARCHAR(20) NOT NULL, row_data TEXT, changeset VARCHAR(200), quarantined_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP);
|
||||
FOR fk IN SELECT * FROM (VALUES
|
||||
('fk_worm_audit_chain_audit_id', 'worm_audit_chain', 'audit_id', '"audit"'),
|
||||
('fk_worm_audit_export_from_checkpoint_id', 'worm_audit_export', 'from_checkpoint_id', 'public.worm_audit_checkpoint'),
|
||||
('fk_worm_audit_export_to_checkpoint_id', 'worm_audit_export', 'to_checkpoint_id', 'public.worm_audit_checkpoint'),
|
||||
('fk_worm_audit_verification_export_id', 'worm_audit_verification', 'export_id', 'public.worm_audit_export')
|
||||
) AS t(name, child, col, parent) LOOP
|
||||
IF NOT EXISTS (SELECT FROM pg_constraint WHERE conname = fk.name) THEN
|
||||
EXECUTE format('INSERT INTO qqq_fk_quarantine (table_name, column_name, row_id, missing_parent_id, action, row_data, changeset)'
|
||||
|| ' SELECT %L, %L, c.id::text, c.%I::text, ''KEPT'', row_to_json(c)::text, ''worm-audit:12'' FROM public.%I c'
|
||||
|| ' WHERE c.%I IS NOT NULL AND NOT EXISTS (SELECT FROM %s p WHERE p.id = c.%I)', fk.child, fk.col, fk.col, fk.child, fk.col, fk.parent, fk.col);
|
||||
GET DIAGNOSTICS orphans = ROW_COUNT;
|
||||
EXECUTE format('ALTER TABLE public.%I ADD CONSTRAINT %I FOREIGN KEY (%I) REFERENCES %s (id) ON DELETE RESTRICT NOT VALID', fk.child, fk.name, fk.col, fk.parent);
|
||||
IF orphans = 0 THEN
|
||||
EXECUTE format('ALTER TABLE public.%I VALIDATE CONSTRAINT %I', fk.child, fk.name);
|
||||
END IF;
|
||||
END IF;
|
||||
END LOOP;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:13 splitStatements:false
|
||||
-- comment: Normalized detail coverage. One insert-only row per audit_detail id covered by a chain link, written by the coverage trigger in the same statement as the link, so the chain writer probes a primary key instead of the GIN array index. Append-only like the chain, with no foreign keys (the owner cannot UPDATE WORM tables, so FK checks would be denied). The advisory lock keeps a running chainer from appending between the backfill and the new trigger body.
|
||||
DO $$
|
||||
BEGIN
|
||||
PERFORM pg_advisory_xact_lock(86753090421);
|
||||
END;
|
||||
$$;
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_chain_detail (
|
||||
detail_id bigint PRIMARY KEY,
|
||||
chain_id bigint NOT NULL,
|
||||
CHECK (detail_id >= 0),
|
||||
CHECK (chain_id > 0)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_chain_detail_chain ON public.worm_audit_chain_detail(chain_id);
|
||||
-- The first link wins if a tampered chain covers a detail twice; the verifier still reports the duplicate from the chain itself.
|
||||
INSERT INTO public.worm_audit_chain_detail(detail_id, chain_id)
|
||||
SELECT x.id::bigint, c.id FROM public.worm_audit_chain c CROSS JOIN LATERAL unnest(string_to_array(c.detail_ids, ',')) AS x(id)
|
||||
WHERE c.detail_ids <> '' ORDER BY c.id
|
||||
ON CONFLICT (detail_id) DO NOTHING;
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_reject() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp SET timezone = 'UTC' AS $fn$
|
||||
DECLARE
|
||||
old_time timestamptz;
|
||||
function_owner name;
|
||||
stack text;
|
||||
row_identity text := 'all';
|
||||
BEGIN
|
||||
IF TG_OP = 'DELETE' AND current_setting('worm_audit.purging', true) = 'on' THEN
|
||||
SELECT pg_get_userbyid(proowner) INTO function_owner FROM pg_proc
|
||||
WHERE oid = 'public.worm_audit_purge(timestamptz)'::regprocedure;
|
||||
GET DIAGNOSTICS stack = PG_CONTEXT;
|
||||
IF current_user = function_owner AND position('function worm_audit_purge(timestamp with time zone)' in stack) > 0 THEN
|
||||
IF TG_RELID = '"audit"'::regclass THEN
|
||||
old_time := OLD.timestamp;
|
||||
ELSIF TG_RELID = '"audit_detail"'::regclass THEN
|
||||
SELECT timestamp INTO old_time FROM "audit" WHERE id = OLD.audit_id;
|
||||
ELSIF TG_RELID = 'public.worm_audit_chain'::regclass THEN
|
||||
SELECT timestamp INTO old_time FROM "audit" WHERE id = OLD.audit_id;
|
||||
ELSIF TG_RELID = 'public.worm_audit_chain_detail'::regclass THEN
|
||||
SELECT a.timestamp INTO old_time FROM public.worm_audit_chain c JOIN "audit" a ON a.id = c.audit_id WHERE c.id = OLD.chain_id;
|
||||
END IF;
|
||||
IF old_time < now() - interval '6 years' THEN
|
||||
RETURN OLD;
|
||||
END IF;
|
||||
END IF;
|
||||
END IF;
|
||||
IF TG_LEVEL = 'ROW' THEN
|
||||
-- Record fields resolve per statement, so each table's key is read in its own branch.
|
||||
IF TG_RELID = 'public.worm_audit_chain_detail'::regclass THEN
|
||||
row_identity := OLD.detail_id::text;
|
||||
ELSE
|
||||
row_identity := OLD.id::text;
|
||||
END IF;
|
||||
END IF;
|
||||
RAISE EXCEPTION USING ERRCODE = 'P0001',
|
||||
MESSAGE = format('worm_audit: audit rows are write-once; %s of "%s" is not allowed (id=%s). Expired rows are removed only by worm_audit_purge().',
|
||||
TG_OP, TG_TABLE_NAME, row_identity),
|
||||
HINT = 'see qbit-worm-audit docs/DESIGN.md section 4';
|
||||
END;
|
||||
$fn$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_detail_coverage() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $fn$
|
||||
BEGIN
|
||||
-- Serialize even direct SQL appends, then record coverage in the same statement as the link.
|
||||
-- The primary key rejects a detail covered twice, within this link or by an earlier one. The
|
||||
-- previous EXISTS overlap check ran as a cached generic plan that chose a sequential scan of
|
||||
-- the whole chain for every inserted link.
|
||||
PERFORM pg_advisory_xact_lock(86753090421);
|
||||
IF NEW.detail_ids = '' THEN
|
||||
RETURN NEW;
|
||||
END IF;
|
||||
BEGIN
|
||||
INSERT INTO public.worm_audit_chain_detail(detail_id, chain_id)
|
||||
SELECT d, NEW.id FROM unnest(string_to_array(NEW.detail_ids, ',')::bigint[]) AS d;
|
||||
EXCEPTION WHEN unique_violation THEN
|
||||
RAISE EXCEPTION USING ERRCODE = '23505', MESSAGE = 'worm_audit: audit detail is already covered by the chain';
|
||||
END;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$fn$;
|
||||
|
||||
-- The array index worm_audit_chain_detail_coverage is no longer used by this release. It stays until
|
||||
-- a later migration drops it, so a previous-release writer still running during a rolling deploy keeps its indexed probe.
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_chain_detail;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_chain_detail
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_chain_detail;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_chain_detail
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_chain_detail FROM "app";
|
||||
GRANT SELECT, INSERT ON public.worm_audit_chain_detail TO "app";
|
||||
|
||||
-- Same function as worm-audit:5, except that coverage rows are locked and purged before their links.
|
||||
-- The migrating role may have revoked its own EXECUTE (a runtime role that owns the schema, as on CNPG);
|
||||
-- PostgreSQL then refuses CREATE OR REPLACE. Grant it back for this statement only, then restore.
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT has_function_privilege(current_user, 'public.worm_audit_purge(timestamptz)', 'EXECUTE') THEN
|
||||
EXECUTE format('GRANT EXECUTE ON FUNCTION public.worm_audit_purge(timestamptz) TO %I', current_user);
|
||||
PERFORM set_config('worm_audit.purge_regrant', 'on', true);
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_purge(p_cutoff timestamptz) RETURNS void
|
||||
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, public, pg_temp SET timezone = 'UTC' AS $fn$
|
||||
DECLARE
|
||||
boundary bigint;
|
||||
smaller bigint;
|
||||
anchor text;
|
||||
audits_count bigint;
|
||||
details_count bigint;
|
||||
links_count bigint;
|
||||
BEGIN
|
||||
IF p_cutoff IS NULL OR p_cutoff > now() - interval '6 years' THEN
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome, reason)
|
||||
VALUES(now(), session_user, p_cutoff, '6 years', 'REFUSED', 'Cutoff is within retention or null');
|
||||
RETURN;
|
||||
END IF;
|
||||
-- Same lock as the Java chainer; table locks also exclude concurrent audit inserts.
|
||||
PERFORM pg_advisory_xact_lock(86753090421);
|
||||
LOCK TABLE "audit", "audit_detail", public.worm_audit_chain, public.worm_audit_chain_detail IN SHARE ROW EXCLUSIVE MODE;
|
||||
SELECT coalesce(min(c.id) - 1, (SELECT max(id) FROM public.worm_audit_chain)) INTO boundary
|
||||
FROM public.worm_audit_chain c LEFT JOIN "audit" a ON a.id = c.audit_id
|
||||
WHERE a.id IS NULL OR a.timestamp IS NULL OR a.timestamp >= p_cutoff;
|
||||
-- Shrink to a prefix closed over every audit's links AND all its stored details.
|
||||
LOOP
|
||||
SELECT min(c.id) - 1 INTO smaller FROM public.worm_audit_chain c
|
||||
WHERE c.id <= boundary AND (
|
||||
EXISTS (SELECT FROM public.worm_audit_chain later WHERE later.audit_id = c.audit_id AND later.id > boundary)
|
||||
OR EXISTS (SELECT FROM "audit_detail" d WHERE d.audit_id = c.audit_id AND NOT EXISTS (
|
||||
SELECT FROM public.worm_audit_chain covered WHERE covered.audit_id = c.audit_id
|
||||
AND d.id::text = ANY(string_to_array(covered.detail_ids, ',')))));
|
||||
EXIT WHEN smaller IS NULL;
|
||||
boundary := smaller;
|
||||
END LOOP;
|
||||
SELECT link_hash INTO anchor FROM public.worm_audit_chain WHERE id = boundary;
|
||||
IF anchor IS NULL THEN
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome, reason)
|
||||
VALUES(now(), session_user, p_cutoff, '6 years', 'REFUSED', 'No complete expired chain prefix');
|
||||
RETURN;
|
||||
END IF;
|
||||
PERFORM set_config('worm_audit.purging', 'on', true);
|
||||
DELETE FROM "audit_detail" d WHERE EXISTS
|
||||
(SELECT FROM public.worm_audit_chain c WHERE c.audit_id = d.audit_id AND c.id <= boundary);
|
||||
GET DIAGNOSTICS details_count = ROW_COUNT;
|
||||
-- Coverage rows go before their links: the guard finds their age through the link.
|
||||
DELETE FROM public.worm_audit_chain_detail WHERE chain_id <= boundary;
|
||||
-- Keep ids locally because links must be deleted before their audit headers.
|
||||
WITH removed AS (DELETE FROM public.worm_audit_chain WHERE id <= boundary RETURNING audit_id),
|
||||
removed_audits AS (DELETE FROM "audit" WHERE id IN (SELECT audit_id FROM removed) RETURNING id)
|
||||
SELECT (SELECT count(*) FROM removed), (SELECT count(*) FROM removed_audits) INTO links_count, audits_count;
|
||||
PERFORM set_config('worm_audit.purging', 'off', true);
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome,
|
||||
audits_deleted, details_deleted, links_deleted, anchor_link_id, anchor_hash)
|
||||
VALUES(now(), session_user, p_cutoff, '6 years', 'PURGED', audits_count, details_count, links_count, boundary, anchor);
|
||||
END;
|
||||
$fn$;
|
||||
DO $$
|
||||
BEGIN
|
||||
IF current_setting('worm_audit.purge_regrant', true) = 'on' THEN
|
||||
EXECUTE format('REVOKE EXECUTE ON FUNCTION public.worm_audit_purge(timestamptz) FROM %I', current_user);
|
||||
PERFORM set_config('worm_audit.purge_regrant', 'off', true);
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
@@ -0,0 +1,80 @@
|
||||
# WORM Audit screenshots
|
||||
|
||||
## RC.2 settings captures (2026-10-14)
|
||||
Home (GOVERNANCE, 30-day exports) and the WORM Audit Settings record at 390 x 844 and 1440 x 1000,
|
||||
light and dark, captured with Playwright against `WormAuditDemoServer` on port 8099:
|
||||
`worm-audit-home-{390,1440}-{light,dark}.jpg` (replaced) and
|
||||
`worm-audit-settings-{390,1440}-{light,dark}.jpg` (new). Overflow check on all eight: no page
|
||||
horizontal overflow, no inner horizontal scrollers, no double scrollbars.
|
||||
|
||||
## Earlier captures
|
||||
|
||||
Refreshed after record-screen polish **2026-10-13 (America/Chicago)** from the rebuilt test-scope demo using
|
||||
packaged Next UI 1.0.0-RC.14 and connected Edge through the browser screenshot
|
||||
and viewport tools. All files are JPEG viewport captures, below 300 KB each.
|
||||
|
||||
```powershell
|
||||
./mvnw.cmd -o test-compile exec:java "-Dexec.classpathScope=test" "-Dexec.mainClass=com.kingsrook.qbits.wormaudit.demo.WormAuditDemoServer" "-Dexec.args=8105"
|
||||
```
|
||||
|
||||
Open <http://localhost:8105/app/worm_audit/>. Port 8105 isolates this rebuilt instance from earlier demos. The seed creates genuine verification
|
||||
results and exports over synthetic data. Records captured: FAIL verification #8,
|
||||
PASS verification #15, sealed export #15, and chain link #329.
|
||||
|
||||
Required matrix: home at **390, 1440, 1920 px**, and all four records at **390,
|
||||
1440 px**, each in **light and dark**. Heights are 844, 1000 and 1080 px respectively.
|
||||
The existing supplementary 390 px light purge-history capture was refreshed too.
|
||||
Theme changes used Preferences and were restored to light; the temporary viewport override was reset afterward.
|
||||
|
||||
Every image was visually inspected. At each capture, the live DOM was checked after responsive layout settled for
|
||||
page-level horizontal overflow (`document.documentElement.scrollWidth > innerWidth`)
|
||||
and all rendered inner horizontal scrollers (`scrollWidth > clientWidth + 1` with
|
||||
computed `overflow-x` of `auto` or `scroll`). The scan included content below the
|
||||
viewport, including loaded child lists. Mobile verification accordions were also explicitly expanded and checked after their child rows loaded. There were **zero** page overflow failures
|
||||
and **zero** inner horizontal scrollers. Each home/verification/export view had
|
||||
one vertical scroll container (main); chain records had none. No double scrollbars
|
||||
were found. Long hashes, object versions and failure diagnostics wrap on mobile.
|
||||
No CSS or DOM overrides were injected for the captures.
|
||||
|
||||
All five tables have explicit field labels and meaningful record-label formats.
|
||||
Verification uses full-width Result, Scope, Run, and Chain links sections. Every
|
||||
child-list widget and section declares 12 grid columns. Kind and outcome use
|
||||
readable possible values; child queries translate those values too. Hashes stay
|
||||
complete because RC.5 has no separate list-only field display format. Numeric
|
||||
chain-reference labels keep the export title concise.
|
||||
|
||||
Core RC.5 shares the table label with its sidebar tree node. Existing plural
|
||||
navigation labels are retained, with singular record titles and record-section
|
||||
labels. Separate singular table labels and plural sidebar labels require core
|
||||
support; no application-wide metadata customizer was replaced to work around it.
|
||||
|
||||
`./mvnw.cmd -o verify` passed: 79 tests, no failures or errors, and 9
|
||||
Docker-dependent skips. Checkstyle and JaCoCo coverage gates passed. Metadata
|
||||
tests cover every field label, record-label formats and fields, verification
|
||||
section coverage, and 12-column sections and child widgets.
|
||||
|
||||
| Screenshot | Viewport | Size | Page horizontal overflow | Inner horizontal scrollers | Double scrollbars |
|
||||
|---|---|---|---|---|---|
|
||||
| [worm-audit-chain-link-1440-dark](worm-audit-chain-link-1440-dark.jpg) | 1440 x 1000 | 41.1 KB | None | None | None |
|
||||
| [worm-audit-chain-link-1440-light](worm-audit-chain-link-1440-light.jpg) | 1440 x 1000 | 41.4 KB | None | None | None |
|
||||
| [worm-audit-chain-link-390-dark](worm-audit-chain-link-390-dark.jpg) | 390 x 844 | 21.8 KB | None | None | None |
|
||||
| [worm-audit-chain-link-390-light](worm-audit-chain-link-390-light.jpg) | 390 x 844 | 21.8 KB | None | None | None |
|
||||
| [worm-audit-home-1440-dark](worm-audit-home-1440-dark.jpg) | 1440 x 1000 | 73.6 KB | None | None | None |
|
||||
| [worm-audit-home-1440-light](worm-audit-home-1440-light.jpg) | 1440 x 1000 | 72.4 KB | None | None | None |
|
||||
| [worm-audit-home-1920-dark](worm-audit-home-1920-dark.jpg) | 1920 x 1080 | 84.7 KB | None | None | None |
|
||||
| [worm-audit-home-1920-light](worm-audit-home-1920-light.jpg) | 1920 x 1080 | 83.6 KB | None | None | None |
|
||||
| [worm-audit-home-390-dark](worm-audit-home-390-dark.jpg) | 390 x 844 | 30.0 KB | None | None | None |
|
||||
| [worm-audit-home-390-light](worm-audit-home-390-light.jpg) | 390 x 844 | 28.8 KB | None | None | None |
|
||||
| [worm-audit-home-390-purges-light](worm-audit-home-390-purges-light.jpg) | 390 x 844 | 29.5 KB | None | None | None |
|
||||
| [worm-audit-sealed-export-1440-dark](worm-audit-sealed-export-1440-dark.jpg) | 1440 x 1000 | 117.6 KB | None | None | None |
|
||||
| [worm-audit-sealed-export-1440-light](worm-audit-sealed-export-1440-light.jpg) | 1440 x 1000 | 118.4 KB | None | None | None |
|
||||
| [worm-audit-sealed-export-390-dark](worm-audit-sealed-export-390-dark.jpg) | 390 x 844 | 38.4 KB | None | None | None |
|
||||
| [worm-audit-sealed-export-390-light](worm-audit-sealed-export-390-light.jpg) | 390 x 844 | 37.3 KB | None | None | None |
|
||||
| [worm-audit-verification-fail-1440-dark](worm-audit-verification-fail-1440-dark.jpg) | 1440 x 1000 | 53.6 KB | None | None | None |
|
||||
| [worm-audit-verification-fail-1440-light](worm-audit-verification-fail-1440-light.jpg) | 1440 x 1000 | 54.1 KB | None | None | None |
|
||||
| [worm-audit-verification-fail-390-dark](worm-audit-verification-fail-390-dark.jpg) | 390 x 844 | 28.1 KB | None | None | None |
|
||||
| [worm-audit-verification-fail-390-light](worm-audit-verification-fail-390-light.jpg) | 390 x 844 | 27.8 KB | None | None | None |
|
||||
| [worm-audit-verification-pass-1440-dark](worm-audit-verification-pass-1440-dark.jpg) | 1440 x 1000 | 49.0 KB | None | None | None |
|
||||
| [worm-audit-verification-pass-1440-light](worm-audit-verification-pass-1440-light.jpg) | 1440 x 1000 | 49.7 KB | None | None | None |
|
||||
| [worm-audit-verification-pass-390-dark](worm-audit-verification-pass-390-dark.jpg) | 390 x 844 | 23.5 KB | None | None | None |
|
||||
| [worm-audit-verification-pass-390-light](worm-audit-verification-pass-390-light.jpg) | 390 x 844 | 23.4 KB | None | None | None |
|
||||
|
After Width: | Height: | Size: 41 KiB |
|
After Width: | Height: | Size: 41 KiB |
|
After Width: | Height: | Size: 22 KiB |
|
After Width: | Height: | Size: 22 KiB |
|
After Width: | Height: | Size: 84 KiB |
|
After Width: | Height: | Size: 82 KiB |
|
After Width: | Height: | Size: 85 KiB |
|
After Width: | Height: | Size: 84 KiB |
|
After Width: | Height: | Size: 32 KiB |
|
After Width: | Height: | Size: 31 KiB |
|
After Width: | Height: | Size: 30 KiB |
|
After Width: | Height: | Size: 118 KiB |
|
After Width: | Height: | Size: 118 KiB |
|
After Width: | Height: | Size: 38 KiB |
|
After Width: | Height: | Size: 37 KiB |
|
After Width: | Height: | Size: 74 KiB |
|
After Width: | Height: | Size: 75 KiB |
|
After Width: | Height: | Size: 33 KiB |
|
After Width: | Height: | Size: 33 KiB |
|
After Width: | Height: | Size: 54 KiB |
|
After Width: | Height: | Size: 54 KiB |
|
After Width: | Height: | Size: 28 KiB |
|
After Width: | Height: | Size: 28 KiB |
|
After Width: | Height: | Size: 49 KiB |
|
After Width: | Height: | Size: 50 KiB |
|
After Width: | Height: | Size: 24 KiB |
|
After Width: | Height: | Size: 23 KiB |
@@ -22,6 +22,7 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
|
||||
<properties>
|
||||
<qqq.version>4.1.0-RC.26</qqq.version> <!-- feeds qbit-build-parent's qqq-bom-pom import -->
|
||||
<revision>0.1.0-SNAPSHOT</revision>
|
||||
<maven.compiler.source>21</maven.compiler.source>
|
||||
<maven.compiler.target>21</maven.compiler.target>
|
||||
@@ -33,7 +34,7 @@
|
||||
<parent>
|
||||
<groupId>com.kingsrook</groupId>
|
||||
<artifactId>qbit-build-parent</artifactId>
|
||||
<version>2.0.0</version>
|
||||
<version>2.2.1</version>
|
||||
</parent>
|
||||
|
||||
<groupId>com.kingsrook.qbits</groupId>
|
||||
@@ -83,6 +84,11 @@
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
<artifactId>qqq-backend-module-rdbms</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qbits</groupId>
|
||||
<artifactId>qbit-object-storage</artifactId>
|
||||
<version>0.1.0-RC.10</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
<artifactId>qqq-middleware-javalin</artifactId>
|
||||
@@ -90,7 +96,7 @@
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
<artifactId>qqq-frontend-next</artifactId>
|
||||
<version>1.0.0-RC.14</version>
|
||||
<version>1.0.0-RC.38</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
@@ -226,7 +232,7 @@
|
||||
</activation>
|
||||
<properties>
|
||||
<!-- Latest QQQ RC on Maven Central (2026-10-13). -->
|
||||
<qqq.snapshot.version>4.1.0-RC.5</qqq.snapshot.version>
|
||||
<qqq.snapshot.version>4.1.0-RC.26</qqq.snapshot.version>
|
||||
</properties>
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
|
||||
@@ -25,6 +25,7 @@ import java.security.PrivateKey;
|
||||
import java.security.PublicKey;
|
||||
import java.util.Map;
|
||||
import java.util.function.Supplier;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
@@ -44,6 +45,15 @@ public class WormAuditQBitConfig implements QBitConfig
|
||||
{
|
||||
}
|
||||
|
||||
private String exportLockMode = "GOVERNANCE";
|
||||
private int exportRetentionDays = 30;
|
||||
private final WormAuditSettings settings = new WormAuditSettings(this);
|
||||
private boolean includeApp = true;
|
||||
private String appName = "worm_audit";
|
||||
private String appLabel = "WORM Audit";
|
||||
private String parentAppName;
|
||||
private String tablePathPrefix = "/app";
|
||||
private Supplier<com.kingsrook.qbits.wormaudit.export.SealedStore> sealedStoreSupplier;
|
||||
private String backendName = null;
|
||||
private int retentionYears = 6;
|
||||
private Supplier<SigningKey> signingKey = null;
|
||||
@@ -55,12 +65,187 @@ public class WormAuditQBitConfig implements QBitConfig
|
||||
private int checkpointEveryLinks = 10000;
|
||||
private int checkpointEveryMinutes = 60;
|
||||
private String verifyCron = "0 0 3 * * ?";
|
||||
private String exportBucketAlias = "worm_audit";
|
||||
private String exportPrefix = "worm-audit";
|
||||
private String exportCron = "0 0 2 * * ?";
|
||||
private String schedulerName = "worm_audit";
|
||||
private Map<String, String> coreAuditTableNames = Map.of();
|
||||
private String appRole = "app";
|
||||
private MetaDataCustomizerInterface<QTableMetaData> tableMetaDataCustomizer = null;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read exportLockMode.
|
||||
*******************************************************************************/
|
||||
public String getExportLockMode()
|
||||
{
|
||||
return (exportLockMode);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure exportLockMode.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withExportLockMode(String exportLockMode)
|
||||
{
|
||||
this.exportLockMode = exportLockMode;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read exportRetentionDays.
|
||||
*******************************************************************************/
|
||||
public int getExportRetentionDays()
|
||||
{
|
||||
return (exportRetentionDays);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure exportRetentionDays.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withExportRetentionDays(int exportRetentionDays)
|
||||
{
|
||||
this.exportRetentionDays = exportRetentionDays;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read effective settings through the instance-scoped 30-second cache.
|
||||
*******************************************************************************/
|
||||
@JsonIgnore
|
||||
public WormAuditSettings.Values effectiveSettings()
|
||||
{
|
||||
return (settings.read());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Discard this instance's cached row after a successful local settings write.
|
||||
*******************************************************************************/
|
||||
public void invalidateSettings()
|
||||
{
|
||||
settings.invalidate();
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether to create the standalone app. Widgets remain available when false.
|
||||
*******************************************************************************/
|
||||
public boolean getIncludeApp()
|
||||
{
|
||||
return (includeApp);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Disable standalone app creation when the host owns the Audit app and help.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withIncludeApp(boolean includeApp)
|
||||
{
|
||||
this.includeApp = includeApp;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the app name (the host-owned destination when includeApp is false).
|
||||
*******************************************************************************/
|
||||
public String getAppName()
|
||||
{
|
||||
return (appName);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure the app name.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withAppName(String appName)
|
||||
{
|
||||
this.appName = appName;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the app label.
|
||||
*******************************************************************************/
|
||||
public String getAppLabel()
|
||||
{
|
||||
return (appLabel);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure the app label.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withAppLabel(String appLabel)
|
||||
{
|
||||
this.appLabel = appLabel;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the optional parent app.
|
||||
*******************************************************************************/
|
||||
public String getParentAppName()
|
||||
{
|
||||
return (parentAppName);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure the optional parent app.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withParentAppName(String parentAppName)
|
||||
{
|
||||
this.parentAppName = parentAppName;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the table route prefix.
|
||||
*******************************************************************************/
|
||||
public String getTablePathPrefix()
|
||||
{
|
||||
return (tablePathPrefix);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure the table route prefix.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withTablePathPrefix(String tablePathPrefix)
|
||||
{
|
||||
this.tablePathPrefix = tablePathPrefix;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the optional storage boundary (defaults to qbit-object-storage).
|
||||
*******************************************************************************/
|
||||
@JsonIgnore
|
||||
public Supplier<com.kingsrook.qbits.wormaudit.export.SealedStore> getSealedStoreSupplier()
|
||||
{
|
||||
return (sealedStoreSupplier);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure the optional storage boundary (defaults to qbit-object-storage).
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withSealedStoreSupplier(Supplier<com.kingsrook.qbits.wormaudit.export.SealedStore> sealedStoreSupplier)
|
||||
{
|
||||
this.sealedStoreSupplier = sealedStoreSupplier;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read backendName.
|
||||
*******************************************************************************/
|
||||
@@ -102,6 +287,7 @@ public class WormAuditQBitConfig implements QBitConfig
|
||||
/*******************************************************************************
|
||||
** Read signingKey.
|
||||
*******************************************************************************/
|
||||
@JsonIgnore
|
||||
public Supplier<SigningKey> getSigningKey()
|
||||
{
|
||||
return (signingKey);
|
||||
@@ -121,6 +307,7 @@ public class WormAuditQBitConfig implements QBitConfig
|
||||
/*******************************************************************************
|
||||
** Read trustedPublicKeys.
|
||||
*******************************************************************************/
|
||||
@JsonIgnore
|
||||
public Map<String, PublicKey> getTrustedPublicKeys()
|
||||
{
|
||||
return (trustedPublicKeys);
|
||||
@@ -260,6 +447,63 @@ public class WormAuditQBitConfig implements QBitConfig
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the Object Lock bucket alias.
|
||||
*******************************************************************************/
|
||||
public String getExportBucketAlias()
|
||||
{
|
||||
return (exportBucketAlias);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure the Object Lock bucket alias.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withExportBucketAlias(String exportBucketAlias)
|
||||
{
|
||||
this.exportBucketAlias = exportBucketAlias;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the export key prefix.
|
||||
*******************************************************************************/
|
||||
public String getExportPrefix()
|
||||
{
|
||||
return (exportPrefix);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure the export key prefix.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withExportPrefix(String exportPrefix)
|
||||
{
|
||||
this.exportPrefix = exportPrefix;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the UTC export schedule.
|
||||
*******************************************************************************/
|
||||
public String getExportCron()
|
||||
{
|
||||
return (exportCron);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure the UTC export schedule.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withExportCron(String exportCron)
|
||||
{
|
||||
this.exportCron = exportCron;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure verifyCron.
|
||||
*******************************************************************************/
|
||||
@@ -352,6 +596,18 @@ public class WormAuditQBitConfig implements QBitConfig
|
||||
*******************************************************************************/
|
||||
public void validate()
|
||||
{
|
||||
if(appName == null || !appName.matches("[A-Za-z0-9_-]+") || appLabel == null || appLabel.isBlank()
|
||||
|| (parentAppName != null && !parentAppName.matches("[A-Za-z0-9_-]+"))
|
||||
|| tablePathPrefix == null || !tablePathPrefix.matches("/[A-Za-z0-9_/-]*") || tablePathPrefix.endsWith("/"))
|
||||
{
|
||||
throw (new IllegalArgumentException("App names and a local table path prefix are required"));
|
||||
}
|
||||
if(exportBucketAlias == null || exportBucketAlias.isBlank() || exportCron == null || exportCron.isBlank()
|
||||
|| exportPrefix == null || !exportPrefix.matches("[A-Za-z0-9_-]+(/[A-Za-z0-9_-]+)*"))
|
||||
{
|
||||
throw (new IllegalArgumentException("Export bucket, prefix and schedule must be configured"));
|
||||
}
|
||||
WormAuditSettings.validate(WormAuditSettings.defaults(this));
|
||||
if(backendName == null || backendName.isBlank() || retentionYears < 1 || chainIntervalSeconds < 1
|
||||
|| lateDetailLookbackHours < 1 || settleHorizonSeconds < 0 || batchSize < 1 || checkpointEveryLinks < 1 || checkpointEveryMinutes < 1)
|
||||
{
|
||||
|
||||
@@ -21,24 +21,30 @@
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.wormaudit.app.WormAuditApp;
|
||||
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
|
||||
import com.kingsrook.qbits.wormaudit.guard.AppendOnlyGuard;
|
||||
import com.kingsrook.qbits.wormaudit.model.LinkRangeWidget;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditHelp;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.ExportProcesses;
|
||||
import com.kingsrook.qbits.wormaudit.processes.WormAuditProcesses;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertDefaultMonitor;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertSeverity;
|
||||
import com.kingsrook.qqq.backend.core.alerts.QAlertTypeMetaData;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaDataProducer;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.quartz.QuartzSchedulerMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@@ -122,7 +128,7 @@ public class WormAuditQBitProducer implements QBitMetaDataProducer<WormAuditQBit
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Register phase (a) tables, processes and schedules on the host backend.
|
||||
** Register audit tables, verification/export actions and schedules.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public MetaDataProducerMultiOutput produce(QInstance qInstance) throws QException
|
||||
@@ -149,6 +155,14 @@ public class WormAuditQBitProducer implements QBitMetaDataProducer<WormAuditQBit
|
||||
{
|
||||
table = qBitConfig.getTableMetaDataCustomizer().customizeMetaData(qInstance, table);
|
||||
}
|
||||
for(var field : table.getFields().values())
|
||||
{
|
||||
if(field.getInlinePossibleValueSource() != null)
|
||||
{
|
||||
output.add(field.getInlinePossibleValueSource());
|
||||
field.withPossibleValueSourceName(field.getInlinePossibleValueSource().getName()).withInlinePossibleValueSource(null);
|
||||
}
|
||||
}
|
||||
guard.customizeMetaData(qInstance, table);
|
||||
table.setSourceQBitName(qBit.getName());
|
||||
output.add(table);
|
||||
@@ -159,37 +173,97 @@ public class WormAuditQBitProducer implements QBitMetaDataProducer<WormAuditQBit
|
||||
}
|
||||
for(String name : WormAuditTables.ALL)
|
||||
{
|
||||
output.add(QPossibleValueSource.newForTable(name));
|
||||
QPossibleValueSource source = QPossibleValueSource.newForTable(name);
|
||||
if(WormAuditTables.CHAIN.equals(name))
|
||||
{
|
||||
source.withValueFormat("%s").withValueFields(java.util.List.of("id"));
|
||||
}
|
||||
output.add(source);
|
||||
}
|
||||
for(String name : java.util.List.of(WormAuditTables.CHECKPOINT, WormAuditTables.EXPORT, WormAuditTables.VERIFICATION))
|
||||
{
|
||||
output.add(new QWidgetMetaData().withName(name + "_links").withLabel("Chain links").withType("childRecordList")
|
||||
output.add(new QWidgetMetaData().withName(name + "_links").withLabel("Chain links").withType("childRecordList").withGridColumns(12)
|
||||
.withCodeReference(new QCodeReference(LinkRangeWidget.class)).withDefaultValue("parentTable", name));
|
||||
}
|
||||
for(var process : WormAuditProcesses.processes(qBitConfig))
|
||||
List<QTableMetaData> producedTables = output.getEach(QTableMetaData.class);
|
||||
output.addSelfToInstance(qInstance);
|
||||
output = new MetaDataProducerMultiOutput();
|
||||
var previousInstance = QContext.getQInstance();
|
||||
var previousSession = QContext.getQSession();
|
||||
var previousTransaction = QContext.getQBackendTransaction();
|
||||
try
|
||||
{
|
||||
process.setSourceQBitName(qBit.getName());
|
||||
output.add(process);
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
// Production is still assembling the host metadata. Do not validate it //
|
||||
// mid-build. The startup settings read is system work: run it as the //
|
||||
// core system user. A plain tokenless QSession fails session validation //
|
||||
// under token-based host authentication (OAuth2, Auth0). //
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
QContext.setQInstance(qInstance);
|
||||
QContext.setQSession(new QSystemUserSession(qInstance));
|
||||
QContext.setTransaction(null);
|
||||
addProcesses(output, qBit);
|
||||
}
|
||||
finally
|
||||
{
|
||||
QContext.setQInstance(previousInstance);
|
||||
QContext.setQSession(previousSession);
|
||||
QContext.setTransaction(previousTransaction);
|
||||
}
|
||||
if(qInstance.getSchedulers() == null || !qInstance.getSchedulers().containsKey(qBitConfig.getSchedulerName()))
|
||||
{
|
||||
qInstance.addScheduler(new QuartzSchedulerMetaData().withName(qBitConfig.getSchedulerName()));
|
||||
}
|
||||
QAppMetaData app = new QAppMetaData().withName("worm_audit").withLabel("WORM Audit").withIcon(new QIcon("policy"));
|
||||
for(String name : WormAuditTables.NAV)
|
||||
if(qBitConfig.getIncludeApp())
|
||||
{
|
||||
QTableMetaData table = output.getEach(QTableMetaData.class).stream().filter(t -> t.getName().equals(name)).findFirst().orElseThrow();
|
||||
app.withSectionOfChildren(new QAppSection().withName(name).withLabel(table.getLabel()), table);
|
||||
output.add(WormAuditApp.app(qBitConfig, producedTables));
|
||||
}
|
||||
for(QWidgetMetaData widget : WormAuditApp.widgets())
|
||||
{
|
||||
output.add(widget);
|
||||
}
|
||||
output.add(new QAlertTypeMetaData().withName("wormaudit.chain.break").withLabel("Audit chain break")
|
||||
.withSourceQBitName(qBit.getName()).withDefaultSeverity(AlertSeverity.CRITICAL)
|
||||
.withDescription("Audit chain verification found a break in content, continuity, coverage or checkpoint trust.")
|
||||
.withSubjectTableName(WormAuditTables.VERIFICATION).withVisibilityPermission("worm_audit_verification.read"));
|
||||
output.add(new QAlertTypeMetaData().withName("wormaudit.export.failed").withLabel("Sealed audit export failed")
|
||||
.withSourceQBitName(qBit.getName()).withDefaultSeverity(AlertSeverity.ERROR)
|
||||
.withDescription("The audit export could not be sealed with verified retention and object versions.")
|
||||
.withSubjectTableName(WormAuditTables.EXPORT).withVisibilityPermission("worm_audit_export.read"));
|
||||
for(String job : List.of("verify", "export"))
|
||||
{
|
||||
String name = "wormaudit." + job + ".daily";
|
||||
output.add(new QAlertTypeMetaData().withName(name).withLabel("Daily audit " + job + " overdue")
|
||||
.withSourceQBitName(qBit.getName()).withDefaultSeverity(AlertSeverity.ERROR)
|
||||
.withDescription("The daily audit " + job + " job has not sent a heartbeat within 26 hours.")
|
||||
.withVisibilityPermission(job.equals("verify") ? "worm_audit_verification.read" : "worm_audit_export.read")
|
||||
.withDefaultMonitor(AlertDefaultMonitor.heartbeat(name, name, 24 * 3600, 2 * 3600)));
|
||||
}
|
||||
output.add(app);
|
||||
// RC.5 rejects underscores in declared alert type names. QAlerts still
|
||||
// dispatches the specified worm_audit.chain.break signal to installed sinks.
|
||||
output.addSelfToInstance(qInstance);
|
||||
WormAuditHelp.apply(qInstance, qBitConfig);
|
||||
denyAssistant(qInstance);
|
||||
return (new MetaDataProducerMultiOutput());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build schedules from settings available at producer startup.
|
||||
*******************************************************************************/
|
||||
private void addProcesses(MetaDataProducerMultiOutput output, QBitMetaData qBit)
|
||||
{
|
||||
for(var process : WormAuditProcesses.processes(qBitConfig))
|
||||
{
|
||||
process.setSourceQBitName(qBit.getName());
|
||||
output.add(process);
|
||||
}
|
||||
for(var process : ExportProcesses.processes(qBitConfig))
|
||||
{
|
||||
process.setSourceQBitName(qBit.getName());
|
||||
output.add(process);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** RC.5 compatibility: replace with a direct call when the qbit builds on RC.6 (qqq#1141, #1163).
|
||||
*******************************************************************************/
|
||||
|
||||
@@ -0,0 +1,253 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import java.sql.SQLException;
|
||||
import java.time.Clock;
|
||||
import java.time.Instant;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import org.quartz.CronExpression;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Singleton settings merged over host defaults. Only absent rows/tables fall
|
||||
** back; database outages and invalid persisted settings fail closed.
|
||||
*******************************************************************************/
|
||||
public class WormAuditSettings
|
||||
{
|
||||
private final WormAuditQBitConfig config;
|
||||
private final Clock clock;
|
||||
private QInstance instance;
|
||||
private QRecord cached;
|
||||
private Instant expires = Instant.MIN;
|
||||
|
||||
/*******************************************************************************
|
||||
** An immutable snapshot for one run.
|
||||
*******************************************************************************/
|
||||
public record Values(int audit_retention_years,
|
||||
String export_lock_mode,
|
||||
int export_retention_days,
|
||||
int checkpoint_every_links,
|
||||
int checkpoint_every_minutes,
|
||||
int chain_interval_seconds,
|
||||
int settle_horizon_seconds,
|
||||
int late_detail_lookback_hours,
|
||||
String export_cron,
|
||||
String verify_cron)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Use the production clock.
|
||||
*******************************************************************************/
|
||||
public WormAuditSettings(WormAuditQBitConfig config)
|
||||
{
|
||||
this(config, Clock.systemUTC());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Inject a clock for cache boundary tests.
|
||||
*******************************************************************************/
|
||||
public WormAuditSettings(WormAuditQBitConfig config, Clock clock)
|
||||
{
|
||||
this.config = config;
|
||||
this.clock = clock;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Invalidate after a local edit; other application instances refresh within 30 seconds.
|
||||
*******************************************************************************/
|
||||
public synchronized void invalidate()
|
||||
{
|
||||
expires = Instant.MIN;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Seed the singleton only when absent; never overwrite an administrator edit.
|
||||
** The database script (changeset worm-audit:10) is the normal seed; this remains
|
||||
** for memory and H2 hosts, whose settings table cannot take generic inserts.
|
||||
**
|
||||
** @deprecated for PostgreSQL hosts, which get row 1 from the database script;
|
||||
** still the supported way to create row 1 on memory and H2 hosts. Reads already
|
||||
** fall back to the configured values while the row is absent.
|
||||
*******************************************************************************/
|
||||
@Deprecated
|
||||
public static void seed(WormAuditQBitConfig config) throws com.kingsrook.qqq.backend.core.exceptions.QException
|
||||
{
|
||||
var store = new AuditStore(null);
|
||||
if(store.get(WormAuditTables.SETTINGS, 1) == null)
|
||||
{
|
||||
store.insert(WormAuditTables.SETTINGS, java.util.List.of(defaults(config)));
|
||||
}
|
||||
config.invalidateSettings();
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read at most once per 30 seconds, isolated by QInstance identity.
|
||||
*******************************************************************************/
|
||||
public synchronized Values read()
|
||||
{
|
||||
QInstance current = QContext.getQInstance();
|
||||
if(current != instance || !clock.instant().isBefore(expires))
|
||||
{
|
||||
QRecord row = null;
|
||||
if(current != null && current.getTable(WormAuditTables.SETTINGS) != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
row = new AuditStore(null).get(WormAuditTables.SETTINGS, 1);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
if(!missingTable(e))
|
||||
{
|
||||
throw (new IllegalStateException("Cannot read WORM Audit Settings", e));
|
||||
}
|
||||
}
|
||||
}
|
||||
cached = row;
|
||||
instance = current;
|
||||
expires = row == null ? Instant.MIN : clock.instant().plusSeconds(30);
|
||||
}
|
||||
QRecord row = defaults(config);
|
||||
if(cached != null)
|
||||
{
|
||||
for(String field : row.getValues().keySet())
|
||||
{
|
||||
if(cached.getValue(field) != null)
|
||||
{
|
||||
row.setValue(field, cached.getValue(field));
|
||||
}
|
||||
}
|
||||
}
|
||||
validate(row);
|
||||
return (new Values(row.getValueInteger("audit_retention_years"),
|
||||
row.getValueString("export_lock_mode"),
|
||||
row.getValueInteger("export_retention_days"),
|
||||
row.getValueInteger("checkpoint_every_links"),
|
||||
row.getValueInteger("checkpoint_every_minutes"),
|
||||
row.getValueInteger("chain_interval_seconds"),
|
||||
row.getValueInteger("settle_horizon_seconds"),
|
||||
row.getValueInteger("late_detail_lookback_hours"),
|
||||
row.getValueString("export_cron"),
|
||||
row.getValueString("verify_cron")));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build insert-only seed values from the host defaults.
|
||||
*******************************************************************************/
|
||||
public static QRecord defaults(WormAuditQBitConfig config)
|
||||
{
|
||||
return (new QRecord().withValue("id", 1)
|
||||
.withValue("audit_retention_years", config.getRetentionYears())
|
||||
.withValue("export_lock_mode", config.getExportLockMode())
|
||||
.withValue("export_retention_days", config.getExportRetentionDays())
|
||||
.withValue("checkpoint_every_links", config.getCheckpointEveryLinks())
|
||||
.withValue("checkpoint_every_minutes", config.getCheckpointEveryMinutes())
|
||||
.withValue("chain_interval_seconds", config.getChainIntervalSeconds())
|
||||
.withValue("settle_horizon_seconds", config.getSettleHorizonSeconds())
|
||||
.withValue("late_detail_lookback_hours", config.getLateDetailLookbackHours())
|
||||
.withValue("export_cron", config.getExportCron())
|
||||
.withValue("verify_cron", config.getVerifyCron()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Validate supplied fields; partial updates leave omitted fields untouched.
|
||||
*******************************************************************************/
|
||||
public static void validate(QRecord row)
|
||||
{
|
||||
if(row.getValue("id") != null && !Integer.valueOf(1).equals(row.getValueInteger("id")))
|
||||
{
|
||||
throw (new IllegalArgumentException("WORM Audit Settings has one row only (ID 1)."));
|
||||
}
|
||||
range(row, "audit_retention_years", "Audit retention years", 1, 100);
|
||||
range(row, "export_retention_days", "Export retention days", 1, 36500);
|
||||
range(row, "checkpoint_every_links", "Checkpoint every links", 1, 2147483647);
|
||||
range(row, "checkpoint_every_minutes", "Checkpoint every minutes", 1, 2147483647);
|
||||
range(row, "chain_interval_seconds", "Chain interval seconds", 1, 2147483647);
|
||||
range(row, "settle_horizon_seconds", "Settle horizon seconds", 0, 2147483647);
|
||||
range(row, "late_detail_lookback_hours", "Late detail lookback hours", 1, 2147483647);
|
||||
if(row.getValues().containsKey("export_lock_mode") && !java.util.Arrays.asList("GOVERNANCE", "COMPLIANCE").contains(row.getValueString("export_lock_mode")))
|
||||
{
|
||||
throw (new IllegalArgumentException("Export lock mode must be GOVERNANCE or COMPLIANCE."));
|
||||
}
|
||||
for(String field : java.util.List.of("export_cron", "verify_cron"))
|
||||
{
|
||||
if(row.getValues().containsKey(field) && (row.getValueString(field) == null || !CronExpression.isValidExpression(row.getValueString(field))))
|
||||
{
|
||||
throw (new IllegalArgumentException((field.equals("export_cron") ? "Export" : "Verify") + " schedule must be a valid Quartz cron expression (UTC)."));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reject null, fractional and out-of-range numbers with a plain field label.
|
||||
*******************************************************************************/
|
||||
private static void range(QRecord row, String field, String label, int minimum, int maximum)
|
||||
{
|
||||
if(!row.getValues().containsKey(field))
|
||||
{
|
||||
return;
|
||||
}
|
||||
try
|
||||
{
|
||||
long value = Long.parseLong(String.valueOf(row.getValue(field)));
|
||||
if(value >= minimum && value <= maximum)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
catch(NumberFormatException ignored)
|
||||
{
|
||||
// Use the same clear message for invalid numbers and invalid ranges.
|
||||
}
|
||||
throw (new IllegalArgumentException(label + " must be a whole number from " + minimum + " to " + maximum + "."));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Recognize PostgreSQL/H2 missing-table errors without hiding other failures.
|
||||
*******************************************************************************/
|
||||
private static boolean missingTable(Throwable error)
|
||||
{
|
||||
for(Throwable cause = error; cause != null; cause = cause.getCause())
|
||||
{
|
||||
if(cause instanceof SQLException sql && ("42P01".equals(sql.getSQLState()) || "42S02".equals(sql.getSQLState()) || "42S04".equals(sql.getSQLState())))
|
||||
{
|
||||
return (true);
|
||||
}
|
||||
}
|
||||
return (false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.app;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSTransaction;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read-only PostgreSQL catalog probes; errors become value-free diagnostics.
|
||||
*******************************************************************************/
|
||||
public final class PostgresSetupWarnings
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private PostgresSetupWarnings()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Probe the runtime connection, never the configured migration role.
|
||||
*******************************************************************************/
|
||||
public static List<String> load(QBackendTransaction transaction, WormAuditQBitConfig config)
|
||||
{
|
||||
List<String> warnings = new ArrayList<>();
|
||||
int years = config.effectiveSettings().audit_retention_years();
|
||||
if(years != config.getRetentionYears())
|
||||
{
|
||||
warnings.add("Audit retention setting differs from the configured database purge interval (" + config.getRetentionYears() + " years). A reviewed migration is required.");
|
||||
}
|
||||
if(!(transaction instanceof RDBMSTransaction rdbms))
|
||||
{
|
||||
return (warnings);
|
||||
}
|
||||
try
|
||||
{
|
||||
var connection = rdbms.getConnection();
|
||||
if(!"PostgreSQL".equals(connection.getMetaData().getDatabaseProductName()))
|
||||
{
|
||||
return (warnings);
|
||||
}
|
||||
// The live function is authoritative on PostgreSQL, rather than the renderer defaults.
|
||||
warnings.clear();
|
||||
List<String> tables = new ArrayList<>(config.getCoreAuditTableNames().values());
|
||||
tables.addAll(WormAuditTables.ALL.stream().filter(t -> !WormAuditTables.SETTINGS.equals(t)).toList());
|
||||
for(String table : tables)
|
||||
{
|
||||
for(String trigger : List.of("worm_audit_row_guard", "worm_audit_truncate_guard"))
|
||||
{
|
||||
try(var statement = connection.prepareStatement("SELECT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid=to_regclass(?) AND tgname=? AND NOT tgisinternal AND tgenabled IN ('O','A'))"))
|
||||
{
|
||||
statement.setString(1, WormAuditDdl.identifier(table));
|
||||
statement.setString(2, trigger);
|
||||
try(var result = statement.executeQuery())
|
||||
{
|
||||
result.next();
|
||||
if(!result.getBoolean(1))
|
||||
{
|
||||
warnings.add("Missing or disabled " + trigger + " on " + table);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
try(var statement = connection.prepareStatement("SELECT EXISTS (SELECT 1 FROM pg_tables WHERE tableowner=current_user AND to_regclass(quote_ident(schemaname)||'.'||quote_ident(tablename))=to_regclass(?))"))
|
||||
{
|
||||
statement.setString(1, WormAuditDdl.identifier(table));
|
||||
try(var result = statement.executeQuery())
|
||||
{
|
||||
result.next();
|
||||
if(result.getBoolean(1))
|
||||
{
|
||||
warnings.add("Runtime role owns audit table " + table);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
try(var statement = connection.createStatement(); var result = statement.executeQuery("SELECT pg_get_functiondef(to_regprocedure('public.worm_audit_purge(timestamp with time zone)'))"))
|
||||
{
|
||||
if(result.next() && result.getString(1) != null)
|
||||
{
|
||||
var interval = java.util.regex.Pattern.compile("(?i)interval\\s+'([0-9]+) years'").matcher(result.getString(1));
|
||||
if(interval.find() && Integer.parseInt(interval.group(1)) != years)
|
||||
{
|
||||
warnings.add("Audit retention setting differs from the purge function's compiled interval (" + interval.group(1) + " years). A reviewed migration is required.");
|
||||
}
|
||||
}
|
||||
}
|
||||
try(var statement = connection.createStatement(); var result = statement.executeQuery("SELECT to_regprocedure('public.worm_audit_purge(timestamp with time zone)') IS NOT NULL"))
|
||||
{
|
||||
result.next();
|
||||
if(!result.getBoolean(1))
|
||||
{
|
||||
warnings.add("Retention purge function is missing");
|
||||
}
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
warnings.add("PostgreSQL setup checks unavailable");
|
||||
}
|
||||
return (warnings);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.app;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.VerificationRuns;
|
||||
import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.CustomPermissionChecker;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML;
|
||||
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** WORM Audit navigation and standard HTML widgets, scoped to this QInstance.
|
||||
*******************************************************************************/
|
||||
public final class WormAuditApp
|
||||
{
|
||||
public static final String HOME_WIDGET = "wormAuditHome";
|
||||
public static final String HEALTH_CARD_WIDGET = "wormAuditHealthCard";
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private WormAuditApp()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The home and optional host-dashboard summary share the app's permissions.
|
||||
*******************************************************************************/
|
||||
public static List<QWidgetMetaData> widgets()
|
||||
{
|
||||
return (List.of(
|
||||
new QWidgetMetaData().withName(HOME_WIDGET).withLabel("Overview").withType(WidgetType.HTML.getType())
|
||||
.withCodeReference(new QCodeReference(HomeRenderer.class)).withGridColumns(12).withIsCard(false).withPermissionRules(permissions()),
|
||||
new QWidgetMetaData().withName(HEALTH_CARD_WIDGET).withLabel("WORM Audit").withType(WidgetType.HTML.getType())
|
||||
.withCodeReference(new QCodeReference(HealthCardRenderer.class)).withGridColumns(12).withIsCard(true).withPermissionRules(permissions())));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Home, Verifications, Sealed Exports, Checkpoints, Audit Chain, Purges.
|
||||
*******************************************************************************/
|
||||
public static QAppMetaData app(WormAuditQBitConfig config, List<QTableMetaData> tables)
|
||||
{
|
||||
QAppMetaData app = new QAppMetaData().withName(config.getAppName()).withLabel(config.getAppLabel())
|
||||
.withIcon(new QIcon("verified_user")).withPermissionRules(permissions()).withWidgets(List.of(HOME_WIDGET));
|
||||
app.setParentAppName(config.getParentAppName());
|
||||
for(String name : WormAuditTables.NAV)
|
||||
{
|
||||
QTableMetaData table = tables.stream().filter(t -> t.getName().equals(name)).findFirst().orElseThrow();
|
||||
app.withSectionOfChildren(new QAppSection().withName(name).withLabel(WormAuditTables.navigationLabel(name)),
|
||||
WormAuditTables.SETTINGS.equals(name) ? table.clone().withLabel("Settings") : table);
|
||||
}
|
||||
return (app);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read on any WORM table grants the value-free aggregate overview.
|
||||
*******************************************************************************/
|
||||
public static QPermissionRules permissions()
|
||||
{
|
||||
return (new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION)
|
||||
.withCustomPermissionChecker(new QCodeReference(HomePermissionChecker.class)));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Honor the host's actual table permission rules, including custom checkers.
|
||||
*******************************************************************************/
|
||||
public static class HomePermissionChecker implements CustomPermissionChecker
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Reject direct widget requests unless at least one table is readable.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void checkPermissionsThrowing(AbstractActionInput input, MetaDataWithPermissionRules metadata) throws QPermissionDeniedException
|
||||
{
|
||||
for(String table : WormAuditTables.ALL)
|
||||
{
|
||||
try
|
||||
{
|
||||
PermissionsHelper.checkTablePermissionThrowing(new QueryInput(table), TablePermissionSubType.READ);
|
||||
return;
|
||||
}
|
||||
catch(QPermissionDeniedException expected)
|
||||
{
|
||||
// Another WORM table may grant access to the aggregate overview.
|
||||
}
|
||||
}
|
||||
throw (new QPermissionDeniedException("Read access to a WORM audit table is required"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Full-width home renderer.
|
||||
*******************************************************************************/
|
||||
public static class HomeRenderer extends AbstractWidgetRenderer
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Render only metadata, counts and value-free reasons.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public RenderWidgetOutput render(RenderWidgetInput input) throws QException
|
||||
{
|
||||
new HomePermissionChecker().checkPermissionsThrowing(input, null);
|
||||
return (new RenderWidgetOutput(new RawHTML(null, WormAuditHomeView.render(WormAuditHomeData.load(), VerificationRuns.config().getTablePathPrefix()))));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Optional compact host dashboard renderer.
|
||||
*******************************************************************************/
|
||||
public static class HealthCardRenderer extends AbstractWidgetRenderer
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Link the status summary to the owning app home.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public RenderWidgetOutput render(RenderWidgetInput input) throws QException
|
||||
{
|
||||
new HomePermissionChecker().checkPermissionsThrowing(input, null);
|
||||
WormAuditQBitConfig config = VerificationRuns.config();
|
||||
String path = config.getTablePathPrefix() + "/" + (config.getParentAppName() == null ? "" : config.getParentAppName() + "/") + config.getAppName();
|
||||
return (new RenderWidgetOutput(new RawHTML(config.getAppLabel(), WormAuditHomeView.renderHealthCard(WormAuditHomeData.load(), path))));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.app;
|
||||
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.export.ObjectStorageSealedStore;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedStore;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.VerificationRuns;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSTransaction;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read-only home snapshot. No audit messages, detail values or canonical content
|
||||
** enter this model. Unexported suffixes are lag; holes before the sealed head
|
||||
** are gaps. Only a later full scheduled chain PASS clears an open chain break.
|
||||
*******************************************************************************/
|
||||
public record WormAuditHomeData(long links, Instant lastLinkAt, long backlog, long settling, int settleSeconds,
|
||||
QRecord lastVerification, QRecord openBreak, QRecord lastExport, long exportLag, Long exportAgeHours,
|
||||
List<String> exportGaps, String bucketAlias, String lockCapability, String lockReason,
|
||||
String exportLockMode, int exportRetentionDays, int retentionYears, QRecord anchor, Instant oldestAudit, Instant nextPurge,
|
||||
List<QRecord> recentPurges, List<String> warnings)
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Load using the current instance's configuration and production clock.
|
||||
*******************************************************************************/
|
||||
public static WormAuditHomeData load() throws QException
|
||||
{
|
||||
return (load(VerificationRuns.config(), null, Instant.now()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The storage and time seams also serve the memory demo and deterministic tests.
|
||||
*******************************************************************************/
|
||||
public static WormAuditHomeData load(WormAuditQBitConfig config, SealedStore storage, Instant now) throws QException
|
||||
{
|
||||
var settings = config.effectiveSettings();
|
||||
List<String> warnings = new ArrayList<>();
|
||||
if("GOVERNANCE".equals(settings.export_lock_mode()))
|
||||
{
|
||||
warnings.add("Sealed exports use GOVERNANCE mode: an administrator with s3:BypassGovernanceRetention can delete them. Switch to COMPLIANCE for write-once copies.");
|
||||
}
|
||||
if(now.plus(Duration.ofDays(settings.export_retention_days())).isBefore(now.atZone(ZoneOffset.UTC).plusYears(settings.audit_retention_years()).toInstant()))
|
||||
{
|
||||
warnings.add("Export retention is shorter than audit retention.");
|
||||
}
|
||||
String capability = "UNKNOWN";
|
||||
String reason = "Object Lock capability could not be confirmed";
|
||||
try
|
||||
{
|
||||
SealedStore resolved = storage != null ? storage : config.getSealedStoreSupplier() == null
|
||||
? new ObjectStorageSealedStore(config.getExportBucketAlias(), "home") : config.getSealedStoreSupplier().get();
|
||||
CapabilityStatus status = resolved.lockCapability();
|
||||
if(status == CapabilityStatus.SUPPORTED || status == CapabilityStatus.UNSUPPORTED)
|
||||
{
|
||||
capability = status.name();
|
||||
reason = status == CapabilityStatus.SUPPORTED ? null : "Export bucket does not support Object Lock";
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
// Provider exception messages can contain credentials or object content.
|
||||
reason = "Object Lock capability check failed";
|
||||
}
|
||||
if(reason != null)
|
||||
{
|
||||
warnings.add(reason);
|
||||
}
|
||||
if(config.getSigningKey() == null)
|
||||
{
|
||||
warnings.add("No signing key configured");
|
||||
}
|
||||
if(config.getTrustedPublicKeys() == null || config.getTrustedPublicKeys().isEmpty())
|
||||
{
|
||||
warnings.add("No trusted public keys configured");
|
||||
}
|
||||
try(QBackendTransaction transaction = QBackendTransaction.openFor(new QueryInput(WormAuditTables.CHAIN)))
|
||||
{
|
||||
try
|
||||
{
|
||||
if(transaction instanceof RDBMSTransaction rdbms)
|
||||
{
|
||||
rdbms.getConnection().setTransactionIsolation(Connection.TRANSACTION_REPEATABLE_READ);
|
||||
}
|
||||
AuditStore store = new AuditStore(transaction);
|
||||
QRecord lastLink = store.last(WormAuditTables.CHAIN);
|
||||
long links = store.count(WormAuditTables.CHAIN, new QQueryFilter());
|
||||
Instant cutoff = now.minusSeconds(settings.settle_horizon_seconds());
|
||||
long backlog = store.unchainedCount(cutoff, true);
|
||||
long settling = store.unchainedCount(cutoff, false);
|
||||
QRecord verification = latest(store, WormAuditTables.VERIFICATION, new QQueryFilter(), "finished_at");
|
||||
QQueryFilter dailyFilter = new QQueryFilter(new QFilterCriteria("kind", QCriteriaOperator.EQUALS, "CHAIN"),
|
||||
new QFilterCriteria("trigger", QCriteriaOperator.EQUALS, "SCHEDULE"));
|
||||
QRecord daily = latest(store, WormAuditTables.VERIFICATION, dailyFilter, "finished_at");
|
||||
if(daily == null || daily.getValueInstant("finished_at") == null)
|
||||
{
|
||||
warnings.add("Daily verification has never run");
|
||||
}
|
||||
else if(daily.getValueInstant("finished_at").isBefore(now.minus(Duration.ofHours(26))))
|
||||
{
|
||||
warnings.add("Daily verification last ran more than 26 hours ago");
|
||||
}
|
||||
QRecord failure = latest(store, WormAuditTables.VERIFICATION, new QQueryFilter(
|
||||
new QFilterCriteria("kind", QCriteriaOperator.EQUALS, "CHAIN"), new QFilterCriteria("outcome", QCriteriaOperator.EQUALS, "FAIL")), "id");
|
||||
QRecord fullPass = latest(store, WormAuditTables.VERIFICATION, new QQueryFilter(
|
||||
new QFilterCriteria("kind", QCriteriaOperator.EQUALS, "CHAIN"), new QFilterCriteria("trigger", QCriteriaOperator.EQUALS, "SCHEDULE"),
|
||||
new QFilterCriteria("outcome", QCriteriaOperator.EQUALS, "PASS")), "id");
|
||||
QRecord openBreak = failure != null && (fullPass == null || failure.getValueLong("id") > fullPass.getValueLong("id")) ? failure : null;
|
||||
QRecord export = store.last(WormAuditTables.EXPORT);
|
||||
QRecord anchor = store.anchor();
|
||||
long anchorId = anchor.getValueLong("anchor_link_id");
|
||||
long exported = export == null ? anchorId : export.getValueLong("to_link_id");
|
||||
long lag = store.count(WormAuditTables.CHAIN, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, exported)));
|
||||
List<String> gaps = gaps(store, anchorId);
|
||||
List<QRecord> oldest = store.summary("audit", new QQueryFilter().withOrderBy(new QFilterOrderBy("timestamp", true)).withLimit(1), Set.of("timestamp"));
|
||||
Instant oldestAt = oldest.isEmpty() ? null : oldest.getFirst().getValueInstant("timestamp");
|
||||
List<QRecord> purges = store.query(WormAuditTables.PURGE, new QQueryFilter()
|
||||
.withOrderBy(new QFilterOrderBy("requested_at", false)).withOrderBy(new QFilterOrderBy("id", false)).withLimit(10));
|
||||
warnings.addAll(PostgresSetupWarnings.load(transaction, config));
|
||||
transaction.commit();
|
||||
return (new WormAuditHomeData(links, lastLink == null ? null : lastLink.getValueInstant("chained_at"), backlog, settling,
|
||||
settings.settle_horizon_seconds(), verification, openBreak, export, lag,
|
||||
export == null || export.getValueInstant("sealed_at") == null ? null : Math.max(0, Duration.between(export.getValueInstant("sealed_at"), now).toHours()),
|
||||
List.copyOf(gaps), config.getExportBucketAlias(), capability, reason, settings.export_lock_mode(), settings.export_retention_days(), settings.audit_retention_years(), anchorId == 0 ? null : anchor,
|
||||
oldestAt, oldestAt == null ? null : oldestAt.atZone(ZoneOffset.UTC).plusYears(settings.audit_retention_years()).toInstant(), List.copyOf(purges), List.copyOf(warnings)));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
transaction.rollback();
|
||||
throw (new QException("Cannot load WORM audit home summary"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read one result, using id to break equal-time ties deterministically.
|
||||
*******************************************************************************/
|
||||
private static QRecord latest(AuditStore store, String table, QQueryFilter filter, String time) throws QException
|
||||
{
|
||||
List<QRecord> rows = store.query(table, filter.withOrderBy(new QFilterOrderBy(time, false))
|
||||
.withOrderBy(new QFilterOrderBy("id", false)).withLimit(1));
|
||||
return (rows.isEmpty() ? null : rows.getFirst());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Sweep export intervals in bounded pages; never enumerate links or objects.
|
||||
** The writer appends ranges in id order. Overlapping ranges extend coverage.
|
||||
*******************************************************************************/
|
||||
private static List<String> gaps(AuditStore store, long anchor) throws QException
|
||||
{
|
||||
List<String> gaps = new ArrayList<>();
|
||||
long next = anchor + 1;
|
||||
long cursor = 0;
|
||||
while(true)
|
||||
{
|
||||
List<QRecord> page = store.page(WormAuditTables.EXPORT, cursor, 1000);
|
||||
if(page.isEmpty())
|
||||
{
|
||||
return (gaps);
|
||||
}
|
||||
for(QRecord export : page)
|
||||
{
|
||||
long from = export.getValueLong("from_link_id");
|
||||
long to = export.getValueLong("to_link_id");
|
||||
if(from > next)
|
||||
{
|
||||
gaps.add(next + "–" + (from - 1));
|
||||
}
|
||||
next = Math.max(next, to + 1);
|
||||
}
|
||||
cursor = page.getLast().getValueLong("id");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.app;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stacked, wrapping definition lists. Colors follow BackupsHomeView exactly:
|
||||
** inherited text/link colors, translucent neutral borders, muted opacity. Only
|
||||
** allowlisted summary fields are rendered; no audit content is ever inspected.
|
||||
*******************************************************************************/
|
||||
public final class WormAuditHomeView
|
||||
{
|
||||
private static final DateTimeFormatter WHEN = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm 'UTC'").withZone(ZoneOffset.UTC);
|
||||
private static final String CARD = "border:1px solid rgba(127,127,127,.3);border-radius:8px;padding:1rem;min-width:0;overflow-wrap:anywhere;";
|
||||
private static final String LINK = "color:inherit;text-decoration:underline;text-underline-offset:2px;";
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private WormAuditHomeView()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Full home with explicit empty states and no fixed-width content.
|
||||
*******************************************************************************/
|
||||
public static String render(WormAuditHomeData data, String prefix)
|
||||
{
|
||||
StringBuilder html = new StringBuilder("<div class=\"worm-audit-home\" style=\"display:grid;gap:1rem;min-width:0;max-width:100%;\">");
|
||||
open(html, "Chain status");
|
||||
row(html, "Links chained", data.links());
|
||||
row(html, "Last link", when(data.lastLinkAt(), "No links chained yet"));
|
||||
row(html, "Unchained backlog", data.backlog() + " settled; " + data.settling() + " settling (" + data.settleSeconds() + " s horizon)");
|
||||
QRecord verification = data.lastVerification();
|
||||
if(verification == null)
|
||||
{
|
||||
row(html, "Last verification", "No verification has run yet");
|
||||
}
|
||||
else
|
||||
{
|
||||
linkedRow(html, "Last verification", recordLink(prefix, WormAuditTables.VERIFICATION, verification.getValueLong("id"), "Verification #" + verification.getValueLong("id")));
|
||||
row(html, "Verified at", when(verification.getValueInstant("finished_at"), "Not recorded"));
|
||||
row(html, "Result", verification.getValueString("outcome") + "; " + verification.getValueLong("links_checked") + " links checked");
|
||||
}
|
||||
QRecord broken = data.openBreak();
|
||||
if(broken == null)
|
||||
{
|
||||
row(html, "Open chain breaks", verification == null ? "Unknown until verification runs" : "None reported");
|
||||
}
|
||||
else
|
||||
{
|
||||
linkedRow(html, "Open chain breaks", recordLink(prefix, WormAuditTables.VERIFICATION, broken.getValueLong("id"), broken.getValueLong("break_count") + " reported"));
|
||||
linkedRow(html, "First break link", recordLink(prefix, WormAuditTables.CHAIN, broken.getValueLong("first_break_link_id"), "Link #" + broken.getValueLong("first_break_link_id")));
|
||||
row(html, "First break audit", broken.getValueLong("first_break_audit_id"));
|
||||
row(html, "Reason", broken.getValueString("first_break_reason"));
|
||||
}
|
||||
close(html);
|
||||
open(html, "Export status");
|
||||
row(html, "Export lock mode", data.exportLockMode());
|
||||
row(html, "Export retention", data.exportRetentionDays() + " days");
|
||||
QRecord export = data.lastExport();
|
||||
if(export == null)
|
||||
{
|
||||
row(html, "Last sealed export", "No sealed exports yet");
|
||||
}
|
||||
else
|
||||
{
|
||||
linkedRow(html, "Last sealed export", recordLink(prefix, WormAuditTables.EXPORT, export.getValueLong("id"), "Export #" + export.getValueLong("id")));
|
||||
row(html, "Link range", export.getValueLong("from_link_id") + "–" + export.getValueLong("to_link_id"));
|
||||
row(html, "Sealed at", when(export.getValueInstant("sealed_at"), "Not recorded"));
|
||||
row(html, "Retain until", when(export.getValueInstant("retain_until"), "Not recorded"));
|
||||
row(html, "Last export lock mode", export.getValueString("lock_mode"));
|
||||
}
|
||||
row(html, "Export lag", data.exportLag() + " links; " + (data.exportAgeHours() == null ? "no previous export time" : data.exportAgeHours() + " h since last export"));
|
||||
row(html, "Gaps in sealed ranges", data.exportGaps().isEmpty() ? (export == null ? "No sealed ranges to check" : "None") : String.join(", ", data.exportGaps()));
|
||||
row(html, "Bucket alias", data.bucketAlias());
|
||||
row(html, "Object Lock capability", data.lockCapability() + (data.lockReason() == null ? "" : ": " + data.lockReason()));
|
||||
close(html);
|
||||
open(html, "Retention");
|
||||
row(html, "Configured retention", data.retentionYears() + " years");
|
||||
QRecord anchor = data.anchor();
|
||||
row(html, "Purge anchor", anchor == null ? "None" : "Link #" + anchor.getValueLong("anchor_link_id") + "; hash " + shortHash(anchor.getValueString("anchor_hash")));
|
||||
row(html, "Oldest live audit", when(data.oldestAudit(), "No live audits"));
|
||||
row(html, "Next eligible purge", when(data.nextPurge(), "No live audits to purge"));
|
||||
close(html);
|
||||
open(html, "Recent purges");
|
||||
if(data.recentPurges().isEmpty())
|
||||
{
|
||||
row(html, "History", "No purge requests yet");
|
||||
}
|
||||
for(QRecord purge : data.recentPurges())
|
||||
{
|
||||
linkedRow(html, "Purge", recordLink(prefix, WormAuditTables.PURGE, purge.getValueLong("id"), "Purge #" + purge.getValueLong("id")));
|
||||
row(html, "Requested at", when(purge.getValueInstant("requested_at"), "Not recorded"));
|
||||
row(html, "Requested by", purge.getValueString("requested_by"));
|
||||
row(html, "Outcome", purge.getValueString("outcome"));
|
||||
row(html, "Cutoff", when(purge.getValueInstant("cutoff"), "Not recorded"));
|
||||
row(html, "Deleted", purge.getValueLong("audits_deleted") + " audits; " + purge.getValueLong("details_deleted") + " details; " + purge.getValueLong("links_deleted") + " links");
|
||||
row(html, "Reason", purge.getValueString("reason"));
|
||||
}
|
||||
close(html);
|
||||
if(!data.warnings().isEmpty())
|
||||
{
|
||||
open(html, "Setup warnings");
|
||||
for(String warning : data.warnings())
|
||||
{
|
||||
row(html, "Warning", warning);
|
||||
}
|
||||
close(html);
|
||||
}
|
||||
return (html.append("</div>").toString());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Honest compact status, owned by and linked to the full home.
|
||||
*******************************************************************************/
|
||||
public static String renderHealthCard(WormAuditHomeData data, String homePath)
|
||||
{
|
||||
String status = data.openBreak() != null ? "Chain breaks reported" : data.lastVerification() == null ? "Not verified yet"
|
||||
: data.warnings().isEmpty() ? "No open chain breaks reported" : "Setup needs attention";
|
||||
return ("<div style=\"" + CARD + "\"><strong>" + esc(status) + "</strong><p>" + data.links() + " links; " + data.backlog()
|
||||
+ " settled audits waiting; " + data.exportLag() + " links awaiting export.</p><a style=\"" + LINK + "\" href=\"" + esc(homePath) + "\">Open WORM Audit</a></div>");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Section and definition list, with fluid wrapping label/value rows.
|
||||
*******************************************************************************/
|
||||
private static void open(StringBuilder html, String title)
|
||||
{
|
||||
html.append("<section style=\"").append(CARD).append("\"><h3 style=\"margin:0 0 .75rem;font-size:1rem;font-weight:600;\">")
|
||||
.append(esc(title)).append("</h3><dl style=\"margin:0;font-size:.9rem;\">");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Close a section.
|
||||
*******************************************************************************/
|
||||
private static void close(StringBuilder html)
|
||||
{
|
||||
html.append("</dl></section>");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Escape every value, including reasons and runtime identities.
|
||||
*******************************************************************************/
|
||||
private static void row(StringBuilder html, String label, Object value)
|
||||
{
|
||||
linkedRow(html, label, esc(value == null ? "Not recorded" : value.toString()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Only internally built and escaped links may supply HTML here.
|
||||
*******************************************************************************/
|
||||
private static void linkedRow(StringBuilder html, String label, String value)
|
||||
{
|
||||
html.append("<div style=\"display:flex;flex-wrap:wrap;gap:.25rem .75rem;margin:.4rem 0;min-width:0;\"><dt style=\"opacity:.75;flex:0 1 12rem;\">")
|
||||
.append(esc(label)).append("</dt><dd style=\"margin:0;flex:1 1 14rem;min-width:0;overflow-wrap:anywhere;\">").append(value).append("</dd></div>");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Standard QQQ record route; identifiers are numeric, never content.
|
||||
*******************************************************************************/
|
||||
private static String recordLink(String prefix, String table, Long id, String text)
|
||||
{
|
||||
return (id == null ? "Not recorded" : "<a style=\"" + LINK + "\" href=\"" + esc(prefix + "/" + table + "/" + id) + "\">" + esc(text) + "</a>");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** UTC time with an explicit empty state.
|
||||
*******************************************************************************/
|
||||
private static String when(Instant time, String empty)
|
||||
{
|
||||
return (time == null ? empty : WHEN.format(time));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Keep hashes compact even on a narrow screen.
|
||||
*******************************************************************************/
|
||||
private static String shortHash(String hash)
|
||||
{
|
||||
return (hash == null ? "Not recorded" : hash.substring(0, Math.min(12, hash.length())) + "…");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** HTML text and quoted attribute escaping.
|
||||
*******************************************************************************/
|
||||
private static String esc(String text)
|
||||
{
|
||||
return (text == null ? "" : text.replace("&", "&").replace("<", "<").replace(">", ">").replace("\"", """).replace("'", "'"));
|
||||
}
|
||||
}
|
||||
@@ -44,10 +44,12 @@ import com.kingsrook.qbits.wormaudit.guard.AppendOnlyGuard;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.VerificationResult;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
@@ -66,9 +68,21 @@ import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDe
|
||||
*******************************************************************************/
|
||||
public class AuditStore
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Every (detail id, link id) pair the chain's own detail lists cover.
|
||||
*******************************************************************************/
|
||||
private static final String CHAIN_PAIRS = "SELECT x.id::bigint AS detail_id, c.id AS chain_id FROM worm_audit_chain c"
|
||||
+ " CROSS JOIN LATERAL unnest(string_to_array(c.detail_ids, ',')) AS x(id) WHERE c.detail_ids <> ''";
|
||||
|
||||
/*******************************************************************************
|
||||
** Chain coverage counted per detail id, aggregated in one pass.
|
||||
*******************************************************************************/
|
||||
private static final String CHAIN_COVERAGE = "SELECT detail_id AS id, count(*) AS n FROM (" + CHAIN_PAIRS + ") pairs GROUP BY detail_id";
|
||||
|
||||
private final QBackendTransaction transaction;
|
||||
private final Map<String, String> names = new LinkedHashMap<>(16, 0.75F, true);
|
||||
private Boolean timestampWithZone;
|
||||
private Boolean postgres;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@@ -91,6 +105,57 @@ public class AuditStore
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Project only summary fields; home reads never fetch audit content.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> summary(String table, QQueryFilter filter, Set<String> fields) throws QException
|
||||
{
|
||||
QueryInput input = new QueryInput(table).withFilter(filter).withTransaction(transaction)
|
||||
.withFieldNamesToInclude(fields).withShouldFetchHeavyFields(true);
|
||||
input.withPermissionBypass("WORM audit home summary after widget permission check");
|
||||
return (new QueryAction().execute(input).getRecords());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Aggregate counts avoid loading the live chain for the home.
|
||||
*******************************************************************************/
|
||||
public long count(String table, QQueryFilter filter) throws QException
|
||||
{
|
||||
CountInput input = new CountInput(table).withFilter(filter).withTransaction(transaction);
|
||||
input.withPermissionBypass("WORM audit home count after widget permission check");
|
||||
return (new CountAction().execute(input).getCount());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Count the same AUDIT anti-join as the writer, split at the settle horizon.
|
||||
*******************************************************************************/
|
||||
public long unchainedCount(Instant cutoff, boolean settled) throws QException
|
||||
{
|
||||
if(isRdbms())
|
||||
{
|
||||
String sql = "SELECT count(*) FROM " + physical("audit") + " a WHERE a." + column("audit", "timestamp")
|
||||
+ (settled ? " < ?" : " >= ?") + unchainedPredicate();
|
||||
return (sqlIds(sql, cutoff).getFirst());
|
||||
}
|
||||
Set<Long> covered = summary(WormAuditTables.CHAIN, new QQueryFilter(new QFilterCriteria("kind", QCriteriaOperator.EQUALS, "AUDIT")), Set.of("audit_id"))
|
||||
.stream().map(r -> r.getValueLong("audit_id")).collect(Collectors.toSet());
|
||||
return (summary("audit", new QQueryFilter(), Set.of("id", "timestamp")).stream()
|
||||
.filter(r -> r.getValueInstant("timestamp") != null && r.getValueInstant("timestamp").isBefore(cutoff) == settled
|
||||
&& !covered.contains(r.getValueLong("id"))).count());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Shared SQL coverage predicate for both writer selection and home counts.
|
||||
*******************************************************************************/
|
||||
private String unchainedPredicate()
|
||||
{
|
||||
return (" AND NOT EXISTS (SELECT 1 FROM worm_audit_chain c WHERE c.audit_id=a." + column("audit", "id") + " AND c.kind='AUDIT')");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** All records ordered by id.
|
||||
*******************************************************************************/
|
||||
@@ -215,13 +280,23 @@ public class AuditStore
|
||||
** memory uses QQQ queries and the same ordering and cutoff semantics.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> unchained(Instant cutoff, int limit) throws QException
|
||||
{
|
||||
return (unchained(cutoff, limit, 0L));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Settled unchained headers above a swept audit id floor (RDBMS only); every
|
||||
** settled header at or below the floor is already known to be chained.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> unchained(Instant cutoff, int limit, long floor) throws QException
|
||||
{
|
||||
if(isRdbms())
|
||||
{
|
||||
String sql = "SELECT a." + column("audit", "id") + " FROM " + physical("audit") + " a WHERE a." + column("audit", "timestamp")
|
||||
+ " < ? AND NOT EXISTS (SELECT 1 FROM worm_audit_chain c WHERE c.audit_id=a." + column("audit", "id")
|
||||
+ " AND c.kind='AUDIT') ORDER BY a." + column("audit", "timestamp") + ", a." + column("audit", "id") + " LIMIT " + limit;
|
||||
Map<Long, QRecord> rows = index(in("audit", "id", sqlIds(sql, cutoff)));
|
||||
String sql = "SELECT a." + column("audit", "id") + " FROM " + physical("audit") + " a WHERE a." + column("audit", "id") + " > ? AND a."
|
||||
+ column("audit", "timestamp") + " < ?" + unchainedPredicate() + " ORDER BY a." + column("audit", "timestamp") + ", a." + column("audit", "id")
|
||||
+ " LIMIT " + limit;
|
||||
Map<Long, QRecord> rows = index(in("audit", "id", sqlIds(sql, floor, cutoff)));
|
||||
return (rows.values().stream().sorted(Comparator.comparing((QRecord r) -> r.getValueInstant("timestamp"))
|
||||
.thenComparing(r -> r.getValueLong("id"))).toList());
|
||||
}
|
||||
@@ -251,6 +326,15 @@ public class AuditStore
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Only the in-memory backend may use an unbounded completeness index.
|
||||
*******************************************************************************/
|
||||
public boolean isMemory()
|
||||
{
|
||||
return ("memory".equals(QContext.getQInstance().getBackendForTable(WormAuditTables.CHAIN).getBackendType()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fetch one keyset page, never using an offset or a whole-chain read.
|
||||
*******************************************************************************/
|
||||
@@ -319,15 +403,26 @@ public class AuditStore
|
||||
** Poll uncovered details in a bounded time window; memory retains its simple fallback.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> lateDetails(Instant cutoff, int hours, int limit) throws QException
|
||||
{
|
||||
return (lateDetails(cutoff, hours, limit, 0L));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Uncovered details above a swept detail id floor (RDBMS only). Coverage is a
|
||||
** primary-key probe of the normalized coverage table, never an array scan.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> lateDetails(Instant cutoff, int hours, int limit, long floor) throws QException
|
||||
{
|
||||
if(isRdbms())
|
||||
{
|
||||
String sql = "SELECT d." + column("auditDetail", "id") + " FROM " + physical("auditDetail") + " d JOIN " + physical("audit")
|
||||
String detailId = "d." + column("auditDetail", "id");
|
||||
String sql = "SELECT " + detailId + " FROM " + physical("auditDetail") + " d JOIN " + physical("audit")
|
||||
+ " a ON a." + column("audit", "id") + "=d." + column("auditDetail", "auditId")
|
||||
+ " WHERE a." + column("audit", "timestamp") + " < ? AND a." + column("audit", "timestamp") + " >= ?"
|
||||
+ " WHERE " + detailId + " > ? AND a." + column("audit", "timestamp") + " < ? AND a." + column("audit", "timestamp") + " >= ?"
|
||||
+ " AND EXISTS (SELECT 1 FROM worm_audit_chain h WHERE h.kind='AUDIT' AND h.audit_id=a." + column("audit", "id") + ")"
|
||||
+ " AND NOT EXISTS (SELECT 1 FROM worm_audit_chain c WHERE " + containsDetail() + ") ORDER BY d." + column("auditDetail", "id") + " LIMIT " + limit;
|
||||
return (in("auditDetail", "id", sqlIds(sql, cutoff, cutoff.minusSeconds(hours * 3600L))));
|
||||
+ " AND NOT EXISTS (SELECT 1 FROM " + WormAuditTables.CHAIN_DETAIL + " x WHERE x.detail_id=" + detailId + ") ORDER BY " + detailId + " LIMIT " + limit;
|
||||
return (in("auditDetail", "id", sqlIds(sql, floor, cutoff, cutoff.minusSeconds(hours * 3600L))));
|
||||
}
|
||||
List<QRecord> links = all(WormAuditTables.CHAIN);
|
||||
Set<Long> headers = links.stream().filter(r -> "AUDIT".equals(r.getValueString("kind"))).map(r -> r.getValueLong("audit_id")).collect(Collectors.toSet());
|
||||
@@ -348,11 +443,89 @@ public class AuditStore
|
||||
String time = "a." + column("audit", "timestamp");
|
||||
String headers = "SELECT " + auditId + " AS id, (SELECT count(*) FROM worm_audit_chain c WHERE c.kind='AUDIT' AND c.audit_id=" + auditId
|
||||
+ ") AS coverage FROM " + physical("audit") + " a WHERE " + time + " < ? OR " + time + " IS NULL";
|
||||
String details = "SELECT d." + column("auditDetail", "id") + " AS id, (SELECT count(*) FROM worm_audit_chain c WHERE " + containsDetail()
|
||||
+ ") AS coverage FROM " + physical("auditDetail") + " d LEFT JOIN " + physical("audit") + " a ON " + auditId + "=d."
|
||||
+ column("auditDetail", "auditId") + " WHERE " + time + " < ? OR " + time + " IS NULL";
|
||||
String detailId = "d." + column("auditDetail", "id");
|
||||
String scope = " FROM " + physical("auditDetail") + " d LEFT JOIN " + physical("audit") + " a ON " + auditId + "=d."
|
||||
+ column("auditDetail", "auditId");
|
||||
String details;
|
||||
if(isPostgres())
|
||||
{
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
// Count coverage from the chain itself (the hashed source of truth), //
|
||||
// aggregating every link's ids once rather than probing per detail. //
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
details = "SELECT " + detailId + " AS id, coalesce(cv.n, 0) AS coverage" + scope + " LEFT JOIN (" + CHAIN_COVERAGE
|
||||
+ ") cv ON cv.id=" + detailId + " WHERE " + time + " < ? OR " + time + " IS NULL";
|
||||
}
|
||||
else
|
||||
{
|
||||
details = "SELECT " + detailId + " AS id, (SELECT count(*) FROM worm_audit_chain c WHERE POSITION(',' || CAST(" + detailId
|
||||
+ " AS varchar) || ',' IN ',' || c.detail_ids || ',') > 0) AS coverage" + scope + " WHERE " + time + " < ? OR " + time + " IS NULL";
|
||||
}
|
||||
reportCoverage(headers, "audit", cutoff, result);
|
||||
reportCoverage(details, "audit_detail", cutoff, result);
|
||||
coverageTable(result);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The coverage table must hold exactly the chain's own detail lists. A
|
||||
** mismatch would let the writer skip or re-chain a detail, so it is a break.
|
||||
*******************************************************************************/
|
||||
private void coverageTable(VerificationResult result) throws QException
|
||||
{
|
||||
String table = "SELECT detail_id, chain_id FROM " + WormAuditTables.CHAIN_DETAIL;
|
||||
String sql = isPostgres()
|
||||
? "SELECT detail_id, count(*) OVER() FROM ((" + CHAIN_PAIRS + " EXCEPT " + table + ") UNION ALL (" + table + " EXCEPT " + CHAIN_PAIRS
|
||||
+ ")) mismatch ORDER BY detail_id LIMIT 100"
|
||||
: "SELECT x.detail_id, count(*) OVER() FROM " + WormAuditTables.CHAIN_DETAIL + " x WHERE NOT EXISTS (SELECT 1 FROM worm_audit_chain c"
|
||||
+ " WHERE c.id=x.chain_id AND POSITION(',' || CAST(x.detail_id AS varchar) || ',' IN ',' || c.detail_ids || ',') > 0) ORDER BY x.detail_id LIMIT 100";
|
||||
try(PreparedStatement statement = ((RDBMSTransaction) transaction).getConnection().prepareStatement(sql); ResultSet rows = statement.executeQuery())
|
||||
{
|
||||
long total = 0;
|
||||
int reported = 0;
|
||||
while(rows.next())
|
||||
{
|
||||
total = rows.getLong(2);
|
||||
result.add(null, null, "audit_detail " + com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier.number(rows.getLong(1))
|
||||
+ " has a detail coverage row that disagrees with the chain");
|
||||
reported++;
|
||||
}
|
||||
result.omitted(total - reported);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw (AppendOnlyGuard.map(e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Highest id in a core audit table, or 0 when it is empty (RDBMS only).
|
||||
*******************************************************************************/
|
||||
public long maxId(String table) throws QException
|
||||
{
|
||||
return (sqlIds("SELECT coalesce(max(" + column(table, "id") + "), 0) FROM " + physical(table)).getFirst());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** PostgreSQL maintains detail coverage in its chain trigger; other backends
|
||||
** have no trigger, so the writer appends coverage rows itself.
|
||||
*******************************************************************************/
|
||||
public boolean isPostgres() throws QException
|
||||
{
|
||||
if(postgres == null)
|
||||
{
|
||||
try
|
||||
{
|
||||
postgres = isRdbms() && "PostgreSQL".equals(((RDBMSTransaction) transaction).getConnection().getMetaData().getDatabaseProductName());
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw (AppendOnlyGuard.map(e));
|
||||
}
|
||||
}
|
||||
return (postgres);
|
||||
}
|
||||
|
||||
|
||||
@@ -392,36 +565,22 @@ public class AuditStore
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** PostgreSQL's containment expression matches the GIN expression index.
|
||||
** Read ids from an anti-join, binding instants by column type and ids as longs.
|
||||
*******************************************************************************/
|
||||
private String containsDetail() throws QException
|
||||
{
|
||||
try
|
||||
{
|
||||
String id = "d." + column("auditDetail", "id");
|
||||
if("PostgreSQL".equals(((RDBMSTransaction) transaction).getConnection().getMetaData().getDatabaseProductName()))
|
||||
{
|
||||
return ("string_to_array(c.detail_ids, ',') @> ARRAY[" + id + "::text]");
|
||||
}
|
||||
return ("POSITION(',' || CAST(" + id + " AS varchar) || ',' IN ',' || c.detail_ids || ',') > 0");
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw (AppendOnlyGuard.map(e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read ids from an anti-join, binding all instants according to column type.
|
||||
*******************************************************************************/
|
||||
private List<Long> sqlIds(String sql, Instant... cutoffs) throws QException
|
||||
private List<Long> sqlIds(String sql, Object... parameters) throws QException
|
||||
{
|
||||
try(PreparedStatement statement = ((RDBMSTransaction) transaction).getConnection().prepareStatement(sql))
|
||||
{
|
||||
for(int i = 0; i < cutoffs.length; i++)
|
||||
for(int i = 0; i < parameters.length; i++)
|
||||
{
|
||||
bindCutoff(statement, i + 1, cutoffs[i]);
|
||||
if(parameters[i] instanceof Instant instant)
|
||||
{
|
||||
bindCutoff(statement, i + 1, instant);
|
||||
}
|
||||
else
|
||||
{
|
||||
statement.setLong(i + 1, (Long) parameters[i]);
|
||||
}
|
||||
}
|
||||
List<Long> ids = new ArrayList<>();
|
||||
try(ResultSet rows = statement.executeQuery())
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.chain;
|
||||
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.Collections;
|
||||
import java.util.Deque;
|
||||
import java.util.Map;
|
||||
import java.util.WeakHashMap;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Per-JVM id floors for the chain writer's two sweeps, so a tick reads only
|
||||
** audit and detail rows newer than the last complete sweep instead of the whole
|
||||
** settle window or late-detail lookback.
|
||||
**
|
||||
** Each tick records the highest audit and audit_detail ids it can see. After a
|
||||
** sweep that returned fewer rows than its limit, every row allocated before
|
||||
** (settle horizon + MARGIN) ago is chained, on the same assumption the settle
|
||||
** horizon already makes: a row whose id was allocated that long ago has
|
||||
** committed. The floor moves to the highest id observed at or before that time.
|
||||
** A row whose transaction commits later than that is still chained: once an
|
||||
** hour (and after every restart) a tick ignores the floors and sweeps
|
||||
** everything, exactly as before floors existed. Floors live in memory only; a
|
||||
** second replica keeps its own. The daily verification checks full coverage
|
||||
** independently of all of this.
|
||||
*******************************************************************************/
|
||||
public final class SweepFloors
|
||||
{
|
||||
public static final Duration MARGIN = Duration.ofMinutes(5);
|
||||
public static final Duration FULL_SWEEP_EVERY = Duration.ofHours(1);
|
||||
|
||||
private static final Map<WormAuditQBitConfig, SweepFloors> BY_CONFIG = Collections.synchronizedMap(new WeakHashMap<>());
|
||||
|
||||
private final Deque<Observation> observations = new ArrayDeque<>();
|
||||
private long audit;
|
||||
private long detail;
|
||||
private Instant lastFullSweep;
|
||||
|
||||
/*******************************************************************************
|
||||
** The highest ids visible at one tick.
|
||||
*******************************************************************************/
|
||||
public record Observation(Instant at, long audit, long detail)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Floors for one installed qbit (config identity), created on first use.
|
||||
*******************************************************************************/
|
||||
public static SweepFloors of(WormAuditQBitConfig config)
|
||||
{
|
||||
return (BY_CONFIG.computeIfAbsent(config, c -> new SweepFloors()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Audit ids at or below this are known chained.
|
||||
*******************************************************************************/
|
||||
public synchronized long audit()
|
||||
{
|
||||
return (audit);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Detail ids at or below this are known covered (or outside the lookback).
|
||||
*******************************************************************************/
|
||||
public synchronized long detail()
|
||||
{
|
||||
return (detail);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether this tick must ignore the floors: first tick, an hour since the last
|
||||
** complete full sweep, or a clock that moved backwards.
|
||||
*******************************************************************************/
|
||||
public synchronized boolean fullSweepDue(Instant now)
|
||||
{
|
||||
return (lastFullSweep == null || now.isBefore(lastFullSweep) || !now.isBefore(lastFullSweep.plus(FULL_SWEEP_EVERY)));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Record the highest visible ids. Ids below a floor mean rows were removed
|
||||
** wholesale (a purge of everything, or a reset test database): start over.
|
||||
*******************************************************************************/
|
||||
public synchronized Observation observe(Instant now, long maxAudit, long maxDetail)
|
||||
{
|
||||
if(maxAudit < audit || maxDetail < detail)
|
||||
{
|
||||
observations.clear();
|
||||
audit = 0;
|
||||
detail = 0;
|
||||
}
|
||||
Observation observation = new Observation(now, maxAudit, maxDetail);
|
||||
observations.addLast(observation);
|
||||
return (observation);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** After a tick, raise each completely swept floor to the newest observation
|
||||
** made at or before the horizon, then drop observations no longer needed. A
|
||||
** full sweep (fullSweepAt not null) counts only when both sweeps completed.
|
||||
*******************************************************************************/
|
||||
public synchronized void swept(Observation current, Instant horizon, boolean auditsComplete, boolean detailsComplete, Instant fullSweepAt)
|
||||
{
|
||||
if(fullSweepAt != null && auditsComplete && detailsComplete)
|
||||
{
|
||||
lastFullSweep = fullSweepAt;
|
||||
}
|
||||
Observation settled = null;
|
||||
for(Observation observation : observations)
|
||||
{
|
||||
if(!observation.at().isAfter(horizon) && (settled == null || !observation.at().isBefore(settled.at())))
|
||||
{
|
||||
settled = observation;
|
||||
}
|
||||
}
|
||||
if(settled == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
if(auditsComplete)
|
||||
{
|
||||
audit = Math.max(audit, Math.min(settled.audit(), current.audit()));
|
||||
}
|
||||
if(detailsComplete)
|
||||
{
|
||||
detail = Math.max(detail, Math.min(settled.detail(), current.detail()));
|
||||
}
|
||||
Observation keep = settled;
|
||||
observations.removeIf(observation -> observation != keep && !observation.at().isAfter(horizon));
|
||||
}
|
||||
}
|
||||
@@ -31,6 +31,7 @@ import java.util.TreeMap;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import java.util.stream.Collectors;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditSettings;
|
||||
import com.kingsrook.qbits.wormaudit.checkpoint.Checkpoints;
|
||||
import com.kingsrook.qbits.wormaudit.guard.AppendOnlyGuard;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
@@ -99,7 +100,7 @@ public class WormAuditChainer
|
||||
}
|
||||
}
|
||||
}
|
||||
Counts counts = append(new AuditStore(transaction), now);
|
||||
Counts counts = append(new AuditStore(transaction), now, config.effectiveSettings());
|
||||
transaction.commit();
|
||||
return (counts);
|
||||
}
|
||||
@@ -119,15 +120,18 @@ public class WormAuditChainer
|
||||
/*******************************************************************************
|
||||
** Append settled headers and uncovered detail chunks, then checkpoint.
|
||||
*******************************************************************************/
|
||||
private Counts append(AuditStore store, Instant now) throws Exception
|
||||
private Counts append(AuditStore store, Instant now, WormAuditSettings.Values settings) throws Exception
|
||||
{
|
||||
QRecord tail = store.last(WormAuditTables.CHAIN);
|
||||
QRecord anchor = store.anchor();
|
||||
long nextId = tail == null ? anchor.getValueLong("anchor_link_id") + 1 : tail.getValueLong("id") + 1;
|
||||
String previous = tail == null ? anchor.getValueString("anchor_hash") : tail.getValueString("link_hash");
|
||||
List<QRecord> appended = new ArrayList<>();
|
||||
Instant cutoff = now.minusSeconds(config.getSettleHorizonSeconds());
|
||||
List<QRecord> audits = store.unchained(cutoff, config.getBatchSize());
|
||||
Instant cutoff = now.minusSeconds(settings.settle_horizon_seconds());
|
||||
SweepFloors floors = SweepFloors.of(config);
|
||||
SweepFloors.Observation observed = store.isRdbms() ? floors.observe(now, store.maxId("audit"), store.maxId("auditDetail")) : null;
|
||||
boolean full = floors.fullSweepDue(now);
|
||||
List<QRecord> audits = store.unchained(cutoff, config.getBatchSize(), full ? 0L : floors.audit());
|
||||
Map<Long, QRecord> details = AuditStore.index(store.in("auditDetail", "auditId", audits.stream().map(r -> r.getValueLong("id")).toList()));
|
||||
Map<Long, List<QRecord>> grouped = details.values().stream().collect(Collectors.groupingBy(r -> r.getValueLong("auditId")));
|
||||
int lateDetails = 0;
|
||||
@@ -149,7 +153,7 @@ public class WormAuditChainer
|
||||
insertLinks(store, appended);
|
||||
int linkCount = appended.size();
|
||||
appended.clear();
|
||||
details = AuditStore.index(store.lateDetails(cutoff, config.getLateDetailLookbackHours(), config.getBatchSize()));
|
||||
details = AuditStore.index(store.lateDetails(cutoff, settings.late_detail_lookback_hours(), config.getBatchSize(), full ? 0L : floors.detail()));
|
||||
grouped = details.values().stream().collect(Collectors.groupingBy(r -> r.getValueLong("auditId"), TreeMap::new, Collectors.toList()));
|
||||
Map<Long, QRecord> headers = AuditStore.index(store.in("audit", "id", grouped.keySet()));
|
||||
for(var entry : grouped.entrySet())
|
||||
@@ -165,12 +169,19 @@ public class WormAuditChainer
|
||||
}
|
||||
}
|
||||
insertLinks(store, appended);
|
||||
return (new Counts(audits.size(), lateDetails, linkCount + appended.size(), checkpoint(store, now), false));
|
||||
if(observed != null)
|
||||
{
|
||||
floors.swept(observed, now.minusSeconds(settings.settle_horizon_seconds()).minus(SweepFloors.MARGIN),
|
||||
audits.size() < config.getBatchSize(), details.size() < config.getBatchSize(), full ? now : null);
|
||||
}
|
||||
return (new Counts(audits.size(), lateDetails, linkCount + appended.size(), checkpoint(store, now, settings), false));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Persist bounded insert batches inside the writer transaction.
|
||||
** Persist bounded insert batches inside the writer transaction. PostgreSQL's
|
||||
** chain trigger writes the detail coverage rows; elsewhere the writer does,
|
||||
** in the same transaction, so coverage never diverges from the chain.
|
||||
*******************************************************************************/
|
||||
private void insertLinks(AuditStore store, List<QRecord> links) throws QException
|
||||
{
|
||||
@@ -178,6 +189,18 @@ public class WormAuditChainer
|
||||
{
|
||||
store.insert(WormAuditTables.CHAIN, links.subList(start, Math.min(start + config.getBatchSize(), links.size())));
|
||||
}
|
||||
if(!store.isPostgres())
|
||||
{
|
||||
List<QRecord> coverage = new ArrayList<>();
|
||||
for(QRecord link : links)
|
||||
{
|
||||
AuditStore.detailIds(link).forEach(id -> coverage.add(new QRecord().withValue("detail_id", id).withValue("chain_id", link.getValueLong("id"))));
|
||||
}
|
||||
for(int start = 0; start < coverage.size(); start += config.getBatchSize())
|
||||
{
|
||||
store.insert(WormAuditTables.CHAIN_DETAIL, coverage.subList(start, Math.min(start + config.getBatchSize(), coverage.size())));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -195,7 +218,7 @@ public class WormAuditChainer
|
||||
/*******************************************************************************
|
||||
** Checkpoint the uncheckpointed suffix by count or elapsed time.
|
||||
*******************************************************************************/
|
||||
private int checkpoint(AuditStore store, Instant now) throws Exception
|
||||
private int checkpoint(AuditStore store, Instant now, WormAuditSettings.Values settings) throws Exception
|
||||
{
|
||||
if(config.getSigningKey() == null)
|
||||
{
|
||||
@@ -206,13 +229,13 @@ public class WormAuditChainer
|
||||
int count = 0;
|
||||
while(true)
|
||||
{
|
||||
List<QRecord> pending = store.page(WormAuditTables.CHAIN, lastId, config.getCheckpointEveryLinks());
|
||||
List<QRecord> pending = store.page(WormAuditTables.CHAIN, lastId, settings.checkpoint_every_links());
|
||||
if(pending.isEmpty())
|
||||
{
|
||||
return (count);
|
||||
}
|
||||
Instant since = latest == null ? pending.getFirst().getValueInstant("chained_at") : latest.getValueInstant("created_at");
|
||||
if(pending.size() < config.getCheckpointEveryLinks() && now.isBefore(since.plusSeconds(config.getCheckpointEveryMinutes() * 60L)))
|
||||
if(pending.size() < settings.checkpoint_every_links() && now.isBefore(since.plusSeconds(settings.checkpoint_every_minutes() * 60L)))
|
||||
{
|
||||
return (count);
|
||||
}
|
||||
|
||||
@@ -102,6 +102,13 @@ public final class WormAuditDdl
|
||||
{
|
||||
sql = sql.replace("${" + entry.getValue() + "}", identifier(names.get(entry.getKey())));
|
||||
}
|
||||
var defaults = com.kingsrook.qbits.wormaudit.WormAuditSettings.defaults(config);
|
||||
com.kingsrook.qbits.wormaudit.WormAuditSettings.validate(defaults);
|
||||
String values = java.util.List.of("id", "audit_retention_years", "export_lock_mode", "export_retention_days", "checkpoint_every_links",
|
||||
"checkpoint_every_minutes", "chain_interval_seconds", "settle_horizon_seconds", "late_detail_lookback_hours", "export_cron", "verify_cron")
|
||||
.stream().map(field -> defaults.getValue(field) instanceof Number ? defaults.getValue(field).toString()
|
||||
: "'" + defaults.getValueString(field).replace("'", "''") + "'").collect(java.util.stream.Collectors.joining(", "));
|
||||
sql = sql.replace("${settings_values}", values);
|
||||
return (sql.replace("${app_role}", identifier(config.getAppRole())).replace("${retention}", config.getRetentionYears() + " years"));
|
||||
}
|
||||
catch(IOException e)
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.export;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.Serializable;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.security.MessageDigest;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HexFormat;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.fasterxml.jackson.core.JsonParser;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.kingsrook.qbits.objectstorage.api.StoredObject;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Version-one export encoding and temporary files. JSON errors never echo input.
|
||||
*******************************************************************************/
|
||||
public class ExportFiles implements AutoCloseable
|
||||
{
|
||||
public static final List<String> NAMES = List.of("links.jsonl", "checkpoints.jsonl", "manifest.json");
|
||||
private static final ObjectMapper JSON = new ObjectMapper().enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION)
|
||||
.enable(DeserializationFeature.USE_LONG_FOR_INTS).enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
|
||||
private final Path directory;
|
||||
|
||||
/*******************************************************************************
|
||||
** Create a private temporary directory, removed on all normal failure paths.
|
||||
*******************************************************************************/
|
||||
public ExportFiles() throws IOException
|
||||
{
|
||||
directory = Files.createTempDirectory("worm-audit-");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve only one of the fixed filenames, never a manifest-supplied path.
|
||||
*******************************************************************************/
|
||||
public Path file(String name)
|
||||
{
|
||||
if(!NAMES.contains(name))
|
||||
{
|
||||
throw (new IllegalArgumentException("Unknown export file"));
|
||||
}
|
||||
return (directory.resolve(name));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Parse the JCS schema with strict duplicate and trailing-token rejection.
|
||||
*******************************************************************************/
|
||||
public static Map<String, Object> parse(byte[] bytes) throws IOException
|
||||
{
|
||||
try
|
||||
{
|
||||
return (JSON.readValue(bytes, new TypeReference<Map<String, Object>>() {}));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw (new IOException("Invalid sealed JSON"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Wire representation of a chain row, excluding database-only timestamps.
|
||||
*******************************************************************************/
|
||||
public static Map<String, Object> link(QRecord row)
|
||||
{
|
||||
return (Map.of("id", row.getValueLong("id"), "kind", row.getValueString("kind"),
|
||||
"auditId", row.getValueLong("audit_id"), "detailIds", AuditStore.detailIds(row),
|
||||
"prevHash", row.getValueString("prev_hash"), "linkHash", row.getValueString("link_hash")));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Preserve every checkpoint field, representing the timestamp as UTC text.
|
||||
*******************************************************************************/
|
||||
public static Map<String, Object> checkpoint(QRecord row)
|
||||
{
|
||||
Map<String, Object> fields = new LinkedHashMap<>();
|
||||
for(String key : List.of("id", "from_link_id", "to_link_id", "link_count"))
|
||||
{
|
||||
fields.put(key, row.getValueLong(key));
|
||||
}
|
||||
for(String key : List.of("head_hash", "merkle_root", "algorithm", "key_id", "public_key", "signature"))
|
||||
{
|
||||
fields.put(key, row.getValueString(key));
|
||||
}
|
||||
fields.put("created_at", row.getValueInstant("created_at").toString());
|
||||
return (fields);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Convert a checkpoint envelope back to the phase (a) signature input.
|
||||
*******************************************************************************/
|
||||
public static QRecord record(Map<String, Object> fields)
|
||||
{
|
||||
QRecord row = new QRecord();
|
||||
fields.forEach((key, value) -> row.setValue(key, (Serializable) value));
|
||||
return (row);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Compute a digest without materializing a file.
|
||||
*******************************************************************************/
|
||||
public static String sha256(Path path) throws Exception
|
||||
{
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
try(InputStream input = Files.newInputStream(path))
|
||||
{
|
||||
byte[] buffer = new byte[65536];
|
||||
int read;
|
||||
while((read = input.read(buffer)) != -1)
|
||||
{
|
||||
digest.update(buffer, 0, read);
|
||||
}
|
||||
}
|
||||
return (HexFormat.of().formatHex(digest.digest()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A lock echo is required even if a provider advertises the capability.
|
||||
*******************************************************************************/
|
||||
public static boolean locked(StoredObject object, String mode, java.time.Instant until)
|
||||
{
|
||||
return (object != null && java.util.Objects.equals(mode, object.getLockMode()) && object.getLockRetainUntil() != null
|
||||
&& !object.getLockRetainUntil().isBefore(until));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A null version belongs to unversioned storage and cannot be sealed.
|
||||
*******************************************************************************/
|
||||
public static boolean validVersion(String versionId)
|
||||
{
|
||||
return (versionId != null && !versionId.isBlank() && !"null".equals(versionId));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fetch covered detail rows in bounded IN batches.
|
||||
*******************************************************************************/
|
||||
public static Map<Long, QRecord> details(AuditStore store, List<QRecord> links) throws Exception
|
||||
{
|
||||
List<Long> ids = new ArrayList<>();
|
||||
for(QRecord link : links)
|
||||
{
|
||||
List<Long> covered = AuditStore.detailIds(link);
|
||||
if(covered.size() > 250)
|
||||
{
|
||||
throw (new IOException("Invalid detail coverage"));
|
||||
}
|
||||
ids.addAll(covered);
|
||||
}
|
||||
Map<Long, QRecord> rows = new LinkedHashMap<>();
|
||||
for(int start = 0; start < ids.size(); start += 1000)
|
||||
{
|
||||
rows.putAll(AuditStore.index(store.in("auditDetail", "id", ids.subList(start, Math.min(start + 1000, ids.size())))));
|
||||
}
|
||||
return (rows);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Remove only the three owned files and their directory.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void close() throws IOException
|
||||
{
|
||||
IOException failure = null;
|
||||
for(String name : NAMES)
|
||||
{
|
||||
try
|
||||
{
|
||||
Files.deleteIfExists(file(name));
|
||||
}
|
||||
catch(IOException e)
|
||||
{
|
||||
failure = new IOException("Cannot remove temporary export files");
|
||||
}
|
||||
}
|
||||
Files.deleteIfExists(directory);
|
||||
if(failure != null)
|
||||
{
|
||||
throw (failure);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.export;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Instant;
|
||||
import com.kingsrook.qbits.objectstorage.api.Capability;
|
||||
import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
|
||||
import com.kingsrook.qbits.objectstorage.api.ListRequest;
|
||||
import com.kingsrook.qbits.objectstorage.api.ObjectStorage;
|
||||
import com.kingsrook.qbits.objectstorage.api.ObjectSummary;
|
||||
import com.kingsrook.qbits.objectstorage.api.Page;
|
||||
import com.kingsrook.qbits.objectstorage.api.PutRequest;
|
||||
import com.kingsrook.qbits.objectstorage.api.StorageBucket;
|
||||
import com.kingsrook.qbits.objectstorage.api.StorageKeys;
|
||||
import com.kingsrook.qbits.objectstorage.api.StoredObject;
|
||||
import com.kingsrook.qbits.objectstorage.config.BucketRole;
|
||||
import com.kingsrook.qbits.objectstorage.internal.ObjectStorageRuntime;
|
||||
import com.kingsrook.qbits.objectstorage.internal.SecretRefs;
|
||||
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Thin system-caller adapter to qbit-object-storage.
|
||||
*******************************************************************************/
|
||||
public class ObjectStorageSealedStore implements SealedStore
|
||||
{
|
||||
private final StorageBucket bucket;
|
||||
|
||||
/*******************************************************************************
|
||||
** Bind the recorded alias, so old exports survive configuration changes.
|
||||
*******************************************************************************/
|
||||
public ObjectStorageSealedStore(String alias, String runId)
|
||||
{
|
||||
bucket = ObjectStorage.asSystem("worm_audit", runId).bucket(alias);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Refuse unknown, partial, or unsupported lock capabilities.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean supportsLock()
|
||||
{
|
||||
return (bucket.config().getRole() == BucketRole.BACKUP
|
||||
&& bucket.capability(Capability.OBJECT_LOCK) == CapabilityStatus.SUPPORTED);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Report the provider capability without collapsing UNKNOWN into UNSUPPORTED.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public CapabilityStatus lockCapability()
|
||||
{
|
||||
return (bucket.capability(Capability.OBJECT_LOCK));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stream a bounded upload and close the source even when the provider fails.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String put(String key, Path file, com.kingsrook.qbits.objectstorage.api.LockMode mode, Instant retainUntil) throws Exception
|
||||
{
|
||||
try(InputStream input = Files.newInputStream(file))
|
||||
{
|
||||
return (bucket.put(PutRequest.of(key, input).withContentLength(Files.size(file))
|
||||
.withObjectLock(mode, retainUntil)).getVersionId());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read retention metadata.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public StoredObject head(String key)
|
||||
{
|
||||
return (bucket.head(key).orElse(null));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** RC.2 has no version GET facade. Reuse its configured system connection,
|
||||
** key normalization and limiter; closing the stream closes the S3 response.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public InputStream getVersion(String key, String versionId) throws IOException
|
||||
{
|
||||
if(!ExportFiles.validVersion(versionId))
|
||||
{
|
||||
throw (new IOException("Missing sealed object version"));
|
||||
}
|
||||
var connection = ObjectStorageRuntime.connection(bucket.config().getConnection());
|
||||
try(var ignored = connection.getRateLimiter().acquire())
|
||||
{
|
||||
var input = connection.client().getObject(GetObjectRequest.builder()
|
||||
.bucket(SecretRefs.resolveSetting(bucket.config().getBucketName())).key(StorageKeys.normalize(key)).versionId(versionId).build());
|
||||
if(!versionId.equals(input.response().versionId()))
|
||||
{
|
||||
input.close();
|
||||
throw (new IOException("Storage returned a different object version"));
|
||||
}
|
||||
return (input);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Forward bounded listing.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public Page<ObjectSummary> list(ListRequest request)
|
||||
{
|
||||
return (bucket.list(request));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,424 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.export;
|
||||
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.sql.Connection;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.canonical.Jcs;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.chain.Hashes;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.checkpoint.Checkpoints;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.VerificationRuns;
|
||||
import com.kingsrook.qbits.wormaudit.verify.VerificationResult;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSTransaction;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify the sealed evidence before comparing it to a repeatable live snapshot.
|
||||
** No exported content or provider/parser exception text enters diagnostics.
|
||||
*******************************************************************************/
|
||||
public class SealedExportVerifier
|
||||
{
|
||||
private final WormAuditQBitConfig config;
|
||||
private final SealedStore suppliedStorage;
|
||||
|
||||
/*******************************************************************************
|
||||
** Findings plus the number verified without live rows after a retention purge.
|
||||
*******************************************************************************/
|
||||
public record Report(VerificationResult result, long purged)
|
||||
{
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Production entry point.
|
||||
*******************************************************************************/
|
||||
public SealedExportVerifier(WormAuditQBitConfig config)
|
||||
{
|
||||
this(config, null);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Testable storage boundary.
|
||||
*******************************************************************************/
|
||||
public SealedExportVerifier(WormAuditQBitConfig config, SealedStore storage)
|
||||
{
|
||||
this.config = config;
|
||||
this.suppliedStorage = storage;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Persist exactly one verification row for this selected export, even on failure.
|
||||
*******************************************************************************/
|
||||
public Report verify(QRecord export, String trigger) throws QException
|
||||
{
|
||||
config.effectiveSettings();
|
||||
VerificationResult failure = new VerificationResult();
|
||||
failure.scope(0, export.getValueLong("from_link_id"), export.getValueLong("to_link_id"));
|
||||
Report report;
|
||||
WormAuditChainer.JVM_LOCK.lock();
|
||||
try(QBackendTransaction transaction = QBackendTransaction.openFor(new QueryInput(WormAuditTables.CHAIN));
|
||||
ExportFiles files = new ExportFiles())
|
||||
{
|
||||
try
|
||||
{
|
||||
if(transaction instanceof RDBMSTransaction rdbms)
|
||||
{
|
||||
rdbms.getConnection().setTransactionIsolation(Connection.TRANSACTION_REPEATABLE_READ);
|
||||
}
|
||||
SealedStore storage = suppliedStorage == null
|
||||
? config.getSealedStoreSupplier() == null || !config.getExportBucketAlias().equals(export.getValueString("bucket_alias"))
|
||||
? new ObjectStorageSealedStore(export.getValueString("bucket_alias"), "verify-" + export.getValueLong("id"))
|
||||
: config.getSealedStoreSupplier().get() : suppliedStorage;
|
||||
Map<String, Object> versions = ExportFiles.parse(export.getValueString("object_versions").getBytes(StandardCharsets.UTF_8));
|
||||
for(String name : ExportFiles.NAMES)
|
||||
{
|
||||
String key = export.getValueString("object_prefix") + name;
|
||||
if(!(versions.get(name) instanceof String version) || !ExportFiles.validVersion(version))
|
||||
{
|
||||
failure.add(null, null, "object " + name + " has no recorded sealed version id");
|
||||
throw (new IOException("Missing sealed object version"));
|
||||
}
|
||||
var head = storage.head(key);
|
||||
if(head == null)
|
||||
{
|
||||
failure.add(null, null, "object " + key + " is missing" + ("GOVERNANCE".equals(export.getValueString("lock_mode"))
|
||||
? "; possible GOVERNANCE bypass removal by an administrator" : ""));
|
||||
}
|
||||
if(head != null && !version.equals(head.getVersionId()))
|
||||
{
|
||||
failure.add(null, null, "object " + name + " was overwritten after sealing (current version " + head.getVersionId()
|
||||
+ ", sealed version " + version + " is still retained under Object Lock)");
|
||||
}
|
||||
if(!ExportFiles.locked(head, export.getValueString("lock_mode"), export.getValueInstant("retain_until")))
|
||||
{
|
||||
failure.add(null, null, "object " + name + " lacks the recorded " + export.getValueString("lock_mode") + " retention");
|
||||
}
|
||||
}
|
||||
for(String name : ExportFiles.NAMES)
|
||||
{
|
||||
String key = export.getValueString("object_prefix") + name;
|
||||
try(InputStream input = storage.getVersion(key, (String) versions.get(name)))
|
||||
{
|
||||
Files.copy(input, files.file(name));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
failure.add(null, null, "object " + key + " sealed version cannot be read" + ("GOVERNANCE".equals(export.getValueString("lock_mode"))
|
||||
? "; if removed, possible GOVERNANCE bypass removal by an administrator" : ""));
|
||||
throw (new IOException("Cannot read sealed object version"));
|
||||
}
|
||||
}
|
||||
report = inspect(files, export, new AuditStore(transaction));
|
||||
for(var item : failure.getBreaks())
|
||||
{
|
||||
report.result().add(item.linkId(), item.auditId(), item.reason());
|
||||
}
|
||||
transaction.commit();
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
transaction.rollback();
|
||||
throw (e);
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
failure.add(null, null, "Sealed export cannot be read or has invalid structure");
|
||||
report = new Report(failure, 0);
|
||||
}
|
||||
finally
|
||||
{
|
||||
WormAuditChainer.JVM_LOCK.unlock();
|
||||
}
|
||||
VerificationRuns.save(report.result(), trigger, export.getValueLong("id"));
|
||||
return (report);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Inspect local files, also used by the exporter before committing any objects.
|
||||
*******************************************************************************/
|
||||
public Report inspect(ExportFiles files, QRecord export, AuditStore store) throws Exception
|
||||
{
|
||||
VerificationResult result = new VerificationResult();
|
||||
long from = export.getValueLong("from_link_id");
|
||||
long to = export.getValueLong("to_link_id");
|
||||
result.scope(0, from, to);
|
||||
if(!Objects.equals(ExportFiles.sha256(files.file("manifest.json")), export.getValueString("manifest_sha256")))
|
||||
{
|
||||
result.add(null, null, "object manifest.json hash does not match the export row");
|
||||
return (new Report(result, 0));
|
||||
}
|
||||
if(Files.size(files.file("manifest.json")) > 65536)
|
||||
{
|
||||
result.add(null, null, "Manifest exceeds the format size limit");
|
||||
return (new Report(result, 0));
|
||||
}
|
||||
Map<String, Object> manifest = ExportFiles.parse(Files.readAllBytes(files.file("manifest.json")));
|
||||
if(!"worm-audit-export/1".equals(manifest.get("format")) || from < 1 || to < from
|
||||
|| !Objects.equals(manifest.get("fromLinkId"), from) || !Objects.equals(manifest.get("toLinkId"), to)
|
||||
|| !Objects.equals(manifest.get("linkCount"), to - from + 1)
|
||||
|| !Objects.equals(manifest.get("createdAt"), export.getValueInstant("sealed_at").toString())
|
||||
|| !java.util.Set.of("GOVERNANCE", "COMPLIANCE").contains(export.getValueString("lock_mode"))
|
||||
|| !Objects.equals(export.getValueString("manifest_key"), export.getValueString("object_prefix") + "manifest.json"))
|
||||
{
|
||||
result.add(null, null, "Manifest metadata does not match the export row");
|
||||
}
|
||||
QQueryFilter previousFilter = new QQueryFilter().withOrderBy(new QFilterOrderBy("id", false)).withLimit(1);
|
||||
if(export.getValueLong("id") != null)
|
||||
{
|
||||
previousFilter.withCriteria(new QFilterCriteria("id", QCriteriaOperator.LESS_THAN, export.getValueLong("id")));
|
||||
}
|
||||
List<QRecord> previousRows = store.query(WormAuditTables.EXPORT, previousFilter);
|
||||
QRecord previous = previousRows.isEmpty() ? null : previousRows.getFirst();
|
||||
String previousHash = previous == null ? null : previous.getValueString("manifest_sha256");
|
||||
if(!Objects.equals(manifest.get("prevManifestSha256"), previousHash)
|
||||
|| !Objects.equals(export.getValueString("prev_manifest_sha256"), previousHash)
|
||||
|| (previous != null && previous.getValueLong("to_link_id") != from - 1))
|
||||
{
|
||||
result.add(null, null, "prevManifestSha256 does not match the previous export row");
|
||||
}
|
||||
Map<?, ?> hashes = (Map<?, ?>) manifest.get("files");
|
||||
for(String name : List.of("links.jsonl", "checkpoints.jsonl"))
|
||||
{
|
||||
if(!Objects.equals(ExportFiles.sha256(files.file(name)), hashes.get(name)))
|
||||
{
|
||||
result.add(null, null, "object " + name + " hash does not match the manifest");
|
||||
}
|
||||
}
|
||||
if(!result.passed())
|
||||
{
|
||||
return (new Report(result, 0));
|
||||
}
|
||||
return (checkLines(files, export, store, manifest, result));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Consume links alongside their checkpoint envelopes. Only one checkpoint's
|
||||
** hashes and one content page are retained, never the whole export.
|
||||
*******************************************************************************/
|
||||
private Report checkLines(ExportFiles files, QRecord export, AuditStore store, Map<String, Object> manifest, VerificationResult result) throws Exception
|
||||
{
|
||||
long from = export.getValueLong("from_link_id");
|
||||
long to = export.getValueLong("to_link_id");
|
||||
long cursor = from - 1;
|
||||
long checkpointCount = 0;
|
||||
long checked = 0;
|
||||
long purged = 0;
|
||||
long anchor = store.anchor().getValueLong("anchor_link_id");
|
||||
String previousHash = from == 1 ? Hashes.GENESIS : null;
|
||||
if(cursor == anchor)
|
||||
{
|
||||
previousHash = store.anchor().getValueString("anchor_hash");
|
||||
}
|
||||
long lastCheckpoint = 0;
|
||||
try(BufferedReader links = Files.newBufferedReader(files.file("links.jsonl"), StandardCharsets.UTF_8);
|
||||
BufferedReader checkpoints = Files.newBufferedReader(files.file("checkpoints.jsonl"), StandardCharsets.UTF_8))
|
||||
{
|
||||
String line;
|
||||
while((line = checkpoints.readLine()) != null)
|
||||
{
|
||||
QRecord checkpoint = ExportFiles.record(ExportFiles.parse(line.getBytes(StandardCharsets.UTF_8)));
|
||||
long checkpointId = checkpoint.getValueLong("id");
|
||||
String label = "checkpoint " + WormAuditVerifier.number(checkpointId) + ": ";
|
||||
long end = checkpoint.getValueLong("to_link_id");
|
||||
long start = checkpoint.getValueLong("from_link_id");
|
||||
long count = checkpoint.getValueLong("link_count");
|
||||
if(start != cursor + 1 || end < start || end > to || count != end - start + 1
|
||||
|| count > Integer.MAX_VALUE || checkpointId <= lastCheckpoint)
|
||||
{
|
||||
result.add(null, null, label + "invalid or incomplete checkpoint range");
|
||||
break;
|
||||
}
|
||||
if(!Checkpoints.verify(checkpoint, config.getTrustedPublicKeys().get(checkpoint.getValueString("key_id"))))
|
||||
{
|
||||
result.add(null, null, label + "signature does not verify");
|
||||
}
|
||||
if(checkpointCount == 0 && !Objects.equals(export.getValueLong("from_checkpoint_id"), checkpointId))
|
||||
{
|
||||
result.add(null, null, label + "first checkpoint does not match the export row");
|
||||
}
|
||||
List<String> hashes = new ArrayList<>();
|
||||
while(cursor < end)
|
||||
{
|
||||
List<Map<String, Object>> page = new ArrayList<>();
|
||||
for(int i = 0, size = (int) Math.min(1000, end - cursor); i < size; i++)
|
||||
{
|
||||
String linkLine = links.readLine();
|
||||
if(linkLine == null)
|
||||
{
|
||||
result.add(cursor + 1, null, "Sealed links are missing from the checkpoint range");
|
||||
result.scope(checked, from, to);
|
||||
return (new Report(result, purged));
|
||||
}
|
||||
page.add(ExportFiles.parse(linkLine.getBytes(StandardCharsets.UTF_8)));
|
||||
}
|
||||
compareLive(page, store, export, anchor, result);
|
||||
for(Map<String, Object> document : page)
|
||||
{
|
||||
Map<?, ?> link = (Map<?, ?>) document.get("link");
|
||||
long id = (Long) link.get("id");
|
||||
long auditId = (Long) link.get("auditId");
|
||||
String prefix = "link " + WormAuditVerifier.number(id) + " (audit " + WormAuditVerifier.number(auditId) + "): ";
|
||||
String prev = (String) link.get("prevHash");
|
||||
String hash = (String) link.get("linkHash");
|
||||
if(id != cursor + 1 || (previousHash != null && !Objects.equals(prev, previousHash)))
|
||||
{
|
||||
result.add(id, auditId, prefix + "sealed chain continuity differs");
|
||||
}
|
||||
Map<?, ?> content = (Map<?, ?>) document.get("content");
|
||||
Map<?, ?> audit = (Map<?, ?>) content.get("audit");
|
||||
List<?> detailIds = (List<?>) link.get("detailIds");
|
||||
List<?> details = (List<?>) content.get("details");
|
||||
List<?> contentIds = details.stream().map(d -> ((Map<?, ?>) d).get("id")).toList();
|
||||
if(!Objects.equals(audit.get("id"), auditId) || !Objects.equals(content.get("v"), 1L)
|
||||
|| !List.of("AUDIT", "LATE_DETAIL").contains(link.get("kind"))
|
||||
|| detailIds.size() > 250 || !detailIds.equals(contentIds)
|
||||
|| detailIds.stream().distinct().count() != detailIds.size()
|
||||
|| ("LATE_DETAIL".equals(link.get("kind")) && detailIds.isEmpty()))
|
||||
{
|
||||
result.add(id, auditId, prefix + "sealed content identifiers or coverage differ");
|
||||
}
|
||||
if(!Hashes.link(prev, Jcs.bytes(content)).equals(hash))
|
||||
{
|
||||
result.add(id, auditId, prefix + "sealed content differs from the chained hash");
|
||||
}
|
||||
hashes.add(hash);
|
||||
previousHash = hash;
|
||||
cursor++;
|
||||
checked++;
|
||||
if(id <= anchor)
|
||||
{
|
||||
purged++;
|
||||
}
|
||||
}
|
||||
}
|
||||
if(!Objects.equals(Hashes.merkle(hashes), checkpoint.getValueString("merkle_root"))
|
||||
|| !Objects.equals(previousHash, checkpoint.getValueString("head_hash")))
|
||||
{
|
||||
result.add(null, null, label + "Merkle root or head hash differs");
|
||||
}
|
||||
lastCheckpoint = checkpointId;
|
||||
checkpointCount++;
|
||||
}
|
||||
if(cursor != to || links.readLine() != null || !Objects.equals(manifest.get("checkpointCount"), checkpointCount)
|
||||
|| !Objects.equals(export.getValueLong("to_checkpoint_id"), lastCheckpoint)
|
||||
|| !Objects.equals(manifest.get("headHash"), previousHash))
|
||||
{
|
||||
result.add(null, null, "Sealed range, counts or head hash differ from the manifest");
|
||||
}
|
||||
}
|
||||
result.scope(checked, from, to);
|
||||
return (new Report(result, purged));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Compare a bounded page by exported ids, so missing live rows cannot vanish
|
||||
** from the verification scope. Purged ids are checked from signed copies only.
|
||||
*******************************************************************************/
|
||||
private void compareLive(List<Map<String, Object>> page, AuditStore store, QRecord export, long anchor, VerificationResult result) throws Exception
|
||||
{
|
||||
List<QRecord> sealed = page.stream().map(document ->
|
||||
{
|
||||
Map<?, ?> link = (Map<?, ?>) document.get("link");
|
||||
List<?> ids = (List<?>) link.get("detailIds");
|
||||
return (new QRecord().withValue("id", (Long) link.get("id")).withValue("audit_id", (Long) link.get("auditId"))
|
||||
.withValue("kind", (String) link.get("kind")).withValue("prev_hash", (String) link.get("prevHash"))
|
||||
.withValue("link_hash", (String) link.get("linkHash"))
|
||||
.withValue("detail_ids", ids.stream().map(Object::toString).collect(java.util.stream.Collectors.joining(","))));
|
||||
}).toList();
|
||||
List<QRecord> liveScope = sealed.stream().filter(r -> r.getValueLong("id") > anchor).toList();
|
||||
Map<Long, QRecord> live = AuditStore.index(store.in(WormAuditTables.CHAIN, "id", liveScope.stream().map(r -> r.getValueLong("id")).toList()));
|
||||
Map<Long, QRecord> audits = AuditStore.index(store.in("audit", "id", liveScope.stream().map(r -> r.getValueLong("audit_id")).toList()));
|
||||
Map<Long, QRecord> details = ExportFiles.details(store, liveScope);
|
||||
for(int i = 0; i < sealed.size(); i++)
|
||||
{
|
||||
QRecord link = sealed.get(i);
|
||||
long id = link.getValueLong("id");
|
||||
long auditId = link.getValueLong("audit_id");
|
||||
if(id <= anchor)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
String prefix = "link " + WormAuditVerifier.number(id) + " (audit " + WormAuditVerifier.number(auditId) + "): ";
|
||||
QRecord current = live.get(id);
|
||||
if(current == null)
|
||||
{
|
||||
result.add(id, auditId, "link " + WormAuditVerifier.number(id) + " is missing from the live chain");
|
||||
}
|
||||
else
|
||||
{
|
||||
for(String field : List.of("kind", "audit_id", "detail_ids", "prev_hash", "link_hash"))
|
||||
{
|
||||
if(!Objects.equals(link.getValue(field), current.getValue(field)))
|
||||
{
|
||||
result.add(id, auditId, prefix + "live chain " + field + " differs from the sealed copy");
|
||||
}
|
||||
}
|
||||
}
|
||||
try
|
||||
{
|
||||
QRecord audit = audits.get(auditId);
|
||||
if(audit == null)
|
||||
{
|
||||
result.add(id, auditId, prefix + "live audit header is missing");
|
||||
}
|
||||
else if(!Arrays.equals(store.canonical(audit, AuditStore.detailIds(link), details), Jcs.bytes(page.get(i).get("content"))))
|
||||
{
|
||||
result.add(id, auditId, prefix + "live content differs from the sealed copy (sealed "
|
||||
+ export.getValueInstant("sealed_at").atZone(ZoneOffset.UTC).toLocalDate() + ")");
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
result.add(id, auditId, prefix + "live content is missing or invalid");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,392 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.export;
|
||||
|
||||
|
||||
import java.io.BufferedWriter;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.Statement;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitProducer;
|
||||
import com.kingsrook.qbits.wormaudit.canonical.Jcs;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.chain.Hashes;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertSignal;
|
||||
import com.kingsrook.qqq.backend.core.alerts.QAlerts;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSTransaction;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Seal the checkpointed suffix under the chainer/purger lock. Only successful
|
||||
** uploads with verified lock echoes produce an export row.
|
||||
*******************************************************************************/
|
||||
public class SealedExporter
|
||||
{
|
||||
private final WormAuditQBitConfig config;
|
||||
private final SealedStore suppliedStorage;
|
||||
|
||||
/*******************************************************************************
|
||||
** A review scope; execution always recomputes it under the writer lock.
|
||||
*******************************************************************************/
|
||||
public record Scope(long from, long to, long checkpoints, QRecord previous)
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Number of links, including the empty checkpoint suffix.
|
||||
*******************************************************************************/
|
||||
public long count()
|
||||
{
|
||||
return (Math.max(0, to - from + 1));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Value-free Review text.
|
||||
*******************************************************************************/
|
||||
public String review()
|
||||
{
|
||||
return (count() == 0 ? "Nothing to seal: no new checkpoint" : WormAuditVerifier.number(count())
|
||||
+ " chain links will be sealed (links " + WormAuditVerifier.number(from) + "–" + WormAuditVerifier.number(to)
|
||||
+ ", " + WormAuditVerifier.number(checkpoints) + " checkpoints)");
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** A completed, empty, or failed run suitable for a Result line.
|
||||
*******************************************************************************/
|
||||
public record Result(QRecord row, String message, boolean failed)
|
||||
{
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Production storage is resolved lazily, so an empty run needs no bucket.
|
||||
*******************************************************************************/
|
||||
public SealedExporter(WormAuditQBitConfig config)
|
||||
{
|
||||
this(config, null);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Inject the tiny storage boundary for deterministic tests.
|
||||
*******************************************************************************/
|
||||
public SealedExporter(WormAuditQBitConfig config, SealedStore storage)
|
||||
{
|
||||
this.config = config;
|
||||
this.suppliedStorage = storage;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Select the contiguous suffix, never silently skipping an unexported purge.
|
||||
*******************************************************************************/
|
||||
public static Scope scope(AuditStore store) throws QException
|
||||
{
|
||||
QRecord previous = store.last(WormAuditTables.EXPORT);
|
||||
long anchor = store.anchor().getValueLong("anchor_link_id");
|
||||
long after = previous == null ? anchor : previous.getValueLong("to_link_id");
|
||||
if(after < anchor)
|
||||
{
|
||||
throw (new QException("Unexported links were purged"));
|
||||
}
|
||||
QRecord checkpoint = store.last(WormAuditTables.CHECKPOINT);
|
||||
long to = checkpoint == null ? after : Math.max(after, checkpoint.getValueLong("to_link_id"));
|
||||
long count = 0;
|
||||
long cursor = 0;
|
||||
while(true)
|
||||
{
|
||||
List<QRecord> page = checkpoints(store, after + 1, to, cursor);
|
||||
if(page.isEmpty())
|
||||
{
|
||||
break;
|
||||
}
|
||||
count += page.size();
|
||||
cursor = page.getLast().getValueLong("id");
|
||||
}
|
||||
return (new Scope(after + 1, to, count, previous));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Select checkpoint envelopes wholly contained in the export range.
|
||||
*******************************************************************************/
|
||||
private static List<QRecord> checkpoints(AuditStore store, long from, long to, long after) throws QException
|
||||
{
|
||||
return (store.query(WormAuditTables.CHECKPOINT, new QQueryFilter(
|
||||
new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, after),
|
||||
new QFilterCriteria("from_link_id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, from),
|
||||
new QFilterCriteria("to_link_id", QCriteriaOperator.LESS_THAN_OR_EQUALS, to))
|
||||
.withOrderBy(new QFilterOrderBy("id", true)).withLimit(1000)));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Execute with alerts on user and scheduled failures; schedules always beat.
|
||||
*******************************************************************************/
|
||||
public Result run(String trigger, Instant now)
|
||||
{
|
||||
try
|
||||
{
|
||||
config.validate();
|
||||
Result result = lockedRun(now);
|
||||
QAlerts.resolve("wormaudit.export.failed", "export", "Sealed audit export completed");
|
||||
return (result);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
// All exposed failures originate here or use a fixed, value-free code.
|
||||
String reason = e instanceof ExportFailure ? e.getMessage() : "Sealed export failed: storage or database operation failed";
|
||||
QAlerts.raise(new AlertSignal("wormaudit.export.failed").withDedupKey("export").withTitle(reason));
|
||||
return (new Result(null, reason, true));
|
||||
}
|
||||
finally
|
||||
{
|
||||
if("SCHEDULE".equals(trigger))
|
||||
{
|
||||
QAlerts.heartbeat("wormaudit.export.daily");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Serialize with writers in this JVM and PostgreSQL writers on other hosts.
|
||||
*******************************************************************************/
|
||||
private Result lockedRun(Instant now) throws Exception
|
||||
{
|
||||
WormAuditChainer.JVM_LOCK.lock();
|
||||
try(QBackendTransaction transaction = QBackendTransaction.openFor(new QueryInput(WormAuditTables.CHAIN)))
|
||||
{
|
||||
try
|
||||
{
|
||||
if(transaction instanceof RDBMSTransaction rdbms && "PostgreSQL".equals(rdbms.getConnection().getMetaData().getDatabaseProductName()))
|
||||
{
|
||||
try(Statement statement = rdbms.getConnection().createStatement();
|
||||
ResultSet result = statement.executeQuery("SELECT pg_advisory_xact_lock(" + WormAuditChainer.ADVISORY_LOCK + ")"))
|
||||
{
|
||||
result.next();
|
||||
}
|
||||
}
|
||||
AuditStore store = new AuditStore(transaction);
|
||||
Scope scope = scope(store);
|
||||
Result result;
|
||||
if(scope.count() == 0)
|
||||
{
|
||||
result = new Result(null, scope.review(), false);
|
||||
}
|
||||
else
|
||||
{
|
||||
SealedStore storage = suppliedStorage == null
|
||||
? config.getSealedStoreSupplier() == null ? new ObjectStorageSealedStore(config.getExportBucketAlias(), java.util.UUID.randomUUID().toString())
|
||||
: config.getSealedStoreSupplier().get() : suppliedStorage;
|
||||
result = seal(store, storage, scope, now.truncatedTo(ChronoUnit.MILLIS));
|
||||
}
|
||||
transaction.commit();
|
||||
return (result);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
transaction.rollback();
|
||||
throw (e);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
WormAuditChainer.JVM_LOCK.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build locally before uploading. Partial remote uploads have no database row.
|
||||
*******************************************************************************/
|
||||
private Result seal(AuditStore store, SealedStore storage, Scope scope, Instant now) throws Exception
|
||||
{
|
||||
if(!storage.supportsLock())
|
||||
{
|
||||
throw (new ExportFailure("Export refused: bucket does not support Object Lock"));
|
||||
}
|
||||
var settings = config.effectiveSettings();
|
||||
var mode = com.kingsrook.qbits.objectstorage.api.LockMode.valueOf(settings.export_lock_mode());
|
||||
Instant until = now.plus(settings.export_retention_days(), ChronoUnit.DAYS);
|
||||
String prefix = config.getExportPrefix() + "/" + DateTimeFormatter.ofPattern("yyyy/MM/dd").withZone(ZoneOffset.UTC).format(now)
|
||||
+ "/" + scope.from() + "-" + scope.to() + "/";
|
||||
QRecord row;
|
||||
try(ExportFiles files = new ExportFiles())
|
||||
{
|
||||
String head = writeLinks(files, store, scope);
|
||||
long[] checkpointIds = writeCheckpoints(files, store, scope);
|
||||
Map<String, Object> manifest = new LinkedHashMap<>();
|
||||
manifest.put("format", "worm-audit-export/1");
|
||||
manifest.put("fromLinkId", scope.from());
|
||||
manifest.put("toLinkId", scope.to());
|
||||
manifest.put("linkCount", scope.count());
|
||||
manifest.put("checkpointCount", scope.checkpoints());
|
||||
manifest.put("files", Map.of("links.jsonl", ExportFiles.sha256(files.file("links.jsonl")),
|
||||
"checkpoints.jsonl", ExportFiles.sha256(files.file("checkpoints.jsonl"))));
|
||||
manifest.put("headHash", head);
|
||||
String previous = scope.previous() == null ? null : scope.previous().getValueString("manifest_sha256");
|
||||
manifest.put("prevManifestSha256", previous);
|
||||
manifest.put("qbitVersion", WormAuditQBitProducer.VERSION);
|
||||
manifest.put("createdAt", now.toString());
|
||||
Files.write(files.file("manifest.json"), Jcs.bytes(manifest));
|
||||
row = new QRecord().withValue("from_link_id", scope.from()).withValue("to_link_id", scope.to())
|
||||
.withValue("from_checkpoint_id", checkpointIds[0]).withValue("to_checkpoint_id", checkpointIds[1])
|
||||
.withValue("bucket_alias", config.getExportBucketAlias()).withValue("object_prefix", prefix)
|
||||
.withValue("manifest_key", prefix + "manifest.json").withValue("manifest_sha256", ExportFiles.sha256(files.file("manifest.json")))
|
||||
.withValue("prev_manifest_sha256", previous).withValue("object_count", 3L)
|
||||
.withValue("lock_mode", mode.name()).withValue("retain_until", until).withValue("sealed_at", now);
|
||||
if(!new SealedExportVerifier(config, storage).inspect(files, row, store).result().passed())
|
||||
{
|
||||
throw (new ExportFailure("Export refused: source does not match the signed chain"));
|
||||
}
|
||||
long bytes = 0;
|
||||
Map<String, String> versions = new LinkedHashMap<>();
|
||||
for(String name : ExportFiles.NAMES)
|
||||
{
|
||||
String version = storage.put(prefix + name, files.file(name), mode, until);
|
||||
if(!ExportFiles.validVersion(version))
|
||||
{
|
||||
throw (new ExportFailure("Export refused: object " + name + " returned no version id"));
|
||||
}
|
||||
var headObject = storage.head(prefix + name);
|
||||
if(!ExportFiles.locked(headObject, mode.name(), until))
|
||||
{
|
||||
throw (new ExportFailure("Export refused: object " + name + " did not echo " + mode.name() + " retention"));
|
||||
}
|
||||
if(!version.equals(headObject.getVersionId()))
|
||||
{
|
||||
throw (new ExportFailure("Export refused: object " + name + " version changed after upload"));
|
||||
}
|
||||
versions.put(name, version);
|
||||
bytes += Files.size(files.file(name));
|
||||
}
|
||||
row.withValue("byte_count", bytes).withValue("object_versions", Jcs.string(versions));
|
||||
}
|
||||
row = store.insert(WormAuditTables.EXPORT, List.of(row)).getFirst();
|
||||
return (new Result(row, WormAuditVerifier.number(scope.count()) + " links sealed in export " + row.getValueLong("id")
|
||||
+ " (retained until " + until + ")", false));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stream canonical lines in pages of 1000 with batched content lookups.
|
||||
*******************************************************************************/
|
||||
private String writeLinks(ExportFiles files, AuditStore store, Scope scope) throws Exception
|
||||
{
|
||||
long cursor = scope.from() - 1;
|
||||
String head = null;
|
||||
try(BufferedWriter writer = Files.newBufferedWriter(files.file("links.jsonl"), StandardCharsets.UTF_8))
|
||||
{
|
||||
while(cursor < scope.to())
|
||||
{
|
||||
List<QRecord> links = store.page(WormAuditTables.CHAIN, cursor, (int) Math.min(1000, scope.to() - cursor));
|
||||
if(links.isEmpty())
|
||||
{
|
||||
throw (new ExportFailure("Export refused: chain range is incomplete"));
|
||||
}
|
||||
Map<Long, QRecord> audits = AuditStore.index(store.in("audit", "id", links.stream().map(r -> r.getValueLong("audit_id")).toList()));
|
||||
Map<Long, QRecord> details = ExportFiles.details(store, links);
|
||||
for(QRecord link : links)
|
||||
{
|
||||
if(link.getValueLong("id") != cursor + 1 || link.getValueLong("id") > scope.to())
|
||||
{
|
||||
throw (new ExportFailure("Export refused: chain range is incomplete"));
|
||||
}
|
||||
byte[] content = store.canonical(audits.get(link.getValueLong("audit_id")), AuditStore.detailIds(link), details);
|
||||
if(!Hashes.link(link.getValueString("prev_hash"), content).equals(link.getValueString("link_hash")))
|
||||
{
|
||||
throw (new ExportFailure("Export refused: content differs at link " + link.getValueLong("id")));
|
||||
}
|
||||
writer.write(Jcs.string(Map.of("link", ExportFiles.link(link), "content", ExportFiles.parse(content))));
|
||||
writer.write('\n');
|
||||
head = link.getValueString("link_hash");
|
||||
cursor = link.getValueLong("id");
|
||||
}
|
||||
}
|
||||
}
|
||||
return (head);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stream complete signed checkpoint envelopes in keyset pages.
|
||||
*******************************************************************************/
|
||||
private long[] writeCheckpoints(ExportFiles files, AuditStore store, Scope scope) throws Exception
|
||||
{
|
||||
long first = 0;
|
||||
long cursor = 0;
|
||||
try(BufferedWriter writer = Files.newBufferedWriter(files.file("checkpoints.jsonl"), StandardCharsets.UTF_8))
|
||||
{
|
||||
while(true)
|
||||
{
|
||||
List<QRecord> page = checkpoints(store, scope.from(), scope.to(), cursor);
|
||||
if(page.isEmpty())
|
||||
{
|
||||
break;
|
||||
}
|
||||
for(QRecord row : page)
|
||||
{
|
||||
first = first == 0 ? row.getValueLong("id") : first;
|
||||
cursor = row.getValueLong("id");
|
||||
writer.write(Jcs.string(ExportFiles.checkpoint(row)));
|
||||
writer.write('\n');
|
||||
}
|
||||
}
|
||||
}
|
||||
return (new long[] { first, cursor });
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Only fixed messages constructed here may reach an alert or Result screen.
|
||||
*******************************************************************************/
|
||||
private static class ExportFailure extends IOException
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Carry a value-free exporter reason without a provider exception cause.
|
||||
*******************************************************************************/
|
||||
ExportFailure(String reason)
|
||||
{
|
||||
super(reason);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.export;
|
||||
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Instant;
|
||||
import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
|
||||
import com.kingsrook.qbits.objectstorage.api.ListRequest;
|
||||
import com.kingsrook.qbits.objectstorage.api.ObjectSummary;
|
||||
import com.kingsrook.qbits.objectstorage.api.Page;
|
||||
import com.kingsrook.qbits.objectstorage.api.StoredObject;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Minimal storage boundary. The application never deletes sealed objects.
|
||||
*******************************************************************************/
|
||||
public interface SealedStore
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Only positively confirmed Object Lock support is acceptable.
|
||||
*******************************************************************************/
|
||||
boolean supportsLock();
|
||||
|
||||
/*******************************************************************************
|
||||
** Preserve unknown provider capabilities for read-only setup diagnostics.
|
||||
*******************************************************************************/
|
||||
default CapabilityStatus lockCapability()
|
||||
{
|
||||
return (supportsLock() ? CapabilityStatus.SUPPORTED : CapabilityStatus.UNSUPPORTED);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Upload a file with the selected Object Lock mode and retention.
|
||||
*******************************************************************************/
|
||||
String put(String key, Path file, com.kingsrook.qbits.objectstorage.api.LockMode mode, Instant retainUntil) throws Exception;
|
||||
|
||||
/*******************************************************************************
|
||||
** Read current object metadata, or null for an absent object.
|
||||
*******************************************************************************/
|
||||
StoredObject head(String key) throws Exception;
|
||||
|
||||
/*******************************************************************************
|
||||
** Open the exact sealed version; the caller owns and closes the stream.
|
||||
*******************************************************************************/
|
||||
InputStream getVersion(String key, String versionId) throws Exception;
|
||||
|
||||
/*******************************************************************************
|
||||
** List one bounded page.
|
||||
*******************************************************************************/
|
||||
Page<ObjectSummary> list(ListRequest request) throws Exception;
|
||||
}
|
||||
|
||||
@@ -45,6 +45,13 @@ public class AppendOnlyGuard implements MetaDataCustomizerInterface<QTableMetaDa
|
||||
@Override
|
||||
public QTableMetaData customizeMetaData(QInstance instance, QTableMetaData table)
|
||||
{
|
||||
if(com.kingsrook.qbits.wormaudit.model.WormAuditTables.SETTINGS.equals(table.getName()))
|
||||
{
|
||||
table.withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_DELETE);
|
||||
table.withAuditRules(new com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules()
|
||||
.withAuditLevel(com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel.FIELD));
|
||||
return (table);
|
||||
}
|
||||
if(CORE_TABLES.contains(table.getName()) || table.getName().startsWith("worm_audit_"))
|
||||
{
|
||||
table.withoutCapabilities(Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
|
||||
|
||||
@@ -81,6 +81,7 @@ public class LinkRangeWidget extends AbstractWidgetRenderer
|
||||
String path = QContext.getQInstance().getTablePath(WormAuditTables.CHAIN);
|
||||
String viewAll = path + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), StandardCharsets.UTF_8);
|
||||
QueryInput query = new QueryInput(WormAuditTables.CHAIN).withFilter(filter.clone().withLimit(100)).withInputSource(QInputSource.USER);
|
||||
query.withShouldTranslatePossibleValues(true);
|
||||
query.withPermissionEnforcement(PermissionEnforcement.ENFORCE);
|
||||
return (new RenderWidgetOutput(new ChildRecordListData("Chain links", new QueryAction().execute(query),
|
||||
QContext.getQInstance().getTable(WormAuditTables.CHAIN), path, viewAll, total)));
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.model;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditSettings;
|
||||
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Validate the editable singleton, including internal insert-only seeds.
|
||||
*******************************************************************************/
|
||||
public class SettingsCustomizer implements TableCustomizerInterface
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Only the fixed singleton may be inserted by the internal seed path.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> preInsert(InsertInput input, List<QRecord> records, boolean preview) throws QException
|
||||
{
|
||||
if(records.size() != 1 || !Integer.valueOf(1).equals(records.getFirst().getValueInteger("id")))
|
||||
{
|
||||
throw (new QUserFacingException("WORM Audit Settings has one row only (ID 1)."));
|
||||
}
|
||||
if(new com.kingsrook.qbits.wormaudit.chain.AuditStore(input.getTransaction()).get(WormAuditTables.SETTINGS, 1) != null)
|
||||
{
|
||||
throw (new QUserFacingException("WORM Audit Settings already exists. Edit record 1 instead of adding another record."));
|
||||
}
|
||||
return (validate(records));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Validate partial edits without replacing the existing settings.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> preUpdate(UpdateInput input, List<QRecord> records, boolean preview, Optional<List<QRecord>> oldRecords) throws QException
|
||||
{
|
||||
PermissionsHelper.checkTablePermissionThrowing(input, TablePermissionSubType.EDIT);
|
||||
return (validate(records));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reject deletion even for direct core calls where capability checks are off.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> preDelete(DeleteInput input, List<QRecord> records, boolean preview) throws QException
|
||||
{
|
||||
throw (new QUserFacingException("WORM Audit Settings cannot be deleted."));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Refresh local settings after persistence; remote JVMs retain the bounded cache.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> postUpdate(UpdateInput input, List<QRecord> records, Optional<List<QRecord>> oldRecords) throws QException
|
||||
{
|
||||
com.kingsrook.qbits.wormaudit.processes.VerificationRuns.config().invalidateSettings();
|
||||
return (records);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stamp the editor and time on every valid write; core writes FIELD audits.
|
||||
*******************************************************************************/
|
||||
private List<QRecord> validate(List<QRecord> records) throws QException
|
||||
{
|
||||
for(QRecord row : records)
|
||||
{
|
||||
try
|
||||
{
|
||||
WormAuditSettings.validate(row);
|
||||
}
|
||||
catch(IllegalArgumentException e)
|
||||
{
|
||||
throw (new QUserFacingException(e.getMessage()));
|
||||
}
|
||||
row.setValue("modify_date", Instant.now());
|
||||
row.setValue("modified_by", QContext.getQSession() == null || QContext.getQSession().getUser() == null
|
||||
? "system" : QContext.getQSession().getUser().getIdReference());
|
||||
}
|
||||
return (records);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.model;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.CustomPermissionChecker;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reading uses the full table name; editing additionally needs settings access.
|
||||
*******************************************************************************/
|
||||
public class SettingsPermissions implements CustomPermissionChecker
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Enforce table permission and the separate settings permission.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void checkPermissionsThrowing(AbstractActionInput input, MetaDataWithPermissionRules metadata) throws QPermissionDeniedException
|
||||
{
|
||||
var session = QContext.getQSession();
|
||||
boolean edit = input instanceof UpdateInput;
|
||||
if(input instanceof InsertInput || input instanceof DeleteInput || session == null
|
||||
|| !session.hasPermission(WormAuditTables.SETTINGS + (edit ? ".edit" : ".read"))
|
||||
|| (edit && !session.hasPermission("worm_audit.settings")))
|
||||
{
|
||||
throw (new QPermissionDeniedException(edit ? "Editing WORM Audit Settings requires worm_audit_settings.edit and worm_audit.settings."
|
||||
: "WORM Audit Settings requires read access; adding or deleting settings is not allowed."));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.model;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Applies help/qbit-worm-audit.yaml (the QQQ help-bundle format) to the WORM
|
||||
** Audit app and its worm_audit_* tables: what each table is for and how its
|
||||
** rows get created, shown on the table screen and when it is empty.
|
||||
** The bundle's app entry is keyed to the default app name; a host that renames
|
||||
** the app gets the same text.
|
||||
** STAND-IN(core=QRun-IO/qqq#1198, removeBy=4.1.0-RC.7): replace with
|
||||
** HelpBundles.register once the pinned QQQ release includes it; the bundle
|
||||
** file stays the same.
|
||||
*******************************************************************************/
|
||||
public final class WormAuditHelp
|
||||
{
|
||||
public static final String RESOURCE = "/help/qbit-worm-audit.yaml";
|
||||
public static final String SLOT = "help";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private WormAuditHelp()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The bundle's entries by key, validated (fails startup if malformed).
|
||||
*******************************************************************************/
|
||||
public static Map<String, JsonNode> entries() throws QException
|
||||
{
|
||||
try(InputStream stream = WormAuditHelp.class.getResourceAsStream(RESOURCE))
|
||||
{
|
||||
if(stream == null)
|
||||
{
|
||||
throw (new IOException("missing resource"));
|
||||
}
|
||||
JsonNode bundle = new ObjectMapper(new YAMLFactory()).readTree(stream);
|
||||
if(!"qbit-worm-audit".equals(bundle.path("bundle").asText()) || !"QBIT".equals(bundle.path("layer").asText()))
|
||||
{
|
||||
throw (new IOException("invalid bundle header"));
|
||||
}
|
||||
Map<String, JsonNode> entries = new LinkedHashMap<>();
|
||||
for(JsonNode entry : bundle.path("entries"))
|
||||
{
|
||||
String key = entry.path("key").asText();
|
||||
String summary = entry.path("summary").asText();
|
||||
if(key.isBlank() || summary.isBlank() || summary.length() > 120 || entry.path("content").asText().isBlank() || entries.put(key, entry) != null)
|
||||
{
|
||||
throw (new IOException("invalid or duplicate entry: " + key));
|
||||
}
|
||||
}
|
||||
return (entries);
|
||||
}
|
||||
catch(IOException | RuntimeException e)
|
||||
{
|
||||
throw (new QException("Could not load " + RESOURCE, e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Apply the help to the app and every worm_audit_* table. A host's own help on
|
||||
** a table or the app is kept (the bundle only fills the slot).
|
||||
*******************************************************************************/
|
||||
public static void apply(QInstance qInstance, WormAuditQBitConfig config) throws QException
|
||||
{
|
||||
for(Map.Entry<String, JsonNode> entry : entries().entrySet())
|
||||
{
|
||||
String key = entry.getKey();
|
||||
if(key.startsWith("table:"))
|
||||
{
|
||||
QTableMetaData table = qInstance.getTable(key.substring("table:".length(), key.indexOf(';')));
|
||||
if(table != null && (table.getHelpContent() == null || !table.getHelpContent().containsKey(SLOT)))
|
||||
{
|
||||
table.withHelpContent(SLOT, content(entry.getValue()));
|
||||
}
|
||||
}
|
||||
else if(config.getIncludeApp() && key.startsWith("app:"))
|
||||
{
|
||||
QAppMetaData app = qInstance.getApp(config.getAppName());
|
||||
if(app != null && (app.getHelpContent() == null || !app.getHelpContent().containsKey(SLOT)))
|
||||
{
|
||||
app.withHelpContent(SLOT, content(entry.getValue()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Summary as the first paragraph, then the content, as markdown.
|
||||
*******************************************************************************/
|
||||
static QHelpContent content(JsonNode entry)
|
||||
{
|
||||
return (new QHelpContent().withContentAsMarkdown(entry.path("summary").asText().strip() + "\n\n" + entry.path("content").asText().strip()));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -24,13 +24,21 @@ package com.kingsrook.qbits.wormaudit.model;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditSettings;
|
||||
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
|
||||
@@ -39,17 +47,19 @@ import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDe
|
||||
|
||||
/*******************************************************************************
|
||||
** Append-only sidecar tables. Snake-case fields map directly to DDL columns.
|
||||
** Export metadata is the integration hook for phase (b); no export runs here.
|
||||
** Labels also supply the standard QQQ app navigation wording.
|
||||
*******************************************************************************/
|
||||
public final class WormAuditTables
|
||||
{
|
||||
public static final String CHAIN = "worm_audit_chain";
|
||||
public static final String CHAIN_DETAIL = "worm_audit_chain_detail";
|
||||
public static final String CHECKPOINT = "worm_audit_checkpoint";
|
||||
public static final String EXPORT = "worm_audit_export";
|
||||
public static final String VERIFICATION = "worm_audit_verification";
|
||||
public static final String PURGE = "worm_audit_purge";
|
||||
public static final List<String> ALL = List.of(CHAIN, CHECKPOINT, EXPORT, VERIFICATION, PURGE);
|
||||
public static final List<String> NAV = List.of(VERIFICATION, EXPORT, CHECKPOINT, CHAIN, PURGE);
|
||||
public static final String SETTINGS = "worm_audit_settings";
|
||||
public static final List<String> ALL = List.of(CHAIN, CHAIN_DETAIL, CHECKPOINT, EXPORT, VERIFICATION, PURGE, SETTINGS);
|
||||
public static final List<String> NAV = List.of(VERIFICATION, EXPORT, CHECKPOINT, CHAIN, CHAIN_DETAIL, PURGE, SETTINGS);
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
@@ -66,22 +76,87 @@ public final class WormAuditTables
|
||||
{
|
||||
List<QTableMetaData> tables = new ArrayList<>();
|
||||
tables.add(table(config, CHAIN, "Audit Chain Link", "id:L kind:S audit_id:L detail_ids:T prev_hash:S link_hash:S chained_at:D"));
|
||||
// Keyed by the covered audit_detail id: one row per detail, written with its link.
|
||||
tables.add(table(config, CHAIN_DETAIL, "Detail Coverage", "detail_id:L chain_id:L").withPrimaryKeyField("detail_id")
|
||||
.withRecordLabelFormatAndFields("Detail #%s (link %s)", "detail_id", "chain_id"));
|
||||
tables.add(table(config, CHECKPOINT, "Checkpoint", "id:L from_link_id:L to_link_id:L link_count:L head_hash:S merkle_root:S algorithm:S key_id:S public_key:T signature:T created_at:D"));
|
||||
tables.add(table(config, EXPORT, "Sealed Export", "id:L from_link_id:L to_link_id:L from_checkpoint_id:L to_checkpoint_id:L bucket_alias:S object_prefix:T manifest_key:T manifest_sha256:S prev_manifest_sha256:S object_count:L byte_count:L lock_mode:S retain_until:D sealed_at:D"));
|
||||
tables.add(table(config, EXPORT, "Sealed Export", "id:L from_link_id:L to_link_id:L from_checkpoint_id:L to_checkpoint_id:L bucket_alias:S object_prefix:T object_versions:T manifest_key:T manifest_sha256:S prev_manifest_sha256:S object_count:L byte_count:L lock_mode:S retain_until:D sealed_at:D"));
|
||||
tables.add(table(config, VERIFICATION, "Verification", "id:L kind:S trigger:S started_at:D finished_at:D from_link_id:L to_link_id:L links_checked:L outcome:S break_count:L first_break_link_id:L first_break_audit_id:L first_break_reason:T export_id:L run_by:S"));
|
||||
tables.add(table(config, PURGE, "Purge", "id:L requested_at:D requested_by:S cutoff:D retention_interval:S outcome:S reason:T audits_deleted:L details_deleted:L links_deleted:L anchor_link_id:L anchor_hash:S"));
|
||||
tables.add(settings(config));
|
||||
return (tables);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Editable, FIELD-audited singleton with standard QQQ record sections.
|
||||
*******************************************************************************/
|
||||
public static QTableMetaData settings(WormAuditQBitConfig config)
|
||||
{
|
||||
QTableMetaData table = new QTableMetaData().withName(SETTINGS).withLabel("WORM Audit Settings")
|
||||
.withBackendName(config.getBackendName()).withBackendDetails(new RDBMSTableBackendDetails().withTableName(SETTINGS))
|
||||
.withPrimaryKeyField("id").withRecordLabelFields("id").withRecordLabelFormat("WORM Audit Settings")
|
||||
.withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD))
|
||||
.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_INSERT_EDIT_DELETE_PERMISSIONS)
|
||||
.withPermissionBaseName(SETTINGS).withCustomPermissionChecker(new QCodeReference(SettingsPermissions.class)))
|
||||
.withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_DELETE)
|
||||
.withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(SettingsCustomizer.class))
|
||||
.withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, new QCodeReference(SettingsCustomizer.class))
|
||||
.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, new QCodeReference(SettingsCustomizer.class))
|
||||
.withCustomizer(TableCustomizers.PRE_DELETE_RECORD, new QCodeReference(SettingsCustomizer.class))
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withLabel("ID").withBackendName("id").withIsEditable(false).withDefaultValue(1));
|
||||
var defaults = WormAuditSettings.defaults(config);
|
||||
table.withField(new QFieldMetaData("audit_retention_years", QFieldType.INTEGER).withBackendName("audit_retention_years").withLabel("Audit retention years")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("audit_retention_years")));
|
||||
table.withField(new QFieldMetaData("export_lock_mode", QFieldType.STRING).withBackendName("export_lock_mode").withLabel("Export lock mode")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("export_lock_mode")));
|
||||
table.withField(new QFieldMetaData("export_retention_days", QFieldType.INTEGER).withBackendName("export_retention_days").withLabel("Export retention days")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("export_retention_days")));
|
||||
table.withField(new QFieldMetaData("checkpoint_every_links", QFieldType.INTEGER).withBackendName("checkpoint_every_links").withLabel("Checkpoint every links")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("checkpoint_every_links")));
|
||||
table.withField(new QFieldMetaData("checkpoint_every_minutes", QFieldType.INTEGER).withBackendName("checkpoint_every_minutes").withLabel("Checkpoint every minutes")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("checkpoint_every_minutes")));
|
||||
table.withField(new QFieldMetaData("chain_interval_seconds", QFieldType.INTEGER).withBackendName("chain_interval_seconds").withLabel("Chain interval seconds")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("chain_interval_seconds")));
|
||||
table.withField(new QFieldMetaData("settle_horizon_seconds", QFieldType.INTEGER).withBackendName("settle_horizon_seconds").withLabel("Settle horizon seconds")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("settle_horizon_seconds")));
|
||||
table.withField(new QFieldMetaData("late_detail_lookback_hours", QFieldType.INTEGER).withBackendName("late_detail_lookback_hours").withLabel("Late detail lookback hours")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("late_detail_lookback_hours")));
|
||||
table.withField(new QFieldMetaData("export_cron", QFieldType.STRING).withBackendName("export_cron").withLabel("Export schedule (UTC)")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("export_cron")));
|
||||
table.withField(new QFieldMetaData("verify_cron", QFieldType.STRING).withBackendName("verify_cron").withLabel("Verify schedule (UTC)")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("verify_cron")));
|
||||
table.getField("export_lock_mode").withInlinePossibleValueSource(new QPossibleValueSource().withName(SETTINGS + "_lock_mode")
|
||||
.withType(QPossibleValueSourceType.ENUM).withIdType(QFieldType.STRING).withEnumValues(List.of(
|
||||
new QPossibleValue<>("GOVERNANCE", "GOVERNANCE"), new QPossibleValue<>("COMPLIANCE", "COMPLIANCE"))));
|
||||
for(String field : List.of("export_cron", "verify_cron", "chain_interval_seconds"))
|
||||
{
|
||||
table.getField(field).withHelpContent(new QHelpContent("Takes effect after restart. Cron schedules use UTC and Quartz syntax."));
|
||||
}
|
||||
table.getField("audit_retention_years").withHelpContent(new QHelpContent("Used for the next eligible purge date. The database purge interval changes only through a reviewed migration."));
|
||||
table.withField(new QFieldMetaData("notes", QFieldType.TEXT).withLabel("Notes").withBackendName("notes"));
|
||||
table.withField(new QFieldMetaData("modify_date", QFieldType.DATE_TIME).withLabel("Modified").withBackendName("modify_date").withIsEditable(false));
|
||||
table.withField(new QFieldMetaData("modified_by", QFieldType.STRING).withLabel("Modified by").withBackendName("modified_by").withIsEditable(false));
|
||||
table.withSection(new QFieldSection("retention", "Retention", new QIcon("lock"), Tier.T1,
|
||||
List.of("id", "audit_retention_years", "export_lock_mode", "export_retention_days")).withGridColumns(12));
|
||||
table.withSection(new QFieldSection("cadence", "Chain and checkpoints", new QIcon("link"), Tier.T2,
|
||||
List.of("checkpoint_every_links", "checkpoint_every_minutes", "settle_horizon_seconds", "late_detail_lookback_hours")).withGridColumns(12));
|
||||
table.withSection(new QFieldSection("schedules", "Schedules (after restart)", new QIcon("schedule"), Tier.T2,
|
||||
List.of("chain_interval_seconds", "export_cron", "verify_cron")).withGridColumns(12));
|
||||
table.withSection(new QFieldSection("changes", "Changes", new QIcon("edit_note"), Tier.T2,
|
||||
List.of("notes", "modify_date", "modified_by")).withGridColumns(12));
|
||||
return (table);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Common metadata for a process-written table.
|
||||
*******************************************************************************/
|
||||
private static QTableMetaData table(WormAuditQBitConfig config, String name, String label, String fields)
|
||||
{
|
||||
QTableMetaData table = new QTableMetaData().withName(name).withLabel(label)
|
||||
QTableMetaData table = new QTableMetaData().withName(name).withLabel(navigationLabel(name))
|
||||
.withBackendName(config.getBackendName()).withBackendDetails(new RDBMSTableBackendDetails().withTableName(name))
|
||||
.withPrimaryKeyField("id").withRecordLabelFields("id").withRecordLabelFormat("%s")
|
||||
.withPrimaryKeyField("id").withRecordLabelFields("id").withRecordLabelFormat(label + " #%s")
|
||||
.withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.NONE))
|
||||
.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_INSERT_EDIT_DELETE_PERMISSIONS).withPermissionBaseName(name));
|
||||
for(String spec : fields.split(" "))
|
||||
@@ -94,9 +169,9 @@ public final class WormAuditTables
|
||||
case "T" -> QFieldType.TEXT;
|
||||
default -> QFieldType.STRING;
|
||||
};
|
||||
table.withField(new QFieldMetaData(parts[0], type).withBackendName(parts[0]));
|
||||
table.withField(new QFieldMetaData(parts[0], type).withBackendName(parts[0]).withLabel(fieldLabel(parts[0])));
|
||||
}
|
||||
for(String reference : List.of("from_link_id", "to_link_id", "first_break_link_id", "anchor_link_id"))
|
||||
for(String reference : List.of("from_link_id", "to_link_id", "first_break_link_id", "anchor_link_id", "chain_id"))
|
||||
{
|
||||
if(table.getFields().containsKey(reference))
|
||||
{
|
||||
@@ -114,11 +189,130 @@ public final class WormAuditTables
|
||||
{
|
||||
table.getField("export_id").withPossibleValueSourceName(EXPORT);
|
||||
}
|
||||
table.withSection(new QFieldSection("record", label, new QIcon("fact_check"), Tier.T1, new ArrayList<>(table.getFields().keySet())));
|
||||
switch(name)
|
||||
{
|
||||
case CHAIN -> table.withRecordLabelFormat("Link #%s");
|
||||
case EXPORT -> table.withRecordLabelFormatAndFields("Export #%s (links %s–%s)", "id", "from_link_id", "to_link_id");
|
||||
case VERIFICATION, PURGE -> table.withRecordLabelFormatAndFields(label + " #%s (%s)", "id", "outcome");
|
||||
default ->
|
||||
{
|
||||
}
|
||||
}
|
||||
if(table.getFields().containsKey("kind"))
|
||||
{
|
||||
table.getField("kind").withInlinePossibleValueSource(new QPossibleValueSource().withName(name + "_kind")
|
||||
.withType(QPossibleValueSourceType.ENUM).withIdType(QFieldType.STRING).withEnumValues(List.of(new QPossibleValue<>("AUDIT", "Audit"),
|
||||
new QPossibleValue<>("LATE_DETAIL", "Late detail"), new QPossibleValue<>("CHAIN", "Chain"),
|
||||
new QPossibleValue<>("SEALED_EXPORT", "Sealed export"))));
|
||||
}
|
||||
if(table.getFields().containsKey("outcome"))
|
||||
{
|
||||
table.getField("outcome").withInlinePossibleValueSource(new QPossibleValueSource().withName(name + "_outcome")
|
||||
.withType(QPossibleValueSourceType.ENUM).withIdType(QFieldType.STRING).withEnumValues(List.of(new QPossibleValue<>("PASS", "Pass"),
|
||||
new QPossibleValue<>("FAIL", "Fail"), new QPossibleValue<>("PURGED", "Purged"), new QPossibleValue<>("REFUSED", "Refused"))));
|
||||
}
|
||||
if(VERIFICATION.equals(name))
|
||||
{
|
||||
table.withSection(new QFieldSection("result", "Result", new QIcon("fact_check"), Tier.T2,
|
||||
List.of("id", "outcome", "break_count", "first_break_link_id", "first_break_audit_id", "first_break_reason")).withGridColumns(12));
|
||||
table.withSection(new QFieldSection("scope", "Scope", new QIcon("link"), Tier.T2,
|
||||
List.of("kind", "trigger", "from_link_id", "to_link_id", "links_checked", "export_id")).withGridColumns(12));
|
||||
table.withSection(new QFieldSection("run", "Run", new QIcon("schedule"), Tier.T2,
|
||||
List.of("started_at", "finished_at", "run_by")).withGridColumns(12));
|
||||
}
|
||||
else
|
||||
{
|
||||
table.withSection(new QFieldSection("record", label, new QIcon("fact_check"), Tier.T1,
|
||||
new ArrayList<>(table.getFields().keySet())).withGridColumns(12));
|
||||
}
|
||||
if(List.of(CHECKPOINT, EXPORT, VERIFICATION).contains(name))
|
||||
{
|
||||
table.withSection(new QFieldSection("links", new QIcon("link"), Tier.T2).withLabel("Chain links").withWidgetName(name + "_links"));
|
||||
table.withSection(new QFieldSection("links", new QIcon("link"), Tier.T2).withLabel("Chain links")
|
||||
.withWidgetName(name + "_links").withGridColumns(12));
|
||||
}
|
||||
return (table);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Core RC.5 shares the table label with its navigation tree node.
|
||||
*******************************************************************************/
|
||||
public static String navigationLabel(String name)
|
||||
{
|
||||
return (switch(name)
|
||||
{
|
||||
case SETTINGS -> "Settings";
|
||||
case CHAIN -> "Audit Chain";
|
||||
case CHAIN_DETAIL -> "Detail Coverage";
|
||||
case CHECKPOINT -> "Checkpoints";
|
||||
case EXPORT -> "Sealed Exports";
|
||||
case VERIFICATION -> "Verifications";
|
||||
default -> "Purges";
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Plain record labels independent of physical column names.
|
||||
*******************************************************************************/
|
||||
private static String fieldLabel(String name)
|
||||
{
|
||||
return (switch(name)
|
||||
{
|
||||
case "id" -> "ID";
|
||||
case "kind" -> "Kind";
|
||||
case "audit_id" -> "Audit";
|
||||
case "detail_ids" -> "Detail ids";
|
||||
case "detail_id" -> "Audit detail";
|
||||
case "chain_id" -> "Link";
|
||||
case "prev_hash" -> "Previous hash";
|
||||
case "link_hash" -> "Link hash";
|
||||
case "chained_at" -> "Chained";
|
||||
case "from_link_id" -> "From link";
|
||||
case "to_link_id" -> "To link";
|
||||
case "link_count" -> "Links";
|
||||
case "head_hash" -> "Head hash";
|
||||
case "merkle_root" -> "Merkle root";
|
||||
case "algorithm" -> "Algorithm";
|
||||
case "key_id" -> "Key ID";
|
||||
case "public_key" -> "Public key";
|
||||
case "signature" -> "Signature";
|
||||
case "created_at" -> "Created";
|
||||
case "from_checkpoint_id" -> "From checkpoint";
|
||||
case "to_checkpoint_id" -> "To checkpoint";
|
||||
case "bucket_alias" -> "Bucket alias";
|
||||
case "object_prefix" -> "Object prefix";
|
||||
case "object_versions" -> "Object versions";
|
||||
case "manifest_key" -> "Manifest key";
|
||||
case "manifest_sha256" -> "Manifest SHA-256";
|
||||
case "prev_manifest_sha256" -> "Previous manifest SHA-256";
|
||||
case "object_count" -> "Objects";
|
||||
case "byte_count" -> "Bytes";
|
||||
case "lock_mode" -> "Lock mode";
|
||||
case "retain_until" -> "Retain until";
|
||||
case "sealed_at" -> "Sealed";
|
||||
case "trigger" -> "Trigger";
|
||||
case "started_at" -> "Started";
|
||||
case "finished_at" -> "Finished";
|
||||
case "links_checked" -> "Links checked";
|
||||
case "outcome" -> "Outcome";
|
||||
case "break_count" -> "Breaks";
|
||||
case "first_break_link_id" -> "First break link";
|
||||
case "first_break_audit_id" -> "First break audit";
|
||||
case "first_break_reason" -> "First break";
|
||||
case "export_id" -> "Sealed export";
|
||||
case "run_by" -> "Run by";
|
||||
case "requested_at" -> "Requested";
|
||||
case "requested_by" -> "Requested by";
|
||||
case "cutoff" -> "Cutoff";
|
||||
case "retention_interval" -> "Retention interval";
|
||||
case "reason" -> "Reason";
|
||||
case "audits_deleted" -> "Audits deleted";
|
||||
case "details_deleted" -> "Details deleted";
|
||||
case "links_deleted" -> "Links deleted";
|
||||
case "anchor_link_id" -> "Anchor link";
|
||||
case "anchor_hash" -> "Anchor hash";
|
||||
default -> throw new IllegalArgumentException("Missing field label: " + name);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,377 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.processes;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedExportVerifier;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedExporter;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedStore;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.UseOtherPermissionNameCustomPermissionChecker;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunInput;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunOutput;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Standard Review/Result actions for sealing and verifying exports.
|
||||
*******************************************************************************/
|
||||
public final class ExportProcesses
|
||||
{
|
||||
public static final String EXPORT = "worm_audit_export_now";
|
||||
public static final String VERIFY = "worm_audit_verify_export";
|
||||
public static final String DAILY = "worm_audit_export_daily";
|
||||
public static final String PERMISSION = "worm_audit.export";
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private ExportProcesses()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Export is a table action; verification requires one or more selected exports.
|
||||
*******************************************************************************/
|
||||
public static List<QProcessMetaData> processes(WormAuditQBitConfig config)
|
||||
{
|
||||
var settings = config.effectiveSettings();
|
||||
Map<String, Serializable> values = new HashMap<>();
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, WormAuditTables.EXPORT);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, WormAuditTables.EXPORT);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION, true);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true);
|
||||
QProcessMetaData export = StreamedETLWithFrontendProcess.defineProcessMetaData(ExportExtract.class, ExportReview.class, ExportLoad.class, new HashMap<>(values))
|
||||
.withName(EXPORT).withLabel("Export now").withTableName(WormAuditTables.EXPORT).withPermissionRules(permission(PERMISSION));
|
||||
QProcessMetaData verify = StreamedETLWithFrontendProcess.defineProcessMetaData(VerifyExtract.class, VerifyReview.class, VerifyLoad.class, new HashMap<>(values))
|
||||
.withName(VERIFY).withLabel("Verify sealed export").withTableName(WormAuditTables.EXPORT).withMinInputRecords(1)
|
||||
.withPermissionRules(permission(WormAuditProcesses.PERMISSION));
|
||||
QProcessMetaData daily = new QProcessMetaData().withName(DAILY).withLabel("Daily sealed audit export")
|
||||
.withStep(new QBackendStepMetaData().withName("export").withCode(new QCodeReference(Daily.class)))
|
||||
.withPermissionRules(permission(PERMISSION)).withSchedule(new QScheduleMetaData().withSchedulerName(config.getSchedulerName())
|
||||
.withCronExpression(settings.export_cron()).withCronTimeZoneId("UTC"));
|
||||
return (List.of(export, verify, daily));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reuse the phase (a) explicit permission-name checker.
|
||||
*******************************************************************************/
|
||||
private static QPermissionRules permission(String name)
|
||||
{
|
||||
return (new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION)
|
||||
.withCustomPermissionChecker(UseOtherPermissionNameCustomPermissionChecker.build(name)));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Plain text summaries must not acquire the framework's default zero count.
|
||||
*******************************************************************************/
|
||||
private static ProcessSummaryLine text(Status status, String message)
|
||||
{
|
||||
ProcessSummaryLine line = new ProcessSummaryLine(status, message);
|
||||
line.setCount(null);
|
||||
return (line);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One synthetic work item allows the first export on an empty export table.
|
||||
*******************************************************************************/
|
||||
public static class ExportExtract extends AbstractExtractStep
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Count the single export operation, not previous export rows.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public Integer doCount(RunBackendStepInput input)
|
||||
{
|
||||
return (1);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Feed one work item through the standard stream.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
getRecordPipe().addRecord(new QRecord().withValue("id", 0L));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Review is read-only; the writer reselects the suffix after confirmation.
|
||||
*******************************************************************************/
|
||||
public static class ExportReview extends AbstractTransformStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private String message;
|
||||
private boolean failed;
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the planned range once without uploading or inserting.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
try
|
||||
{
|
||||
message = SealedExporter.scope(new AuditStore(null)).review();
|
||||
failed = false;
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
message = "Cannot select a contiguous export range";
|
||||
failed = true;
|
||||
}
|
||||
output.getRecords().addAll(input.getRecords());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Show the range, including the explicit no-checkpoint case.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
return (new ArrayList<>(List.of(text(failed ? Status.ERROR : Status.INFO, message == null ? "Nothing to seal: no new checkpoint" : message))));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Seal once after the stream is complete.
|
||||
*******************************************************************************/
|
||||
public static class ExportLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final SealedStore storage;
|
||||
private SealedExporter.Result result;
|
||||
|
||||
/*******************************************************************************
|
||||
** Framework constructor.
|
||||
*******************************************************************************/
|
||||
public ExportLoad()
|
||||
{
|
||||
this(null);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Storage-injected action for unit tests.
|
||||
*******************************************************************************/
|
||||
public ExportLoad(SealedStore storage)
|
||||
{
|
||||
this.storage = storage;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Forward the work item without retaining content.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output)
|
||||
{
|
||||
output.getRecords().addAll(input.getRecords());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The same execution path as the daily schedule.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void postRun(BackendStepPostRunInput input, BackendStepPostRunOutput output)
|
||||
{
|
||||
result = new SealedExporter(VerificationRuns.config(), storage).run("USER", Instant.now());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Result text is value-free even when the provider fails.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
return (result == null ? new ArrayList<>() : new ArrayList<>(List.of(text(result.failed() ? Status.ERROR : Status.OK, result.message()))));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Respect core's selected-record/filter interpretation; no implicit all-export run.
|
||||
*******************************************************************************/
|
||||
public static class VerifyExtract extends ExtractViaQueryStep
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Deterministic selection with full text fields from the export table.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QQueryFilter getQueryFilter(RunBackendStepInput input) throws QException
|
||||
{
|
||||
input.addValue(FIELD_SOURCE_TABLE, WormAuditTables.EXPORT);
|
||||
QQueryFilter filter = super.getQueryFilter(input).clone();
|
||||
filter.setOrderBys(List.of(new QFilterOrderBy("id", true)));
|
||||
return (filter);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Review selected exports without reading their content.
|
||||
*******************************************************************************/
|
||||
public static class VerifyReview extends AbstractTransformStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private long count;
|
||||
|
||||
/*******************************************************************************
|
||||
** Count and forward the selected export records.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output)
|
||||
{
|
||||
count += input.getRecords().size();
|
||||
output.getRecords().addAll(input.getRecords());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One Review summary.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
return (new ArrayList<>(List.of(text(Status.INFO, WormAuditVerifier.number(count) + " sealed exports will be verified"))));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify each selected export and persist its independent result.
|
||||
*******************************************************************************/
|
||||
public static class VerifyLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final SealedStore storage;
|
||||
private final ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
|
||||
/*******************************************************************************
|
||||
** Framework constructor.
|
||||
*******************************************************************************/
|
||||
public VerifyLoad()
|
||||
{
|
||||
this(null);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Storage-injected action for unit tests.
|
||||
*******************************************************************************/
|
||||
public VerifyLoad(SealedStore storage)
|
||||
{
|
||||
this.storage = storage;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Process the current page without retaining exported content.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
for(QRecord selected : input.getRecords())
|
||||
{
|
||||
QRecord row = new AuditStore(null).get(WormAuditTables.EXPORT, selected.getValueLong("id"));
|
||||
if(row == null)
|
||||
{
|
||||
lines.add(text(Status.ERROR, "Selected export " + selected.getValueLong("id") + " is missing"));
|
||||
continue;
|
||||
}
|
||||
var report = new SealedExportVerifier(VerificationRuns.config(), storage).verify(row, "USER");
|
||||
if(report.result().passed())
|
||||
{
|
||||
lines.add(text(Status.OK, WormAuditVerifier.number(report.result().getLinksChecked()) + " links verified in export " + row.getValueLong("id")));
|
||||
}
|
||||
report.result().getBreaks().forEach(b -> lines.add(text(Status.ERROR, b.reason())));
|
||||
if(report.purged() > 0 && report.result().passed())
|
||||
{
|
||||
lines.add(text(Status.INFO, WormAuditVerifier.number(report.purged()) + " links verified from the sealed copy only (purged from the live database)"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exact per-link differences and purged-copy information.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Daily export sends heartbeat even when sealing fails.
|
||||
*******************************************************************************/
|
||||
public static class Daily implements BackendStep
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Run the exporter with the installed host configuration.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output)
|
||||
{
|
||||
var result = new SealedExporter(VerificationRuns.config()).run("SCHEDULE", Instant.now());
|
||||
output.addValue("outcome", result.failed() ? "FAIL" : "PASS");
|
||||
output.addValue("result", result.message());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,6 +24,7 @@ package com.kingsrook.qbits.wormaudit.processes;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
@@ -43,6 +44,11 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
*******************************************************************************/
|
||||
public final class VerificationRuns
|
||||
{
|
||||
/** How many break reasons the alert summary spells out; the title carries the total. */
|
||||
private static final int ALERT_SUMMARY_REASONS = 3;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
@@ -75,12 +81,19 @@ public final class VerificationRuns
|
||||
{
|
||||
if(result.passed())
|
||||
{
|
||||
QAlerts.resolve("worm_audit.chain.break", "chain", "Audit chain verification passed");
|
||||
QAlerts.resolve("wormaudit.chain.break", "chain", "Audit chain verification passed");
|
||||
}
|
||||
else
|
||||
{
|
||||
QAlerts.raise(new AlertSignal("worm_audit.chain.break").withDedupKey("chain")
|
||||
.withTitle(result.getBreaks().getFirst().reason()).withRecordRef(new AlertRecordRef(WormAuditTables.VERIFICATION, row.getValue("id"))));
|
||||
long problems = result.getBreakCount();
|
||||
String summary = result.getBreaks().stream().limit(ALERT_SUMMARY_REASONS).map(b -> b.reason()).collect(Collectors.joining("; "));
|
||||
if(problems > ALERT_SUMMARY_REASONS)
|
||||
{
|
||||
summary += "; and " + (problems - ALERT_SUMMARY_REASONS) + " more";
|
||||
}
|
||||
QAlerts.raise(new AlertSignal("wormaudit.chain.break").withDedupKey("chain")
|
||||
.withTitle("Audit chain verification found " + problems + (problems == 1 ? " problem" : " problems")).withSummary(summary)
|
||||
.withRecordRef(new AlertRecordRef(WormAuditTables.VERIFICATION, row.getValue("id"))));
|
||||
}
|
||||
}
|
||||
return (result);
|
||||
@@ -89,7 +102,7 @@ public final class VerificationRuns
|
||||
{
|
||||
if(scheduled)
|
||||
{
|
||||
QAlerts.heartbeat("worm_audit.verify.daily");
|
||||
QAlerts.heartbeat("wormaudit.verify.daily");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -99,10 +112,28 @@ public final class VerificationRuns
|
||||
** Persist identifiers and a value-free first reason, never audit content.
|
||||
*******************************************************************************/
|
||||
public static QRecord save(VerificationResult result, String trigger) throws QException
|
||||
{
|
||||
return (save(result, trigger, null));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Append a chain or sealed-export result using the same immutable schema.
|
||||
*******************************************************************************/
|
||||
public static QRecord save(VerificationResult result, String trigger, Long exportId) throws QException
|
||||
{
|
||||
return (save(result, trigger, exportId, Instant.now()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Clock-injected persistence for genuine historical verification runs.
|
||||
*******************************************************************************/
|
||||
public static QRecord save(VerificationResult result, String trigger, Long exportId, Instant finishedAt) throws QException
|
||||
{
|
||||
String user = QContext.getQSession() == null || QContext.getQSession().getUser() == null ? "system" : QContext.getQSession().getUser().getIdReference();
|
||||
QRecord row = new QRecord().withValue("kind", "CHAIN").withValue("trigger", trigger)
|
||||
.withValue("started_at", result.getStartedAt()).withValue("finished_at", Instant.now())
|
||||
QRecord row = new QRecord().withValue("kind", exportId == null ? "CHAIN" : "SEALED_EXPORT").withValue("export_id", exportId).withValue("trigger", trigger)
|
||||
.withValue("started_at", result.getStartedAt()).withValue("finished_at", finishedAt)
|
||||
.withValue("from_link_id", result.getFromLinkId()).withValue("to_link_id", result.getToLinkId())
|
||||
.withValue("links_checked", result.getLinksChecked()).withValue("outcome", result.passed() ? "PASS" : "FAIL")
|
||||
.withValue("break_count", result.getBreakCount()).withValue("run_by", user);
|
||||
|
||||
@@ -89,6 +89,7 @@ public final class WormAuditProcesses
|
||||
*******************************************************************************/
|
||||
public static List<QProcessMetaData> processes(WormAuditQBitConfig config)
|
||||
{
|
||||
var settings = config.effectiveSettings();
|
||||
Map<String, Serializable> values = new HashMap<>();
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, WormAuditTables.CHAIN);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, WormAuditTables.CHAIN);
|
||||
@@ -100,11 +101,11 @@ public final class WormAuditProcesses
|
||||
QProcessMetaData chain = new QProcessMetaData().withName(CHAIN).withLabel("Chain settled audit records")
|
||||
.withStep(new QBackendStepMetaData().withName("chain").withCode(new QCodeReference(ChainTick.class)))
|
||||
.withPermissionRules(permission()).withSchedule(new QScheduleMetaData().withSchedulerName(config.getSchedulerName())
|
||||
.withRepeatSeconds(config.getChainIntervalSeconds()).withInitialDelaySeconds(config.getChainIntervalSeconds()));
|
||||
.withRepeatSeconds(settings.chain_interval_seconds()).withInitialDelaySeconds(settings.chain_interval_seconds()));
|
||||
QProcessMetaData daily = new QProcessMetaData().withName(DAILY).withLabel("Daily audit chain verification")
|
||||
.withStep(new QBackendStepMetaData().withName("verify").withCode(new QCodeReference(Daily.class)))
|
||||
.withPermissionRules(permission()).withSchedule(new QScheduleMetaData().withSchedulerName(config.getSchedulerName())
|
||||
.withCronExpression(config.getVerifyCron()).withCronTimeZoneId("UTC"));
|
||||
.withCronExpression(settings.verify_cron()).withCronTimeZoneId("UTC"));
|
||||
return (List.of(verify, chain, daily));
|
||||
}
|
||||
|
||||
|
||||
@@ -41,12 +41,30 @@ public class VerificationResult
|
||||
|
||||
private final List<Break> breaks = new ArrayList<>();
|
||||
private final List<String> warnings = new ArrayList<>();
|
||||
private final Instant startedAt = Instant.now();
|
||||
private final Instant startedAt;
|
||||
private long breakCount;
|
||||
private long linksChecked;
|
||||
private Long fromLinkId;
|
||||
private Long toLinkId;
|
||||
|
||||
/*******************************************************************************
|
||||
** Use the production clock by default.
|
||||
*******************************************************************************/
|
||||
public VerificationResult()
|
||||
{
|
||||
this(Instant.now());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Bind the verification start to the caller's clock.
|
||||
*******************************************************************************/
|
||||
public VerificationResult(Instant startedAt)
|
||||
{
|
||||
this.startedAt = startedAt;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Add a finding in deterministic verification order.
|
||||
*******************************************************************************/
|
||||
|
||||
@@ -54,13 +54,28 @@ public class WormAuditVerifier
|
||||
{
|
||||
public static final int PAGE_SIZE = 1000;
|
||||
private final WormAuditQBitConfig config;
|
||||
private final int pageSize;
|
||||
|
||||
/*******************************************************************************
|
||||
** Bind trust pins and settle horizon.
|
||||
*******************************************************************************/
|
||||
public WormAuditVerifier(WormAuditQBitConfig config)
|
||||
{
|
||||
this(config, PAGE_SIZE);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Smaller pages support constrained hosts and boundary regression tests.
|
||||
*******************************************************************************/
|
||||
public WormAuditVerifier(WormAuditQBitConfig config, int pageSize)
|
||||
{
|
||||
if(pageSize < 1 || pageSize > PAGE_SIZE)
|
||||
{
|
||||
throw (new IllegalArgumentException("Verification page size must be between 1 and 1000"));
|
||||
}
|
||||
this.config = config;
|
||||
this.pageSize = pageSize;
|
||||
}
|
||||
|
||||
|
||||
@@ -77,6 +92,15 @@ public class WormAuditVerifier
|
||||
** Null means full verification; an empty set is an empty selection.
|
||||
*******************************************************************************/
|
||||
public VerificationResult verify(Set<Long> selection) throws QException
|
||||
{
|
||||
return (verify(selection, Instant.now()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Clock-injected verification uses the same horizon and snapshot as production.
|
||||
*******************************************************************************/
|
||||
public VerificationResult verify(Set<Long> selection, Instant now) throws QException
|
||||
{
|
||||
WormAuditChainer.JVM_LOCK.lock();
|
||||
try(QBackendTransaction transaction = QBackendTransaction.openFor(new QueryInput(WormAuditTables.CHAIN)))
|
||||
@@ -87,7 +111,7 @@ public class WormAuditVerifier
|
||||
{
|
||||
rdbms.getConnection().setTransactionIsolation(Connection.TRANSACTION_REPEATABLE_READ);
|
||||
}
|
||||
VerificationResult result = verify(new AuditStore(transaction), selection, Instant.now());
|
||||
VerificationResult result = verify(new AuditStore(transaction), selection, now);
|
||||
transaction.commit();
|
||||
return (result);
|
||||
}
|
||||
@@ -107,9 +131,10 @@ public class WormAuditVerifier
|
||||
/*******************************************************************************
|
||||
** Verify one consistent storage snapshot.
|
||||
*******************************************************************************/
|
||||
private VerificationResult verify(AuditStore store, Set<Long> selection, Instant now) throws QException
|
||||
VerificationResult verify(AuditStore store, Set<Long> selection, Instant now) throws QException
|
||||
{
|
||||
VerificationResult result = new VerificationResult();
|
||||
var settings = config.effectiveSettings();
|
||||
VerificationResult result = new VerificationResult(now);
|
||||
if(config.getSigningKey() == null)
|
||||
{
|
||||
result.warn("no signing key configured");
|
||||
@@ -125,7 +150,7 @@ public class WormAuditVerifier
|
||||
CheckpointCursor checkpoints = new CheckpointCursor(store, selection, anchor, result);
|
||||
while(true)
|
||||
{
|
||||
List<QRecord> page = store.page(WormAuditTables.CHAIN, lastId, PAGE_SIZE);
|
||||
List<QRecord> page = store.page(WormAuditTables.CHAIN, lastId, pageSize);
|
||||
if(page.isEmpty())
|
||||
{
|
||||
break;
|
||||
@@ -150,9 +175,9 @@ public class WormAuditVerifier
|
||||
}
|
||||
Map<Long, QRecord> details = new HashMap<>();
|
||||
List<Long> ids = new ArrayList<>(detailIds);
|
||||
for(int start = 0; start < ids.size(); start += PAGE_SIZE)
|
||||
for(int start = 0; start < ids.size(); start += pageSize)
|
||||
{
|
||||
details.putAll(AuditStore.index(store.in("auditDetail", "id", ids.subList(start, Math.min(start + PAGE_SIZE, ids.size())))));
|
||||
details.putAll(AuditStore.index(store.in("auditDetail", "id", ids.subList(start, Math.min(start + pageSize, ids.size())))));
|
||||
}
|
||||
for(QRecord link : page)
|
||||
{
|
||||
@@ -180,8 +205,8 @@ public class WormAuditVerifier
|
||||
previous = link.getValueString("link_hash");
|
||||
lastId = id;
|
||||
}
|
||||
checkpoints.through(lastId);
|
||||
}
|
||||
// The main scan's page is out of scope before checkpoint pages are read.
|
||||
checkpoints.through(Long.MAX_VALUE);
|
||||
result.scope(checked, first, last);
|
||||
if(missing != null)
|
||||
@@ -190,15 +215,19 @@ public class WormAuditVerifier
|
||||
}
|
||||
if(selection == null)
|
||||
{
|
||||
Instant cutoff = now.minusSeconds(config.getSettleHorizonSeconds());
|
||||
Instant cutoff = now.minusSeconds(settings.settle_horizon_seconds());
|
||||
if(store.isRdbms())
|
||||
{
|
||||
store.completeness(cutoff, result);
|
||||
}
|
||||
else
|
||||
else if(store.isMemory())
|
||||
{
|
||||
completeness(store, store.all(WormAuditTables.CHAIN), cutoff, result);
|
||||
}
|
||||
else
|
||||
{
|
||||
pagedCompleteness(store, cutoff, result);
|
||||
}
|
||||
}
|
||||
return (result);
|
||||
}
|
||||
@@ -263,6 +292,73 @@ public class WormAuditVerifier
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Portable completeness for non-SQL backends. Trade repeated scans for bounded
|
||||
** memory; only the memory backend uses the whole-chain coverage index.
|
||||
*******************************************************************************/
|
||||
private void pagedCompleteness(AuditStore store, Instant cutoff, VerificationResult result) throws QException
|
||||
{
|
||||
for(String table : List.of("audit", "auditDetail"))
|
||||
{
|
||||
long after = 0;
|
||||
while(true)
|
||||
{
|
||||
List<QRecord> rows = store.page(table, after, pageSize);
|
||||
if(rows.isEmpty())
|
||||
{
|
||||
break;
|
||||
}
|
||||
Map<Long, Integer> counts = new HashMap<>();
|
||||
for(QRecord row : rows)
|
||||
{
|
||||
QRecord audit = table.equals("audit") ? row : store.get("audit", row.getValueLong("auditId"));
|
||||
if(audit == null || audit.getValueInstant("timestamp") == null || audit.getValueInstant("timestamp").isBefore(cutoff))
|
||||
{
|
||||
counts.put(row.getValueLong("id"), 0);
|
||||
if(table.equals("audit") && audit.getValueInstant("timestamp") == null)
|
||||
{
|
||||
result.add(null, row.getValueLong("id"), "audit " + number(row.getValueLong("id")) + " has no timestamp");
|
||||
}
|
||||
}
|
||||
}
|
||||
long linkAfter = 0;
|
||||
while(!counts.isEmpty())
|
||||
{
|
||||
List<QRecord> links = store.page(WormAuditTables.CHAIN, linkAfter, pageSize);
|
||||
if(links.isEmpty())
|
||||
{
|
||||
break;
|
||||
}
|
||||
for(QRecord link : links)
|
||||
{
|
||||
if(table.equals("audit"))
|
||||
{
|
||||
if("AUDIT".equals(link.getValueString("kind")))
|
||||
{
|
||||
counts.computeIfPresent(link.getValueLong("audit_id"), (id, count) -> count + 1);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
try
|
||||
{
|
||||
AuditStore.detailIds(link).forEach(id -> counts.computeIfPresent(id, (key, count) -> count + 1));
|
||||
}
|
||||
catch(IllegalArgumentException e)
|
||||
{
|
||||
// Malformed coverage was reported by content verification.
|
||||
}
|
||||
}
|
||||
}
|
||||
linkAfter = links.getLast().getValueLong("id");
|
||||
}
|
||||
counts.forEach((id, count) -> coverage(table.equals("audit") ? "audit" : "audit_detail", id, count, result));
|
||||
after = rows.getLast().getValueLong("id");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every settled header and detail must occur once; orphan details also fail.
|
||||
*******************************************************************************/
|
||||
@@ -405,7 +501,7 @@ public class WormAuditVerifier
|
||||
result.add(null, null, prefix + "signature does not verify");
|
||||
}
|
||||
if(from < 1 || to < from || checkpoint.getValueLong("link_count") != to - from + 1
|
||||
|| to - from + 1 > config.getCheckpointEveryLinks())
|
||||
|| to - from + 1 > Integer.MAX_VALUE)
|
||||
{
|
||||
result.add(null, null, prefix + "invalid link range or count");
|
||||
return;
|
||||
@@ -429,7 +525,7 @@ public class WormAuditVerifier
|
||||
String head = null;
|
||||
while(cursor < to)
|
||||
{
|
||||
List<QRecord> page = store.page(WormAuditTables.CHAIN, cursor, (int) Math.min(PAGE_SIZE, to - cursor));
|
||||
List<QRecord> page = store.page(WormAuditTables.CHAIN, cursor, (int) Math.min(pageSize, to - cursor));
|
||||
if(page.isEmpty())
|
||||
{
|
||||
break;
|
||||
|
||||
@@ -37,6 +37,7 @@ CREATE TABLE IF NOT EXISTS public.worm_audit_export (
|
||||
to_checkpoint_id bigint,
|
||||
bucket_alias text,
|
||||
object_prefix text,
|
||||
object_versions text,
|
||||
manifest_key text,
|
||||
manifest_sha256 text,
|
||||
prev_manifest_sha256 text,
|
||||
@@ -323,3 +324,233 @@ CREATE INDEX IF NOT EXISTS worm_audit_chain_detail_coverage ON public.worm_audit
|
||||
DROP TRIGGER IF EXISTS worm_audit_detail_coverage_guard ON public.worm_audit_chain;
|
||||
CREATE TRIGGER worm_audit_detail_coverage_guard BEFORE INSERT ON public.worm_audit_chain
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_detail_coverage();
|
||||
|
||||
-- changeset worm-audit:10
|
||||
-- Editable singleton: deliberately excluded from append-only triggers.
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_settings (
|
||||
id integer PRIMARY KEY CHECK (id = 1),
|
||||
audit_retention_years integer NOT NULL DEFAULT 6 CHECK (audit_retention_years BETWEEN 1 AND 100),
|
||||
export_lock_mode text NOT NULL DEFAULT 'GOVERNANCE' CHECK (export_lock_mode IN ('GOVERNANCE', 'COMPLIANCE')),
|
||||
export_retention_days integer NOT NULL DEFAULT 30 CHECK (export_retention_days BETWEEN 1 AND 36500),
|
||||
checkpoint_every_links integer NOT NULL DEFAULT 10000 CHECK (checkpoint_every_links > 0),
|
||||
checkpoint_every_minutes integer NOT NULL DEFAULT 60 CHECK (checkpoint_every_minutes > 0),
|
||||
chain_interval_seconds integer NOT NULL DEFAULT 30 CHECK (chain_interval_seconds > 0),
|
||||
settle_horizon_seconds integer NOT NULL DEFAULT 60 CHECK (settle_horizon_seconds >= 0),
|
||||
late_detail_lookback_hours integer NOT NULL DEFAULT 24 CHECK (late_detail_lookback_hours > 0),
|
||||
export_cron text NOT NULL DEFAULT '0 0 2 * * ?',
|
||||
verify_cron text NOT NULL DEFAULT '0 0 3 * * ?',
|
||||
notes text,
|
||||
modify_date timestamptz NOT NULL DEFAULT now(),
|
||||
modified_by text NOT NULL DEFAULT current_user
|
||||
);
|
||||
REVOKE DELETE, TRUNCATE ON public.worm_audit_settings FROM ${app_role};
|
||||
GRANT SELECT, INSERT, UPDATE ON public.worm_audit_settings TO ${app_role};
|
||||
INSERT INTO public.worm_audit_settings
|
||||
(id, audit_retention_years, export_lock_mode, export_retention_days, checkpoint_every_links,
|
||||
checkpoint_every_minutes, chain_interval_seconds, settle_horizon_seconds, late_detail_lookback_hours,
|
||||
export_cron, verify_cron)
|
||||
SELECT ${settings_values}
|
||||
WHERE NOT EXISTS (SELECT 1 FROM public.worm_audit_settings WHERE id = 1);
|
||||
|
||||
-- changeset worm-audit:11
|
||||
-- comment: Index the foreign key columns added by worm-audit:12 (worm_audit_chain.audit_id already leads worm_audit_chain_audit_all).
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_export_from_checkpoint ON public.worm_audit_export(from_checkpoint_id);
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_export_to_checkpoint ON public.worm_audit_export(to_checkpoint_id);
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_verification_export ON public.worm_audit_verification(export_id);
|
||||
|
||||
-- changeset worm-audit:12 splitStatements:false
|
||||
-- comment: Foreign keys, ON DELETE RESTRICT only: nothing cascades into or out of audit evidence. A chain link needs its audit header (the purge deletes links before headers); a sealed export needs its signed checkpoints and an export verification its export (neither is ever deleted). Links referenced by checkpoints, exports, verifications and purges are not foreign keys: the purge deletes the sealed prefix those rows outlive. Rows whose parent is already missing are evidence and are never changed: they are copied to qqq_fk_quarantine (action KEPT) and the constraint is left NOT VALID (enforced for new rows only).
|
||||
DO $$
|
||||
DECLARE
|
||||
fk record;
|
||||
orphans bigint;
|
||||
BEGIN
|
||||
CREATE TABLE IF NOT EXISTS qqq_fk_quarantine (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, table_name VARCHAR(100) NOT NULL, column_name VARCHAR(100) NOT NULL, row_id VARCHAR(100), missing_parent_id VARCHAR(100), action VARCHAR(20) NOT NULL, row_data TEXT, changeset VARCHAR(200), quarantined_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP);
|
||||
FOR fk IN SELECT * FROM (VALUES
|
||||
('fk_worm_audit_chain_audit_id', 'worm_audit_chain', 'audit_id', '${audit}'),
|
||||
('fk_worm_audit_export_from_checkpoint_id', 'worm_audit_export', 'from_checkpoint_id', 'public.worm_audit_checkpoint'),
|
||||
('fk_worm_audit_export_to_checkpoint_id', 'worm_audit_export', 'to_checkpoint_id', 'public.worm_audit_checkpoint'),
|
||||
('fk_worm_audit_verification_export_id', 'worm_audit_verification', 'export_id', 'public.worm_audit_export')
|
||||
) AS t(name, child, col, parent) LOOP
|
||||
IF NOT EXISTS (SELECT FROM pg_constraint WHERE conname = fk.name) THEN
|
||||
EXECUTE format('INSERT INTO qqq_fk_quarantine (table_name, column_name, row_id, missing_parent_id, action, row_data, changeset)'
|
||||
|| ' SELECT %L, %L, c.id::text, c.%I::text, ''KEPT'', row_to_json(c)::text, ''worm-audit:12'' FROM public.%I c'
|
||||
|| ' WHERE c.%I IS NOT NULL AND NOT EXISTS (SELECT FROM %s p WHERE p.id = c.%I)', fk.child, fk.col, fk.col, fk.child, fk.col, fk.parent, fk.col);
|
||||
GET DIAGNOSTICS orphans = ROW_COUNT;
|
||||
EXECUTE format('ALTER TABLE public.%I ADD CONSTRAINT %I FOREIGN KEY (%I) REFERENCES %s (id) ON DELETE RESTRICT NOT VALID', fk.child, fk.name, fk.col, fk.parent);
|
||||
IF orphans = 0 THEN
|
||||
EXECUTE format('ALTER TABLE public.%I VALIDATE CONSTRAINT %I', fk.child, fk.name);
|
||||
END IF;
|
||||
END IF;
|
||||
END LOOP;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:13 splitStatements:false
|
||||
-- comment: Normalized detail coverage. One insert-only row per audit_detail id covered by a chain link, written by the coverage trigger in the same statement as the link, so the chain writer probes a primary key instead of the GIN array index. Append-only like the chain, with no foreign keys (the owner cannot UPDATE WORM tables, so FK checks would be denied). The advisory lock keeps a running chainer from appending between the backfill and the new trigger body.
|
||||
DO $$
|
||||
BEGIN
|
||||
PERFORM pg_advisory_xact_lock(86753090421);
|
||||
END;
|
||||
$$;
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_chain_detail (
|
||||
detail_id bigint PRIMARY KEY,
|
||||
chain_id bigint NOT NULL,
|
||||
CHECK (detail_id >= 0),
|
||||
CHECK (chain_id > 0)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_chain_detail_chain ON public.worm_audit_chain_detail(chain_id);
|
||||
-- The first link wins if a tampered chain covers a detail twice; the verifier still reports the duplicate from the chain itself.
|
||||
INSERT INTO public.worm_audit_chain_detail(detail_id, chain_id)
|
||||
SELECT x.id::bigint, c.id FROM public.worm_audit_chain c CROSS JOIN LATERAL unnest(string_to_array(c.detail_ids, ',')) AS x(id)
|
||||
WHERE c.detail_ids <> '' ORDER BY c.id
|
||||
ON CONFLICT (detail_id) DO NOTHING;
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_reject() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp SET timezone = 'UTC' AS $fn$
|
||||
DECLARE
|
||||
old_time timestamptz;
|
||||
function_owner name;
|
||||
stack text;
|
||||
row_identity text := 'all';
|
||||
BEGIN
|
||||
IF TG_OP = 'DELETE' AND current_setting('worm_audit.purging', true) = 'on' THEN
|
||||
SELECT pg_get_userbyid(proowner) INTO function_owner FROM pg_proc
|
||||
WHERE oid = 'public.worm_audit_purge(timestamptz)'::regprocedure;
|
||||
GET DIAGNOSTICS stack = PG_CONTEXT;
|
||||
IF current_user = function_owner AND position('function worm_audit_purge(timestamp with time zone)' in stack) > 0 THEN
|
||||
IF TG_RELID = '${audit}'::regclass THEN
|
||||
old_time := OLD.timestamp;
|
||||
ELSIF TG_RELID = '${audit_detail}'::regclass THEN
|
||||
SELECT timestamp INTO old_time FROM ${audit} WHERE id = OLD.audit_id;
|
||||
ELSIF TG_RELID = 'public.worm_audit_chain'::regclass THEN
|
||||
SELECT timestamp INTO old_time FROM ${audit} WHERE id = OLD.audit_id;
|
||||
ELSIF TG_RELID = 'public.worm_audit_chain_detail'::regclass THEN
|
||||
SELECT a.timestamp INTO old_time FROM public.worm_audit_chain c JOIN ${audit} a ON a.id = c.audit_id WHERE c.id = OLD.chain_id;
|
||||
END IF;
|
||||
IF old_time < now() - interval '${retention}' THEN
|
||||
RETURN OLD;
|
||||
END IF;
|
||||
END IF;
|
||||
END IF;
|
||||
IF TG_LEVEL = 'ROW' THEN
|
||||
-- Record fields resolve per statement, so each table's key is read in its own branch.
|
||||
IF TG_RELID = 'public.worm_audit_chain_detail'::regclass THEN
|
||||
row_identity := OLD.detail_id::text;
|
||||
ELSE
|
||||
row_identity := OLD.id::text;
|
||||
END IF;
|
||||
END IF;
|
||||
RAISE EXCEPTION USING ERRCODE = 'P0001',
|
||||
MESSAGE = format('worm_audit: audit rows are write-once; %s of "%s" is not allowed (id=%s). Expired rows are removed only by worm_audit_purge().',
|
||||
TG_OP, TG_TABLE_NAME, row_identity),
|
||||
HINT = 'see qbit-worm-audit docs/DESIGN.md section 4';
|
||||
END;
|
||||
$fn$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_detail_coverage() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $fn$
|
||||
BEGIN
|
||||
-- Serialize even direct SQL appends, then record coverage in the same statement as the link.
|
||||
-- The primary key rejects a detail covered twice, within this link or by an earlier one. The
|
||||
-- previous EXISTS overlap check ran as a cached generic plan that chose a sequential scan of
|
||||
-- the whole chain for every inserted link.
|
||||
PERFORM pg_advisory_xact_lock(86753090421);
|
||||
IF NEW.detail_ids = '' THEN
|
||||
RETURN NEW;
|
||||
END IF;
|
||||
BEGIN
|
||||
INSERT INTO public.worm_audit_chain_detail(detail_id, chain_id)
|
||||
SELECT d, NEW.id FROM unnest(string_to_array(NEW.detail_ids, ',')::bigint[]) AS d;
|
||||
EXCEPTION WHEN unique_violation THEN
|
||||
RAISE EXCEPTION USING ERRCODE = '23505', MESSAGE = 'worm_audit: audit detail is already covered by the chain';
|
||||
END;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$fn$;
|
||||
|
||||
-- The array index worm_audit_chain_detail_coverage is no longer used by this release. It stays until
|
||||
-- a later migration drops it, so a previous-release writer still running during a rolling deploy keeps its indexed probe.
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_chain_detail;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_chain_detail
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_chain_detail;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_chain_detail
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_chain_detail FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON public.worm_audit_chain_detail TO ${app_role};
|
||||
|
||||
-- Same function as worm-audit:5, except that coverage rows are locked and purged before their links.
|
||||
-- The migrating role may have revoked its own EXECUTE (a runtime role that owns the schema, as on CNPG);
|
||||
-- PostgreSQL then refuses CREATE OR REPLACE. Grant it back for this statement only, then restore.
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT has_function_privilege(current_user, 'public.worm_audit_purge(timestamptz)', 'EXECUTE') THEN
|
||||
EXECUTE format('GRANT EXECUTE ON FUNCTION public.worm_audit_purge(timestamptz) TO %I', current_user);
|
||||
PERFORM set_config('worm_audit.purge_regrant', 'on', true);
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_purge(p_cutoff timestamptz) RETURNS void
|
||||
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, public, pg_temp SET timezone = 'UTC' AS $fn$
|
||||
DECLARE
|
||||
boundary bigint;
|
||||
smaller bigint;
|
||||
anchor text;
|
||||
audits_count bigint;
|
||||
details_count bigint;
|
||||
links_count bigint;
|
||||
BEGIN
|
||||
IF p_cutoff IS NULL OR p_cutoff > now() - interval '${retention}' THEN
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome, reason)
|
||||
VALUES(now(), session_user, p_cutoff, '${retention}', 'REFUSED', 'Cutoff is within retention or null');
|
||||
RETURN;
|
||||
END IF;
|
||||
-- Same lock as the Java chainer; table locks also exclude concurrent audit inserts.
|
||||
PERFORM pg_advisory_xact_lock(86753090421);
|
||||
LOCK TABLE ${audit}, ${audit_detail}, public.worm_audit_chain, public.worm_audit_chain_detail IN SHARE ROW EXCLUSIVE MODE;
|
||||
SELECT coalesce(min(c.id) - 1, (SELECT max(id) FROM public.worm_audit_chain)) INTO boundary
|
||||
FROM public.worm_audit_chain c LEFT JOIN ${audit} a ON a.id = c.audit_id
|
||||
WHERE a.id IS NULL OR a.timestamp IS NULL OR a.timestamp >= p_cutoff;
|
||||
-- Shrink to a prefix closed over every audit's links AND all its stored details.
|
||||
LOOP
|
||||
SELECT min(c.id) - 1 INTO smaller FROM public.worm_audit_chain c
|
||||
WHERE c.id <= boundary AND (
|
||||
EXISTS (SELECT FROM public.worm_audit_chain later WHERE later.audit_id = c.audit_id AND later.id > boundary)
|
||||
OR EXISTS (SELECT FROM ${audit_detail} d WHERE d.audit_id = c.audit_id AND NOT EXISTS (
|
||||
SELECT FROM public.worm_audit_chain covered WHERE covered.audit_id = c.audit_id
|
||||
AND d.id::text = ANY(string_to_array(covered.detail_ids, ',')))));
|
||||
EXIT WHEN smaller IS NULL;
|
||||
boundary := smaller;
|
||||
END LOOP;
|
||||
SELECT link_hash INTO anchor FROM public.worm_audit_chain WHERE id = boundary;
|
||||
IF anchor IS NULL THEN
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome, reason)
|
||||
VALUES(now(), session_user, p_cutoff, '${retention}', 'REFUSED', 'No complete expired chain prefix');
|
||||
RETURN;
|
||||
END IF;
|
||||
PERFORM set_config('worm_audit.purging', 'on', true);
|
||||
DELETE FROM ${audit_detail} d WHERE EXISTS
|
||||
(SELECT FROM public.worm_audit_chain c WHERE c.audit_id = d.audit_id AND c.id <= boundary);
|
||||
GET DIAGNOSTICS details_count = ROW_COUNT;
|
||||
-- Coverage rows go before their links: the guard finds their age through the link.
|
||||
DELETE FROM public.worm_audit_chain_detail WHERE chain_id <= boundary;
|
||||
-- Keep ids locally because links must be deleted before their audit headers.
|
||||
WITH removed AS (DELETE FROM public.worm_audit_chain WHERE id <= boundary RETURNING audit_id),
|
||||
removed_audits AS (DELETE FROM ${audit} WHERE id IN (SELECT audit_id FROM removed) RETURNING id)
|
||||
SELECT (SELECT count(*) FROM removed), (SELECT count(*) FROM removed_audits) INTO links_count, audits_count;
|
||||
PERFORM set_config('worm_audit.purging', 'off', true);
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome,
|
||||
audits_deleted, details_deleted, links_deleted, anchor_link_id, anchor_hash)
|
||||
VALUES(now(), session_user, p_cutoff, '${retention}', 'PURGED', audits_count, details_count, links_count, boundary, anchor);
|
||||
END;
|
||||
$fn$;
|
||||
DO $$
|
||||
BEGIN
|
||||
IF current_setting('worm_audit.purge_regrant', true) = 'on' THEN
|
||||
EXECUTE format('REVOKE EXECUTE ON FUNCTION public.worm_audit_purge(timestamptz) FROM %I', current_user);
|
||||
PERFORM set_config('worm_audit.purge_regrant', 'off', true);
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
@@ -0,0 +1,50 @@
|
||||
bundle: qbit-worm-audit
|
||||
version: 1
|
||||
layer: QBIT
|
||||
# Help for the WORM Audit app and every worm_audit_* table, in the QQQ help-bundle
|
||||
# format (qqq#1198). Each table entry says what the table is for and how its rows
|
||||
# get created, so an empty table explains itself. WormAuditHelp applies it until
|
||||
# the core HelpBundles registry ships in a QQQ release
|
||||
# (STAND-IN(core=QRun-IO/qqq#1198, removeBy=4.1.0-RC.7)). Summaries stay under 120
|
||||
# characters. Only the settings table is seeded (by the database script, see the
|
||||
# README); every other table is written by the qbit's jobs and starts empty.
|
||||
entries:
|
||||
- key: app:worm_audit;slot:help
|
||||
summary: "Tamper-evident audit: a hash chain over every audit record, signed checkpoints, and sealed exports."
|
||||
content: |
|
||||
Install the database script (the Setup warnings on the home list what is missing) and the one settings row is created with it. Everything else fills by itself once the first audit record has settled: the chain job links records every 30 seconds by default, checkpoints follow, the daily job verifies and exports. Use Verify audit chain to check at any time.
|
||||
|
||||
- key: table:worm_audit_settings;slot:help
|
||||
summary: "The one settings row: retention, export lock, checkpoint and chain cadence, and the job schedules."
|
||||
content: |
|
||||
**How rows get created:** the database script inserts the one row (ID 1) with the host's configured values, only when it is absent; it can never be added to or deleted. Edit record 1 to change a value; changes are audited and take effect within 30 seconds, except the schedules and chain interval, which need a restart. If the row is missing the qbit runs on the host's configured values, so nothing stops, but run the script to make them visible and editable here. Audit retention here is the next purge date only; the purge interval in the database changes through a reviewed migration.
|
||||
|
||||
- key: table:worm_audit_chain;slot:help
|
||||
summary: "One link per core audit record: a hash that includes the previous link, so any change breaks the chain."
|
||||
content: |
|
||||
**How rows get created:** the Chain settled audit records job appends links for audit records older than the settle horizon (every 30 seconds by default, set in Settings). Nobody adds, edits or deletes a link; the database refuses. An empty table means no audit record has settled yet, or the chain job is not running.
|
||||
|
||||
- key: table:worm_audit_chain_detail;slot:help
|
||||
summary: "Which chain link covers each audit detail: one row per detail, written with the link."
|
||||
content: |
|
||||
**How rows get created:** whenever the chain job appends a link, the database writes one row here for every audit detail the link covers, in the same transaction (the database script also fills it for links that existed before this table). It lets the chain job find details that are not chained yet without scanning the chain. Nobody adds, edits or deletes a row; the database refuses, and verification checks that it matches the chain exactly. An empty table means no audit detail has been chained yet.
|
||||
|
||||
- key: table:worm_audit_checkpoint;slot:help
|
||||
summary: "Signed ranges of links (Ed25519), proving the chain's head at a point in time."
|
||||
content: |
|
||||
**How rows get created:** the chain job writes one after the configured number of links or minutes (Settings), signing the range. Sealed exports and verification check against them. An empty table means the chain is too short for a first checkpoint yet.
|
||||
|
||||
- key: table:worm_audit_export;slot:help
|
||||
summary: "Sealed exports of chain ranges to object-lock storage, with their retention and manifest."
|
||||
content: |
|
||||
**How rows get created:** the Daily sealed audit export job (schedule in Settings) and the Export now action seal the links since the last export into the configured bucket, then record it here. Verify sealed export re-reads one. An empty table means no export has run: check the bucket setup warning on the home.
|
||||
|
||||
- key: table:worm_audit_verification;slot:help
|
||||
summary: "Every run of the chain verification: what was checked and whether it passed or broke."
|
||||
content: |
|
||||
**How rows get created:** the Verify audit chain action and the Daily audit chain verification job (schedule in Settings) each add a row, with the first break found, if any. A break also raises an alert. An empty table means verification has not run yet.
|
||||
|
||||
- key: table:worm_audit_purge;slot:help
|
||||
summary: "Record of each retention purge: the cutoff, what was removed and the new anchor link."
|
||||
content: |
|
||||
**How rows get created:** only the database's owner role can purge, by calling the purge function after the retention date has passed; the function writes this row. The application can never delete audit rows. An empty table means nothing has been purged, which is normal until records pass retention.
|
||||
@@ -167,6 +167,9 @@ class CanonicalAndCryptoTest
|
||||
java.nio.file.Files.readString(
|
||||
java.nio.file.Path.of("docs/ddl/worm-audit-postgres.sql")));
|
||||
assertThat(sql).doesNotContain("${");
|
||||
// Liquibase parses every line starting with "-- changeset" as a directive, comments included.
|
||||
assertThat(sql.lines().filter(line -> line.startsWith("-- changeset")))
|
||||
.allMatch(line -> line.matches("-- changeset worm-audit:[0-9]+( splitStatements:false)?"));
|
||||
assertThat(sql).contains("GET DIAGNOSTICS stack = PG_CONTEXT", "function worm_audit_purge(timestamp with time zone)", "SET timezone = 'UTC'");
|
||||
// The documented migration contract deliberately uses physical snake_case columns.
|
||||
assertThat(sql).contains("OLD.audit_id", "OLD.timestamp", "NEW.name", "GRANT UPDATE (label, create_date, modify_date)");
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import java.net.URI;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.objectstorage.ObjectStorageQBitConfig;
|
||||
import com.kingsrook.qbits.objectstorage.api.Capability;
|
||||
import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
|
||||
import com.kingsrook.qbits.objectstorage.api.ListRequest;
|
||||
import com.kingsrook.qbits.objectstorage.api.ObjectStorage;
|
||||
import com.kingsrook.qbits.objectstorage.config.AddressingStyle;
|
||||
import com.kingsrook.qbits.objectstorage.config.BucketRole;
|
||||
import com.kingsrook.qbits.objectstorage.config.ChecksumMode;
|
||||
import com.kingsrook.qbits.objectstorage.config.ProviderType;
|
||||
import com.kingsrook.qbits.objectstorage.config.StorageBucketConfig;
|
||||
import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig;
|
||||
import com.kingsrook.qbits.objectstorage.internal.CapabilityProbe;
|
||||
import com.kingsrook.qbits.objectstorage.internal.CapabilityStore;
|
||||
import com.kingsrook.qbits.objectstorage.internal.ObjectStorageRuntime;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.export.ExportFiles;
|
||||
import com.kingsrook.qbits.wormaudit.export.ObjectStorageSealedStore;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedExportVerifier;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedExporter;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.ExportProcesses;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.testcontainers.containers.GenericContainer;
|
||||
import org.testcontainers.containers.wait.strategy.Wait;
|
||||
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
|
||||
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
||||
import software.amazon.awssdk.core.checksums.RequestChecksumCalculation;
|
||||
import software.amazon.awssdk.core.checksums.ResponseChecksumValidation;
|
||||
import software.amazon.awssdk.core.sync.RequestBody;
|
||||
import software.amazon.awssdk.regions.Region;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
import software.amazon.awssdk.services.s3.model.S3Exception;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.STORE;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.detail;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.tamper;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Real Object Lock acceptance. Missing Docker skips locally and fails in CI.
|
||||
*******************************************************************************/
|
||||
@ExtendWith(RequiresDockerCondition.class)
|
||||
class MinioExportAcceptanceTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Production adapter, GOVERNANCE and COMPLIANCE retention, version preservation and UI findings.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void lockedRoundTripVersionRetentionLiveTamperAndUnsupportedBucket() throws Exception
|
||||
{
|
||||
String image = System.getProperty("qqq.test.minioImage",
|
||||
"cgr.dev/chainguard/minio@sha256:dc455cdd4db6e390b1b26514f15ff37681a777bf8dab41aac445a0259f751e56");
|
||||
try(GenericContainer<?> minio = new GenericContainer<>(image)
|
||||
.withEnv("MINIO_ROOT_USER", "worm-test-access").withEnv("MINIO_ROOT_PASSWORD", "worm-test-secret")
|
||||
.withCommand("server", "/tmp/minio-data").withExposedPorts(9000)
|
||||
.waitingFor(Wait.forHttp("/minio/health/live").forPort(9000).withStartupTimeout(Duration.ofMinutes(2))))
|
||||
{
|
||||
minio.start();
|
||||
String endpoint = "http://" + minio.getHost() + ":" + minio.getMappedPort(9000);
|
||||
try(S3Client s3 = S3Client.builder().endpointOverride(URI.create(endpoint)).region(Region.US_EAST_1).forcePathStyle(true)
|
||||
.credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("worm-test-access", "worm-test-secret")))
|
||||
.requestChecksumCalculation(RequestChecksumCalculation.WHEN_REQUIRED)
|
||||
.responseChecksumValidation(ResponseChecksumValidation.WHEN_REQUIRED).build())
|
||||
{
|
||||
s3.createBucket(b -> b.bucket("worm-locked").objectLockEnabledForBucket(true));
|
||||
s3.createBucket(b -> b.bucket("worm-plain"));
|
||||
var config = ChainVerificationTest.signed();
|
||||
var instance = init(config);
|
||||
AlertsInstanceMetaData.ofOrWithNew(instance).withSink(new QCodeReference(ProcessesAndAlertsTest.Sink.class));
|
||||
ProcessesAndAlertsTest.Sink.raised.clear();
|
||||
System.setProperty("worm.test.access", "worm-test-access");
|
||||
System.setProperty("worm.test.secret", "worm-test-secret");
|
||||
ObjectStorage.configureStandalone(new ObjectStorageQBitConfig().withBackendName("memory")
|
||||
.withAllowedSecretRefPatterns(List.of("prop.worm.test.*"))
|
||||
.withConnection(new StorageConnectionConfig("minio").withProviderType(ProviderType.MINIO).withEndpoint(endpoint)
|
||||
.withRegion("us-east-1").withAddressingStyle(AddressingStyle.PATH).withChecksumMode(ChecksumMode.WHEN_REQUIRED)
|
||||
.withAccessKeyRef("${prop.worm.test.access}").withSecretKeyRef("${prop.worm.test.secret}"))
|
||||
.withBucket(new StorageBucketConfig("worm_audit").withConnection("minio").withBucketName("worm-locked").withRole(BucketRole.BACKUP).withIndexed(false))
|
||||
.withBucket(new StorageBucketConfig("plain").withConnection("minio").withBucketName("worm-plain").withRole(BucketRole.BACKUP).withIndexed(false)));
|
||||
CapabilityProbe.probeConnection("minio", false);
|
||||
assertThat(ObjectStorage.asSystem("test", "1").bucket("worm_audit").capability(Capability.OBJECT_LOCK)).isEqualTo(CapabilityStatus.SUPPORTED);
|
||||
assertThat(ObjectStorage.asSystem("test", "1").bucket("plain").capability(Capability.OBJECT_LOCK)).isEqualTo(CapabilityStatus.UNSUPPORTED);
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
Instant now = Instant.now();
|
||||
var exported = new SealedExporter(config).run("USER", now);
|
||||
assertThat(exported.failed()).as(exported.message()).isFalse();
|
||||
var row = exported.row();
|
||||
var storage = new ObjectStorageSealedStore("worm_audit", "acceptance");
|
||||
assertThat(storage.list(ListRequest.prefix(row.getValueString("object_prefix"))).getItems()).hasSize(3);
|
||||
var verifier = new SealedExportVerifier(config);
|
||||
assertThat(verifier.verify(row, "USER").result().passed()).isTrue();
|
||||
for(String name : ExportFiles.NAMES)
|
||||
{
|
||||
String key = row.getValueString("object_prefix") + name;
|
||||
var head = storage.head(key);
|
||||
assertThat(head.getLockMode()).isEqualTo("GOVERNANCE");
|
||||
assertThat(Duration.between(now.plus(Duration.ofDays(30)), head.getLockRetainUntil()).abs()).isLessThan(Duration.ofSeconds(2));
|
||||
String originalVersion = head.getVersionId();
|
||||
assertThat(originalVersion).isNotBlank();
|
||||
assertThat(ExportFiles.parse(row.getValueString("object_versions").getBytes(java.nio.charset.StandardCharsets.UTF_8)))
|
||||
.containsEntry(name, originalVersion);
|
||||
assertThatThrownBy(() -> s3.deleteObject(b -> b.bucket("worm-locked").key(key).versionId(originalVersion).bypassGovernanceRetention(false)))
|
||||
.isInstanceOf(S3Exception.class);
|
||||
byte[] original = s3.getObjectAsBytes(b -> b.bucket("worm-locked").key(key).versionId(originalVersion)).asByteArray();
|
||||
// S3 overwrites create new versions; they cannot replace the locked version.
|
||||
var replacement = s3.putObject(b -> b.bucket("worm-locked").key(key).objectLockMode("GOVERNANCE")
|
||||
.objectLockRetainUntilDate(row.getValueInstant("retain_until")), RequestBody.fromString("different replacement bytes"));
|
||||
assertThat(replacement.versionId()).isNotEqualTo(originalVersion);
|
||||
assertThat(s3.getObjectAsBytes(b -> b.bucket("worm-locked").key(key).versionId(originalVersion)).asByteArray()).isEqualTo(original);
|
||||
try(var sealed = storage.getVersion(key, originalVersion))
|
||||
{
|
||||
assertThat(sealed.readAllBytes()).isEqualTo(original);
|
||||
}
|
||||
var report = verifier.verify(row, "USER");
|
||||
assertThat(report.result().passed()).isFalse();
|
||||
assertThat(report.result().getLinksChecked()).isEqualTo(1);
|
||||
assertThat(report.result().getBreaks()).allSatisfy(b -> assertThat(b.reason()).contains("was overwritten after sealing"));
|
||||
assertThat(report.result().getBreaks().toString()).contains("object " + name + " was overwritten after sealing (current version "
|
||||
+ replacement.versionId() + ", sealed version " + originalVersion + " is still retained under Object Lock)");
|
||||
}
|
||||
tamper("auditDetail", 1, "newValue", "tampered live value");
|
||||
var load = new ExportProcesses.VerifyLoad();
|
||||
load.runOnePage(new RunBackendStepInput().withRecords(List.of(row)), new RunBackendStepOutput());
|
||||
assertThat(load.getProcessSummary(new RunBackendStepOutput(), true).toString())
|
||||
.contains("ERROR", "link 1 (audit 1): live content differs from the sealed copy");
|
||||
tamper("auditDetail", 1, "newValue", "new");
|
||||
audit(2);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
config.withExportLockMode("COMPLIANCE");
|
||||
var compliance = new SealedExporter(config).run("USER", Instant.now());
|
||||
assertThat(compliance.failed()).isFalse();
|
||||
assertThat(compliance.row().getValueString("lock_mode")).isEqualTo("COMPLIANCE");
|
||||
assertThat(verifier.verify(compliance.row(), "USER").result().passed()).isTrue();
|
||||
for(String name : ExportFiles.NAMES)
|
||||
{
|
||||
String key = compliance.row().getValueString("object_prefix") + name;
|
||||
var head = storage.head(key);
|
||||
assertThat(head.getLockMode()).isEqualTo("COMPLIANCE");
|
||||
assertThatThrownBy(() -> s3.deleteObject(b -> b.bucket("worm-locked").key(key).versionId(head.getVersionId())))
|
||||
.isInstanceOf(S3Exception.class);
|
||||
}
|
||||
audit(3);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
config.withExportBucketAlias("plain");
|
||||
var failed = new SealedExporter(config).run("SCHEDULE", Instant.now());
|
||||
assertThat(failed.failed()).isTrue();
|
||||
assertThat(failed.message()).contains("does not support Object Lock");
|
||||
assertThat(STORE.all(WormAuditTables.EXPORT)).hasSize(2);
|
||||
assertThat(ProcessesAndAlertsTest.Sink.raised).hasSize(1);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ObjectStorageRuntime.reset();
|
||||
CapabilityStore.clearCache();
|
||||
System.clearProperty("worm.test.access");
|
||||
System.clearProperty("worm.test.secret");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,7 +55,7 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for phi canary test.
|
||||
*******************************************************************************/
|
||||
class PhiCanaryTest
|
||||
public class PhiCanaryTest
|
||||
{
|
||||
// RC.5 has AiFieldMetaData but not DataClassification; use a test-local marker.
|
||||
/*******************************************************************************
|
||||
@@ -93,6 +93,10 @@ class PhiCanaryTest
|
||||
{
|
||||
for(String column : List.of("oldValue", "newValue"))
|
||||
{
|
||||
if(detail.getValueString(column) == null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
Mac mac = Mac.getInstance("HmacSHA256");
|
||||
|
||||
@@ -113,7 +113,6 @@ class PostgresAcceptanceTest
|
||||
void database() throws Exception
|
||||
{
|
||||
config = ChainVerificationTest.signed().withSettleHorizonSeconds(0);
|
||||
instance = RdbmsTestSupport.instance(POSTGRES.getJdbcUrl(), "app", "fixture", config, true);
|
||||
try(var c = owner())
|
||||
{
|
||||
sql(
|
||||
@@ -124,6 +123,7 @@ class PostgresAcceptanceTest
|
||||
c,
|
||||
"DO $$ BEGIN IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='app') THEN"
|
||||
+ " CREATE ROLE app LOGIN PASSWORD 'fixture'; END IF; END $$");
|
||||
instance = RdbmsTestSupport.instance(POSTGRES.getJdbcUrl(), "app", "fixture", config, true);
|
||||
RdbmsTestSupport.createTables(c, instance, true);
|
||||
String ddl = WormAuditDdl.postgres(config);
|
||||
sql(c, ddl);
|
||||
@@ -133,6 +133,39 @@ class PostgresAcceptanceTest
|
||||
lookups();
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Home setup probes see dropped/disabled guards, missing purge and runtime ownership.
|
||||
** This test uses the same Docker condition as the existing PostgreSQL suite.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void homeSetupWarnings() throws Exception
|
||||
{
|
||||
var storage = new com.kingsrook.qbits.wormaudit.demo.MemorySealedStore();
|
||||
var clean = com.kingsrook.qbits.wormaudit.app.WormAuditHomeData.load(config, storage, java.time.Instant.now());
|
||||
assertThat(clean.warnings()).contains("Daily verification has never run").noneMatch(w -> w.contains("guard") && w.contains("settings"));
|
||||
try(var connection = owner())
|
||||
{
|
||||
sql(connection, "UPDATE worm_audit_settings SET audit_retention_years=7 WHERE id=1");
|
||||
config.invalidateSettings();
|
||||
assertThat(com.kingsrook.qbits.wormaudit.app.WormAuditHomeData.load(config, storage, java.time.Instant.now()).warnings())
|
||||
.anyMatch(w -> w.contains("purge function's compiled interval (6 years)"));
|
||||
sql(connection, WormAuditDdl.postgres(config));
|
||||
assertThat(count(connection, "SELECT audit_retention_years FROM worm_audit_settings WHERE id=1")).isEqualTo(7);
|
||||
sql(connection, "UPDATE worm_audit_settings SET audit_retention_years=6 WHERE id=1");
|
||||
config.invalidateSettings();
|
||||
sql(connection, "DROP TRIGGER worm_audit_row_guard ON audit_detail");
|
||||
sql(connection, "ALTER TABLE worm_audit_chain DISABLE TRIGGER worm_audit_truncate_guard");
|
||||
sql(connection, "DROP FUNCTION public.worm_audit_purge(timestamptz)");
|
||||
}
|
||||
var missing = com.kingsrook.qbits.wormaudit.app.WormAuditHomeData.load(config, storage, java.time.Instant.now());
|
||||
assertThat(missing.warnings()).contains("Missing or disabled worm_audit_row_guard on audit_detail",
|
||||
"Missing or disabled worm_audit_truncate_guard on worm_audit_chain", "Retention purge function is missing");
|
||||
instance = RdbmsTestSupport.instance(POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword(), config, true);
|
||||
QContext.init(instance, new QSession());
|
||||
assertThat(com.kingsrook.qbits.wormaudit.app.WormAuditHomeData.load(config, storage, java.time.Instant.now()).warnings())
|
||||
.contains("Runtime role owns audit table audit", "Runtime role owns audit table worm_audit_purge");
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify grants triggers lookup rules and spoofed flag.
|
||||
*******************************************************************************/
|
||||
@@ -196,11 +229,21 @@ class PostgresAcceptanceTest
|
||||
assertThat(((SQLException) e).getSQLState())
|
||||
.isEqualTo("42501"));
|
||||
sql(c, "GRANT UPDATE, DELETE, TRUNCATE ON " + table + " TO app");
|
||||
for(String operation :
|
||||
List.of(
|
||||
"UPDATE " + table + " SET id=id",
|
||||
"DELETE FROM " + table,
|
||||
"TRUNCATE " + table))
|
||||
// A table a foreign key references (audit, checkpoint, export) refuses TRUNCATE before
|
||||
// its guard runs; TRUNCATE ... CASCADE (owner) reaches the guards, which refuse it.
|
||||
if(count(c, "SELECT count(*) FROM pg_constraint WHERE contype='f' AND confrelid='" + table + "'::regclass") > 0)
|
||||
{
|
||||
assertThatThrownBy(() -> sql(runtime, "TRUNCATE " + table))
|
||||
.isInstanceOf(SQLException.class)
|
||||
.hasMessageContaining("referenced in a foreign key constraint");
|
||||
assertThatThrownBy(() -> sql(c, "TRUNCATE " + table + " CASCADE"))
|
||||
.isInstanceOf(SQLException.class)
|
||||
.hasMessageContaining("worm_audit:");
|
||||
}
|
||||
boolean referenced = count(c, "SELECT count(*) FROM pg_constraint WHERE contype='f' AND confrelid='" + table + "'::regclass") > 0;
|
||||
for(String operation : referenced
|
||||
? List.of("UPDATE " + table + " SET id=id", "DELETE FROM " + table)
|
||||
: List.of("UPDATE " + table + " SET id=id", "DELETE FROM " + table, "TRUNCATE " + table))
|
||||
{
|
||||
assertThatThrownBy(() -> sql(runtime, operation))
|
||||
.isInstanceOf(SQLException.class)
|
||||
@@ -285,6 +328,11 @@ class PostgresAcceptanceTest
|
||||
assertThat(purge.getValueLong("details_deleted")).isEqualTo(2);
|
||||
assertThat(purge.getValueLong("links_deleted")).isEqualTo(2);
|
||||
assertThat(purge.getValueString("anchor_hash")).isEqualTo(anchor);
|
||||
try(var c = owner())
|
||||
{
|
||||
assertThat(count(c, "SELECT count(*) FROM worm_audit_chain_detail WHERE chain_id <= 2")).as("coverage purged with its links").isZero();
|
||||
assertThat(count(c, "SELECT count(*) FROM worm_audit_chain_detail")).isZero();
|
||||
}
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).isEmpty();
|
||||
detail(3, 3);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
@@ -313,7 +361,7 @@ class PostgresAcceptanceTest
|
||||
void prefixCannotSplitAnAuditAndAdvisoryLockSkipsTick() throws Exception
|
||||
{
|
||||
// This retention fixture deliberately polls a detail for a six-year-old header.
|
||||
config.withLateDetailLookbackHours(24 * 365 * 10);
|
||||
setting("late_detail_lookback_hours", 24 * 365 * 10);
|
||||
audit(1);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
try(var c = owner())
|
||||
@@ -487,7 +535,7 @@ class PostgresAcceptanceTest
|
||||
void chainerUsesUtcCutoffInChicago() throws Exception
|
||||
{
|
||||
var original = java.util.TimeZone.getDefault();
|
||||
config.withSettleHorizonSeconds(60);
|
||||
setting("settle_horizon_seconds", 60);
|
||||
try(var connection = owner())
|
||||
{
|
||||
sql(connection, "ALTER TABLE audit ALTER COLUMN timestamp TYPE timestamp without time zone USING timestamp AT TIME ZONE 'UTC'");
|
||||
@@ -518,7 +566,7 @@ class PostgresAcceptanceTest
|
||||
@Test
|
||||
void postgresLateDetailWindowAndCompleteness() throws Exception
|
||||
{
|
||||
config.withSettleHorizonSeconds(60);
|
||||
setting("settle_horizon_seconds", 60);
|
||||
var now = java.time.Instant.now().truncatedTo(java.time.temporal.ChronoUnit.SECONDS);
|
||||
var cutoff = now.minusSeconds(60);
|
||||
try(var connection = owner(); var statement = connection.prepareStatement(
|
||||
@@ -549,4 +597,166 @@ class PostgresAcceptanceTest
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).anyMatch(b -> b.reason().contains("audit_detail 1 is not covered"));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The chain trigger writes one coverage row per detail with its link; the
|
||||
** table is write-once, backfilled from the chain by the DDL, purged with its
|
||||
** links, and verification breaks if it ever disagrees with the chain.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void detailCoverageTableIsWrittenByTheTriggerBackfilledAndWriteOnce() throws Exception
|
||||
{
|
||||
// The fixture's audit headers are years old; poll them as late-detail candidates.
|
||||
setting("late_detail_lookback_hours", 24 * 365 * 10);
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
detail(2, 1);
|
||||
audit(2);
|
||||
detail(3, 2);
|
||||
var writer = new WormAuditChainer(config);
|
||||
assertThat(writer.chainOnce().links()).isEqualTo(2);
|
||||
detail(4, 1);
|
||||
assertThat(writer.chainOnce().lateDetails()).isEqualTo(1);
|
||||
assertThat(writer.chainOnce().links()).isZero();
|
||||
String pairs = "SELECT string_agg(detail_id || '@' || chain_id, ',' ORDER BY detail_id) FROM worm_audit_chain_detail";
|
||||
try(var c = owner(); var runtime = app(); var s = c.createStatement())
|
||||
{
|
||||
try(var r = s.executeQuery(pairs))
|
||||
{
|
||||
r.next();
|
||||
assertThat(r.getString(1)).isEqualTo("1@1,2@1,3@2,4@3");
|
||||
}
|
||||
assertThatThrownBy(() -> sql(runtime, "INSERT INTO worm_audit_chain_detail(detail_id,chain_id) VALUES(4,1)"))
|
||||
.isInstanceOf(SQLException.class).satisfies(e -> assertThat(((SQLException) e).getSQLState()).isEqualTo("23505"));
|
||||
for(String privilege : List.of("UPDATE", "DELETE", "TRUNCATE"))
|
||||
{
|
||||
assertThat(count(c, "SELECT has_table_privilege('app','worm_audit_chain_detail','" + privilege + "')::int")).isZero();
|
||||
}
|
||||
sql(c, "GRANT UPDATE, DELETE, TRUNCATE ON worm_audit_chain_detail TO app");
|
||||
for(String operation : List.of("UPDATE worm_audit_chain_detail SET chain_id=chain_id", "DELETE FROM worm_audit_chain_detail", "TRUNCATE worm_audit_chain_detail"))
|
||||
{
|
||||
assertThatThrownBy(() -> sql(runtime, operation)).isInstanceOf(SQLException.class).hasMessageContaining("worm_audit:")
|
||||
.satisfies(e -> assertThat(((SQLException) e).getSQLState()).isEqualTo("P0001"));
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
// An install that predates the table: the DDL backfills it from the //
|
||||
// chain, and the writer then finds nothing left to chain. //
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
sql(c, "DROP TABLE worm_audit_chain_detail");
|
||||
sql(c, WormAuditDdl.postgres(config));
|
||||
try(var r = s.executeQuery(pairs))
|
||||
{
|
||||
r.next();
|
||||
assertThat(r.getString(1)).isEqualTo("1@1,2@1,3@2,4@3");
|
||||
}
|
||||
assertThat(writer.chainOnce().links()).isZero();
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).isEmpty();
|
||||
|
||||
sql(c, "ALTER TABLE worm_audit_chain_detail DISABLE TRIGGER ALL; UPDATE worm_audit_chain_detail SET chain_id=1 WHERE detail_id=3;"
|
||||
+ " ALTER TABLE worm_audit_chain_detail ENABLE TRIGGER ALL");
|
||||
}
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).anyMatch(b -> b.reason().contains("audit_detail 3 has a detail coverage row that disagrees"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Referential integrity guardrail: every table reference of the WORM tables
|
||||
** (possible-value fields, and the chain's audit header) has a RESTRICT
|
||||
** foreign key, or an exemption with its reason. Nothing may cascade into or
|
||||
** out of audit evidence.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void everyReferenceHasARestrictForeignKeyOrAReason() throws Exception
|
||||
{
|
||||
String links = "the retention purge deletes the sealed prefix of chain links; this evidence row outlives the links it names";
|
||||
java.util.Map<String, String> exempt = java.util.Map.of(
|
||||
"worm_audit_checkpoint.from_link_id", links,
|
||||
"worm_audit_checkpoint.to_link_id", links,
|
||||
"worm_audit_export.from_link_id", links,
|
||||
"worm_audit_export.to_link_id", links,
|
||||
"worm_audit_verification.from_link_id", links,
|
||||
"worm_audit_verification.to_link_id", links,
|
||||
"worm_audit_verification.first_break_link_id", links + "; a break may name a link that is missing",
|
||||
"worm_audit_purge.anchor_link_id", "the anchor is the last link the same purge deleted",
|
||||
"worm_audit_chain_detail.chain_id", "the chain is WORM: the owner has no UPDATE, so a foreign key check into it is denied;"
|
||||
+ " the coverage trigger writes the row with its link and the purge deletes it first");
|
||||
|
||||
java.util.Set<String> references = new java.util.TreeSet<>(List.of("worm_audit_chain.audit_id"));
|
||||
for(var table : instance.getTables().values())
|
||||
{
|
||||
if(!table.getName().startsWith("worm_audit_"))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
for(var field : table.getFields().values())
|
||||
{
|
||||
if(field.getPossibleValueSourceName() != null && instance.getTable(field.getPossibleValueSourceName()) != null)
|
||||
{
|
||||
references.add(table.getName() + "." + field.getBackendName());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
java.util.Map<String, String> foreignKeys = new java.util.HashMap<>();
|
||||
try(var c = owner(); var s = c.createStatement(); var r = s.executeQuery("""
|
||||
SELECT c.conrelid::regclass::text || '.' || a.attname, c.confdeltype::text || c.convalidated::text
|
||||
FROM pg_constraint c JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = c.conkey[1]
|
||||
WHERE c.contype = 'f' AND c.conrelid::regclass::text LIKE 'worm_audit_%'"""))
|
||||
{
|
||||
while(r.next())
|
||||
{
|
||||
foreignKeys.put(r.getString(1), r.getString(2));
|
||||
}
|
||||
}
|
||||
assertThat(references.stream().filter(ref -> !foreignKeys.containsKey(ref) && !exempt.containsKey(ref)).toList())
|
||||
.as("references without a foreign key (add one to worm-audit.sql, or an exemption with a reason)").isEmpty();
|
||||
assertThat(foreignKeys).as("RESTRICT and validated").containsOnlyKeys("worm_audit_chain.audit_id", "worm_audit_export.from_checkpoint_id",
|
||||
"worm_audit_export.to_checkpoint_id", "worm_audit_verification.export_id").allSatisfy((k, v) -> assertThat(v).isEqualTo("rtrue"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Upgrading a schema whose chain names a missing audit header: the link is
|
||||
** evidence, so it is copied to qqq_fk_quarantine (KEPT), never changed, and
|
||||
** the constraint is added NOT VALID (new links are still checked). A re-run
|
||||
** changes nothing.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void orphanEvidenceIsKeptAndTheConstraintIsNotValidated() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
try(var c = owner())
|
||||
{
|
||||
sql(c, "ALTER TABLE worm_audit_chain DROP CONSTRAINT fk_worm_audit_chain_audit_id");
|
||||
sql(c, "ALTER TABLE audit DISABLE TRIGGER worm_audit_row_guard; DELETE FROM audit WHERE id = 1; ALTER TABLE audit ENABLE TRIGGER worm_audit_row_guard");
|
||||
sql(c, WormAuditDdl.postgres(config));
|
||||
sql(c, WormAuditDdl.postgres(config));
|
||||
assertThat(count(c, "SELECT count(*) FROM qqq_fk_quarantine WHERE table_name = 'worm_audit_chain' AND column_name = 'audit_id'"
|
||||
+ " AND row_id = '1' AND missing_parent_id = '1' AND action = 'KEPT' AND row_data LIKE '%\"link_hash\"%'")).isEqualTo(1);
|
||||
assertThat(count(c, "SELECT count(*) FROM qqq_fk_quarantine")).isEqualTo(1);
|
||||
assertThat(count(c, "SELECT count(*) FROM worm_audit_chain WHERE id = 1 AND audit_id = 1")).isEqualTo(1);
|
||||
assertThat(count(c, "SELECT count(*) FROM pg_constraint WHERE conname = 'fk_worm_audit_chain_audit_id' AND NOT convalidated")).isEqualTo(1);
|
||||
assertThatThrownBy(() -> sql(c, "INSERT INTO worm_audit_chain(id,kind,audit_id,detail_ids,prev_hash,link_hash,chained_at)"
|
||||
+ " VALUES(99,'AUDIT',404,'',repeat('0',64),repeat('0',64),now())"))
|
||||
.hasMessageContaining("fk_worm_audit_chain_audit_id");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The settings row (seeded by the DDL) overrides config: change it there.
|
||||
*******************************************************************************/
|
||||
private void setting(String column, int value) throws Exception
|
||||
{
|
||||
try(var connection = owner())
|
||||
{
|
||||
sql(connection, "UPDATE worm_audit_settings SET " + column + "=" + value + " WHERE id=1");
|
||||
}
|
||||
config.invalidateSettings();
|
||||
}
|
||||
}
|
||||
@@ -84,6 +84,38 @@ class ProcessesAndAlertsTest
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** RC.5 validates declared severities, names and daily heartbeat expectations.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void alertTypesAndDailyMonitors() throws Exception
|
||||
{
|
||||
var instance = init(new WormAuditQBitConfig());
|
||||
new com.kingsrook.qqq.backend.core.instances.QInstanceValidator().validate(instance);
|
||||
assertThat(new com.kingsrook.qbits.wormaudit.chain.AuditStore(null).isMemory()).isTrue();
|
||||
var types = AlertsInstanceMetaData.of(instance).getAlertTypes();
|
||||
assertThat(types).containsOnlyKeys("wormaudit.chain.break", "wormaudit.export.failed", "wormaudit.verify.daily", "wormaudit.export.daily");
|
||||
assertThat(types.get("wormaudit.chain.break").getDefaultSeverity())
|
||||
.isEqualTo(com.kingsrook.qqq.backend.core.alerts.AlertSeverity.CRITICAL);
|
||||
assertThat(types.get("wormaudit.export.failed").getDefaultSeverity())
|
||||
.isEqualTo(com.kingsrook.qqq.backend.core.alerts.AlertSeverity.ERROR);
|
||||
types.values().forEach(type ->
|
||||
{
|
||||
assertThat(type.getLabel()).isNotBlank();
|
||||
assertThat(type.getDescription()).isNotBlank();
|
||||
assertThat(AlertsInstanceMetaData.ALERT_TYPE_NAME_PATTERN.matcher(type.getName()).matches()).isTrue();
|
||||
});
|
||||
for(String name : List.of("wormaudit.verify.daily", "wormaudit.export.daily"))
|
||||
{
|
||||
var monitors = types.get(name).getDefaultMonitors();
|
||||
assertThat(monitors).hasSize(1);
|
||||
assertThat(monitors.getFirst().getKind()).isEqualTo(com.kingsrook.qqq.backend.core.alerts.AlertMonitorKind.HEARTBEAT);
|
||||
assertThat(monitors.getFirst().getDefinition()).containsEntry("heartbeatKey", name)
|
||||
.containsEntry("expectedEverySeconds", 86400).containsEntry("graceSeconds", 7200);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify daily raise resolve heartbeat and user result.
|
||||
*******************************************************************************/
|
||||
@@ -106,11 +138,12 @@ class ProcessesAndAlertsTest
|
||||
daily.run(new RunBackendStepInput(), output);
|
||||
assertThat(output.getValueString("outcome")).isEqualTo("FAIL");
|
||||
assertThat(Sink.raised).hasSize(1);
|
||||
assertThat(Sink.raised.getFirst().getAlertTypeName()).isEqualTo("worm_audit.chain.break");
|
||||
assertThat(Sink.raised.getFirst().getAlertTypeName()).isEqualTo("wormaudit.chain.break");
|
||||
assertThat(Sink.raised.getFirst().getDedupKey()).isEqualTo("chain");
|
||||
assertThat(Sink.raised.getFirst().getRecordRefs().getFirst().getTableName())
|
||||
.isEqualTo(WormAuditTables.VERIFICATION);
|
||||
assertThat(Sink.raised.getFirst().getTitle()).contains("Break at link 1");
|
||||
assertThat(Sink.raised.getFirst().getTitle()).isEqualTo("Audit chain verification found 1 problem");
|
||||
assertThat(Sink.raised.getFirst().getSummary()).contains("Break at link 1");
|
||||
var first = STORE.all(WormAuditTables.VERIFICATION).getFirst();
|
||||
assertThat(first.getValueLong("first_break_link_id")).isEqualTo(1);
|
||||
assertThat(first.getValueLong("first_break_audit_id")).isEqualTo(1);
|
||||
@@ -119,7 +152,7 @@ class ProcessesAndAlertsTest
|
||||
daily.run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
assertThat(Sink.resolved).hasSize(1);
|
||||
assertThat(Sink.beats)
|
||||
.containsExactly("worm_audit.verify.daily", "worm_audit.verify.daily");
|
||||
.containsExactly("wormaudit.verify.daily", "wormaudit.verify.daily");
|
||||
assertThat(STORE.all(WormAuditTables.VERIFICATION)).hasSize(2);
|
||||
assertThat(VerificationRuns.run(null, "USER").passed()).isTrue();
|
||||
assertThat(Sink.beats).hasSize(2);
|
||||
|
||||
@@ -23,6 +23,7 @@ package com.kingsrook.qbits.wormaudit;
|
||||
import java.sql.DriverManager;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import com.kingsrook.qbits.wormaudit.chain.SweepFloors;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
@@ -209,4 +210,55 @@ class RdbmsChainTest
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Without a coverage trigger the writer records coverage itself, and the
|
||||
** sweep floors advance only past settled ids: a later header and a late
|
||||
** detail for an old, already chained header are both still chained.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void coverageRowsAndSweepFloors() throws Exception
|
||||
{
|
||||
String url = "jdbc:h2:mem:floors_" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE";
|
||||
var config = new WormAuditQBitConfig().withBatchSize(500);
|
||||
var instance = RdbmsTestSupport.instance(url, "sa", "", config, false);
|
||||
try(var connection = DriverManager.getConnection(url, "sa", ""); var statement = connection.createStatement())
|
||||
{
|
||||
RdbmsTestSupport.createTables(connection, instance, false);
|
||||
QContext.init(instance, new QSession());
|
||||
lookups();
|
||||
statement.execute("INSERT INTO audit(id,audit_table_id,audit_user_id,timestamp) VALUES(1,1,1,CURRENT_TIMESTAMP - INTERVAL '2' MINUTE)");
|
||||
detail(10, 1);
|
||||
detail(20, 1);
|
||||
var writer = new WormAuditChainer(config);
|
||||
var floors = SweepFloors.of(config);
|
||||
var now = java.time.Instant.now();
|
||||
assertThat(writer.chainOnce(now).audits()).isEqualTo(1);
|
||||
assertThat(STORE.query(WormAuditTables.CHAIN_DETAIL, new com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter())).extracting(r -> r.getValueLong("detail_id") + "@" + r.getValueLong("chain_id"))
|
||||
.containsExactlyInAnyOrder("10@1", "20@1");
|
||||
assertThat(floors.audit()).isZero();
|
||||
assertThat(writer.chainOnce(now.plusSeconds(600)).links()).isZero();
|
||||
assertThat(floors.audit()).isEqualTo(1);
|
||||
assertThat(floors.detail()).isEqualTo(20);
|
||||
detail(30, 1);
|
||||
statement.execute("INSERT INTO audit(id,audit_table_id,audit_user_id,timestamp) VALUES(2,1,1,CURRENT_TIMESTAMP - INTERVAL '2' MINUTE)");
|
||||
detail(40, 2);
|
||||
var counts = writer.chainOnce(now.plusSeconds(1200));
|
||||
assertThat(counts.audits()).isEqualTo(1);
|
||||
assertThat(counts.lateDetails()).isEqualTo(1);
|
||||
assertThat(STORE.query(WormAuditTables.CHAIN_DETAIL, new com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter())).hasSize(4);
|
||||
assertThat(writer.chainOnce(now.plusSeconds(1800)).links()).isZero();
|
||||
assertThat(floors.audit()).isEqualTo(2);
|
||||
assertThat(floors.detail()).isEqualTo(40);
|
||||
// A detail whose id sits below the floor (its transaction committed very late)
|
||||
// is skipped by floored ticks and chained by the next hourly full sweep.
|
||||
statement.execute("INSERT INTO audit_detail(id,audit_id,field_name,new_value) VALUES(5,2,'title','late')");
|
||||
assertThat(writer.chainOnce(now.plusSeconds(2400)).lateDetails()).isZero();
|
||||
assertThat(writer.chainOnce(now.plus(SweepFloors.FULL_SWEEP_EVERY)).lateDetails()).isEqualTo(1);
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).isEmpty();
|
||||
statement.execute("DELETE FROM worm_audit_chain_detail WHERE detail_id=30");
|
||||
statement.execute("INSERT INTO worm_audit_chain_detail(detail_id,chain_id) VALUES(30,1)");
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).anyMatch(b -> b.reason().contains("audit_detail 30 has a detail coverage row that disagrees"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -127,10 +127,10 @@ class RdbmsTestSupport
|
||||
case DATE_TIME -> "timestamp(6) with time zone";
|
||||
default -> "text";
|
||||
};
|
||||
if(field.getName().equals("id"))
|
||||
if(field.getName().equals(table.getPrimaryKeyField()))
|
||||
{
|
||||
type +=
|
||||
table.getName().equals("worm_audit_chain")
|
||||
table.getName().equals("worm_audit_chain") || table.getName().equals("worm_audit_chain_detail")
|
||||
? " PRIMARY KEY"
|
||||
: " GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY";
|
||||
}
|
||||
|
||||
@@ -45,6 +45,45 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
*******************************************************************************/
|
||||
class RecordIntegrationTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** All record screens have readable labels and full-width sections and lists.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void recordScreensHaveReadableLabelsAndFullWidthLists() throws Exception
|
||||
{
|
||||
var instance = init(new WormAuditQBitConfig());
|
||||
Map<String, String> formats = Map.of(
|
||||
WormAuditTables.CHAIN, "Link #%s", WormAuditTables.CHAIN_DETAIL, "Detail #%s (link %s)", WormAuditTables.CHECKPOINT, "Checkpoint #%s",
|
||||
WormAuditTables.EXPORT, "Export #%s (links %s–%s)", WormAuditTables.VERIFICATION, "Verification #%s (%s)",
|
||||
WormAuditTables.PURGE, "Purge #%s (%s)", WormAuditTables.SETTINGS, "WORM Audit Settings");
|
||||
for(String name : WormAuditTables.ALL)
|
||||
{
|
||||
var table = instance.getTable(name);
|
||||
assertThat(table.getRecordLabelFormat()).isEqualTo(formats.get(name));
|
||||
assertThat(table.getRecordLabelFields()).allMatch(table.getFields()::containsKey);
|
||||
table.getFields().forEach((fieldName, field) ->
|
||||
assertThat(field.getLabel()).as(name + "." + fieldName).isNotBlank().isNotEqualTo(fieldName).doesNotContain("_"));
|
||||
table.getSections().forEach(section ->
|
||||
{
|
||||
assertThat(section.getGridColumns()).isEqualTo(12);
|
||||
if(section.getWidgetName() != null)
|
||||
{
|
||||
assertThat(instance.getWidget(section.getWidgetName()).getGridColumns()).isEqualTo(12);
|
||||
}
|
||||
});
|
||||
}
|
||||
assertThat(instance.getTable(WormAuditTables.EXPORT).getRecordLabelFields()).containsExactly("id", "from_link_id", "to_link_id");
|
||||
for(String name : List.of(WormAuditTables.VERIFICATION, WormAuditTables.PURGE))
|
||||
{
|
||||
assertThat(instance.getTable(name).getRecordLabelFields()).containsExactly("id", "outcome");
|
||||
}
|
||||
var verification = instance.getTable(WormAuditTables.VERIFICATION);
|
||||
assertThat(verification.getSections()).extracting(section -> section.getLabel()).containsExactly("Result", "Scope", "Run", "Chain links");
|
||||
assertThat(verification.getSections().stream().filter(section -> section.getWidgetName() == null)
|
||||
.flatMap(section -> section.getFieldNames().stream()).toList()).containsExactlyInAnyOrderElementsOf(verification.getFields().keySet());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for event sink.
|
||||
*******************************************************************************/
|
||||
@@ -93,6 +132,8 @@ class RecordIntegrationTest
|
||||
assertThat(data.getTotalRows()).isEqualTo(2);
|
||||
assertThat(data.getQueryOutput().getRecords()).hasSize(2);
|
||||
assertThat(data.getViewAllLink()).contains("filter=");
|
||||
String json = com.kingsrook.qqq.backend.core.utils.JsonUtils.toJson(data);
|
||||
assertThat(json).contains("Chain links").doesNotContain("signingKey", "trustedPublicKeys", "sealedStoreSupplier", "privateKey");
|
||||
input.setQueryParams(Map.of("id", "999"));
|
||||
assertThatThrownBy(() -> new LinkRangeWidget().render(input))
|
||||
.hasMessageContaining("was not found");
|
||||
|
||||
@@ -0,0 +1,710 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Instant;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.objectstorage.api.ListRequest;
|
||||
import com.kingsrook.qbits.objectstorage.api.ObjectSummary;
|
||||
import com.kingsrook.qbits.objectstorage.api.Page;
|
||||
import com.kingsrook.qbits.objectstorage.api.StoredObject;
|
||||
import com.kingsrook.qbits.wormaudit.canonical.Jcs;
|
||||
import com.kingsrook.qbits.wormaudit.chain.Hashes;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.export.ExportFiles;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedExportVerifier;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedExporter;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedStore;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.ExportProcesses;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunInput;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunOutput;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.STORE;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.detail;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.remove;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.tamper;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Offline sealed-export contract, corruption, retention, batching and UI tests.
|
||||
*******************************************************************************/
|
||||
class SealedExportTest
|
||||
{
|
||||
static final Instant NOW = Instant.parse("2026-10-14T02:00:00Z");
|
||||
WormAuditQBitConfig config;
|
||||
MemorySealedStore storage;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tiny object boundary; bytes exist only in tests.
|
||||
*******************************************************************************/
|
||||
static class MemorySealedStore implements SealedStore
|
||||
{
|
||||
final Map<String, byte[]> objects = new LinkedHashMap<>();
|
||||
final Map<String, StoredObject> heads = new LinkedHashMap<>();
|
||||
final Map<String, Map<String, byte[]>> history = new LinkedHashMap<>();
|
||||
int nextVersion;
|
||||
String missingVersion;
|
||||
boolean omitVersion;
|
||||
boolean racePut;
|
||||
boolean supported = true;
|
||||
String echoMode;
|
||||
long shortenSeconds;
|
||||
int failPut;
|
||||
|
||||
@Override
|
||||
public boolean supportsLock()
|
||||
{
|
||||
return (supported);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String put(String key, Path file, com.kingsrook.qbits.objectstorage.api.LockMode mode, Instant until) throws Exception
|
||||
{
|
||||
if(failPut > 0 && objects.size() + 1 == failPut)
|
||||
{
|
||||
throw (new IllegalStateException("CANARY-PRIVATE-PROVIDER-TEXT"));
|
||||
}
|
||||
if(objects.containsKey(key))
|
||||
{
|
||||
history.computeIfAbsent(key, k -> new LinkedHashMap<>()).put(heads.get(key).getVersionId(), objects.get(key));
|
||||
}
|
||||
String version = omitVersion ? missingVersion : "v" + ++nextVersion;
|
||||
objects.put(key, Files.readAllBytes(file));
|
||||
heads.put(key, new StoredObject().withKey(key).withVersionId(racePut ? "replacement" : version)
|
||||
.withLockMode(echoMode == null ? mode.name() : echoMode).withLockRetainUntil(until.minusSeconds(shortenSeconds)));
|
||||
return (version);
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoredObject head(String key)
|
||||
{
|
||||
return (heads.get(key));
|
||||
}
|
||||
|
||||
@Override
|
||||
public InputStream getVersion(String key, String versionId)
|
||||
{
|
||||
byte[] bytes = versionId.equals(heads.get(key).getVersionId()) ? objects.get(key) : history.get(key).get(versionId);
|
||||
return (new ByteArrayInputStream(bytes));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Page<ObjectSummary> list(ListRequest request)
|
||||
{
|
||||
return (new Page<ObjectSummary>().withItems(objects.keySet().stream().filter(k -> k.startsWith(request.getPrefix()))
|
||||
.map(k -> new ObjectSummary().withKey(k)).toList()));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise setUp.
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp() throws Exception
|
||||
{
|
||||
config = ChainVerificationTest.signed();
|
||||
var instance = init(config);
|
||||
storage = new MemorySealedStore();
|
||||
ProcessesAndAlertsTest.Sink.raised.clear();
|
||||
ProcessesAndAlertsTest.Sink.resolved.clear();
|
||||
ProcessesAndAlertsTest.Sink.beats.clear();
|
||||
AlertsInstanceMetaData.ofOrWithNew(instance).withSink(new QCodeReference(ProcessesAndAlertsTest.Sink.class));
|
||||
}
|
||||
|
||||
|
||||
QRecord seal() throws Exception
|
||||
{
|
||||
new WormAuditChainer(config).chainOnce(NOW);
|
||||
var result = new SealedExporter(config, storage).run("USER", NOW);
|
||||
assertThat(result.failed()).as(result.message()).isFalse();
|
||||
assertThat(result.row()).isNotNull();
|
||||
return (result.row());
|
||||
}
|
||||
|
||||
|
||||
SealedExportVerifier.Report verify(QRecord row) throws Exception
|
||||
{
|
||||
return (new SealedExportVerifier(config, storage).verify(row, "USER"));
|
||||
}
|
||||
|
||||
|
||||
String findings(QRecord row) throws Exception
|
||||
{
|
||||
return (verify(row).result().getBreaks().toString());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise canonicalGoldenFilesAndRoundTrip.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void canonicalGoldenFilesAndRoundTrip() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
QRecord row = seal();
|
||||
String prefix = "worm-audit/2026/10/14/1-1/";
|
||||
assertThat(row.getValueString("object_prefix")).isEqualTo(prefix);
|
||||
String content = "{\"audit\":{\"clientChannel\":null,\"id\":1,\"message\":\"Record was Edited\",\"recordId\":\"17\",\"table\":\"cms_page\",\"timestamp\":\"2020-01-02T03:04:06.123456Z\",\"user\":\"james\"},\"details\":[{\"fieldName\":\"title\",\"id\":1,\"message\":\"Changed Title\",\"newValue\":\"new\",\"oldValue\":\"old\"}],\"v\":1}";
|
||||
String hash = Hashes.link(Hashes.GENESIS, content.getBytes(StandardCharsets.UTF_8));
|
||||
String golden = "{\"content\":" + content + ",\"link\":{\"auditId\":1,\"detailIds\":[1],\"id\":1,\"kind\":\"AUDIT\",\"linkHash\":\"" + hash
|
||||
+ "\",\"prevHash\":\"" + Hashes.GENESIS + "\"}}\n";
|
||||
assertThat(new String(storage.objects.get(prefix + "links.jsonl"), StandardCharsets.UTF_8)).isEqualTo(golden);
|
||||
var manifest = ExportFiles.parse(storage.objects.get(prefix + "manifest.json"));
|
||||
assertThat(manifest.keySet()).containsExactly("checkpointCount", "createdAt", "files", "format", "fromLinkId", "headHash", "linkCount",
|
||||
"prevManifestSha256", "qbitVersion", "toLinkId");
|
||||
assertThat(manifest).containsEntry("format", "worm-audit-export/1").containsEntry("linkCount", 1L).containsEntry("checkpointCount", 1L)
|
||||
.containsEntry("prevManifestSha256", null).containsEntry("headHash", hash).containsEntry("createdAt", NOW.toString());
|
||||
assertThat(storage.objects.get(prefix + "manifest.json")).isEqualTo(Jcs.bytes(manifest));
|
||||
assertThat(row.getValues()).containsKeys("from_checkpoint_id", "to_checkpoint_id", "bucket_alias", "manifest_key", "manifest_sha256",
|
||||
"prev_manifest_sha256", "object_count", "byte_count", "lock_mode", "retain_until", "sealed_at");
|
||||
assertThat(row.getValueLong("byte_count")).isEqualTo(storage.objects.values().stream().mapToLong(b -> b.length).sum());
|
||||
assertThat(row.getValueInstant("retain_until")).isEqualTo(NOW.plusSeconds(30 * 86400L));
|
||||
assertThat(verify(row).result().passed()).isTrue();
|
||||
QRecord verification = STORE.last(WormAuditTables.VERIFICATION);
|
||||
assertThat(verification.getValueString("kind")).isEqualTo("SEALED_EXPORT");
|
||||
assertThat(verification.getValueLong("export_id")).isEqualTo(row.getValueLong("id"));
|
||||
assertThat(verification.getValueString("outcome")).isEqualTo("PASS");
|
||||
assertThat(ProcessesAndAlertsTest.Sink.resolved).hasSize(1);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every overwrite is a break while verification still uses the sealed bytes.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void detectsOverwritesAndReadsSealedVersions() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
QRecord row = seal();
|
||||
Map<String, Object> versions = ExportFiles.parse(row.getValueString("object_versions").getBytes(StandardCharsets.UTF_8));
|
||||
assertThat(versions.keySet()).containsExactly("checkpoints.jsonl", "links.jsonl", "manifest.json");
|
||||
assertThat(row.getValueString("object_versions")).isEqualTo(Jcs.string(versions));
|
||||
assertThat(STORE.get(WormAuditTables.EXPORT, row.getValueLong("id")).getValueString("object_versions"))
|
||||
.isEqualTo(row.getValueString("object_versions"));
|
||||
try(var files = new ExportFiles())
|
||||
{
|
||||
String linksKey = row.getValueString("object_prefix") + "links.jsonl";
|
||||
Files.write(files.file("links.jsonl"), storage.objects.get(linksKey));
|
||||
storage.put(linksKey, files.file("links.jsonl"), com.kingsrook.qbits.objectstorage.api.LockMode.valueOf(row.getValueString("lock_mode")), row.getValueInstant("retain_until"));
|
||||
var identical = verify(row);
|
||||
assertThat(identical.result().passed()).isFalse();
|
||||
assertThat(identical.result().getLinksChecked()).isEqualTo(1);
|
||||
assertThat(identical.result().getBreaks()).singleElement().satisfies(b ->
|
||||
assertThat(b.reason()).contains("object links.jsonl was overwritten after sealing"));
|
||||
for(String name : ExportFiles.NAMES)
|
||||
{
|
||||
String key = row.getValueString("object_prefix") + name;
|
||||
Files.writeString(files.file(name), "different bytes");
|
||||
String current = storage.put(key, files.file(name), com.kingsrook.qbits.objectstorage.api.LockMode.valueOf(row.getValueString("lock_mode")), row.getValueInstant("retain_until"));
|
||||
var report = verify(row);
|
||||
assertThat(report.result().passed()).isFalse();
|
||||
assertThat(report.result().getLinksChecked()).isEqualTo(1);
|
||||
assertThat(report.result().getBreaks()).allSatisfy(b -> assertThat(b.reason()).contains("was overwritten after sealing"));
|
||||
assertThat(report.result().getBreaks().toString()).contains("object " + name + " was overwritten after sealing (current version "
|
||||
+ current + ", sealed version " + versions.get(name) + " is still retained under Object Lock)");
|
||||
}
|
||||
}
|
||||
var load = new ExportProcesses.VerifyLoad(storage);
|
||||
load.runOnePage(new RunBackendStepInput().withRecords(List.of(row)), new RunBackendStepOutput());
|
||||
assertThat(load.getProcessSummary(new RunBackendStepOutput(), true).toString()).contains("ERROR", "was overwritten after sealing");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unversioned uploads and upload/HEAD races cannot create a sealed catalog row.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void refusesMissingVersionsAndUploadRaces() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
new WormAuditChainer(config).chainOnce(NOW);
|
||||
storage.omitVersion = true;
|
||||
for(String version : java.util.Arrays.asList(null, "", " ", "null"))
|
||||
{
|
||||
storage.missingVersion = version;
|
||||
var result = new SealedExporter(config, storage).run("USER", NOW);
|
||||
assertThat(result.failed()).isTrue();
|
||||
assertThat(result.message()).contains("returned no version id");
|
||||
assertThat(STORE.all(WormAuditTables.EXPORT)).isEmpty();
|
||||
}
|
||||
storage.omitVersion = false;
|
||||
storage.racePut = true;
|
||||
assertThat(new SealedExporter(config, storage).run("USER", NOW).message()).contains("version changed after upload");
|
||||
assertThat(STORE.all(WormAuditTables.EXPORT)).isEmpty();
|
||||
assertThat(ProcessesAndAlertsTest.Sink.raised).hasSize(5);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Missing catalog versions must never silently fall back to current content.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void refusesMissingRecordedVersion() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
QRecord row = seal();
|
||||
row.setValue("object_versions", "{}");
|
||||
assertThat(findings(row)).contains("object links.jsonl has no recorded sealed version id");
|
||||
row.setValue("object_versions", null);
|
||||
assertThat(verify(row).result().passed()).isFalse();
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise rangeSelectionNoCheckpointContiguousAndPurgeAnchor.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void rangeSelectionNoCheckpointContiguousAndPurgeAnchor() throws Exception
|
||||
{
|
||||
assertThat(new SealedExporter(config, storage).run("SCHEDULE", NOW).message()).isEqualTo("Nothing to seal: no new checkpoint");
|
||||
assertThat(ProcessesAndAlertsTest.Sink.beats).containsExactly("wormaudit.export.daily");
|
||||
audit(1);
|
||||
QRecord first = seal();
|
||||
assertThat(new SealedExporter(config, storage).run("USER", NOW).row()).isNull();
|
||||
audit(2);
|
||||
QRecord second = seal();
|
||||
assertThat(second.getValueLong("from_link_id")).isEqualTo(2L);
|
||||
assertThat(second.getValueString("prev_manifest_sha256")).isEqualTo(first.getValueString("manifest_sha256"));
|
||||
assertThat(verify(second).result().passed()).isTrue();
|
||||
|
||||
init(config);
|
||||
STORE.insert(WormAuditTables.PURGE, List.of(new QRecord().withValue("outcome", "PURGED").withValue("anchor_link_id", 10L).withValue("anchor_hash", Hashes.GENESIS)));
|
||||
audit(11);
|
||||
QRecord anchored = seal();
|
||||
assertThat(anchored.getValueLong("from_link_id")).isEqualTo(11L);
|
||||
assertThat(anchored.getValueString("prev_manifest_sha256")).isNull();
|
||||
assertThat(verify(anchored).result().passed()).isTrue();
|
||||
STORE.insert(WormAuditTables.PURGE, List.of(new QRecord().withValue("outcome", "PURGED").withValue("anchor_link_id", 20L).withValue("anchor_hash", Hashes.GENESIS)));
|
||||
assertThatThrownBy(() -> SealedExporter.scope(STORE)).hasMessage("Unexported links were purged");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise refusesUnsupportedAndIncorrectLockEchoAndPartialUpload.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void refusesUnsupportedAndIncorrectLockEchoAndPartialUpload() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
new WormAuditChainer(config).chainOnce(NOW);
|
||||
storage.supported = false;
|
||||
assertThat(new SealedExporter(config, storage).run("SCHEDULE", NOW).message()).contains("does not support Object Lock");
|
||||
assertThat(storage.objects).isEmpty();
|
||||
storage.supported = true;
|
||||
storage.echoMode = "COMPLIANCE";
|
||||
assertThat(new SealedExporter(config, storage).run("USER", NOW).message()).contains("did not echo GOVERNANCE");
|
||||
storage.echoMode = null;
|
||||
storage.shortenSeconds = 1;
|
||||
assertThat(new SealedExporter(config, storage).run("USER", NOW).failed()).isTrue();
|
||||
storage.shortenSeconds = 0;
|
||||
storage.objects.clear();
|
||||
storage.failPut = 2;
|
||||
assertThat(new SealedExporter(config, storage).run("SCHEDULE", NOW).message()).doesNotContain("CANARY").contains("storage or database");
|
||||
assertThat(STORE.all(WormAuditTables.EXPORT)).isEmpty();
|
||||
assertThat(ProcessesAndAlertsTest.Sink.raised).hasSize(4);
|
||||
assertThat(ProcessesAndAlertsTest.Sink.beats).hasSize(2);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise tamperedObjectsBrokenManifestChainAndLockRemoval.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void tamperedObjectsBrokenManifestChainAndLockRemoval() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
QRecord first = seal();
|
||||
String key = first.getValueString("object_prefix") + "links.jsonl";
|
||||
byte[] original = storage.objects.get(key);
|
||||
storage.objects.put(key, "tampered".getBytes(StandardCharsets.UTF_8));
|
||||
assertThat(findings(first)).contains("object links.jsonl hash does not match the manifest");
|
||||
storage.objects.put(key, original);
|
||||
storage.heads.get(key).withLockMode("COMPLIANCE");
|
||||
assertThat(findings(first)).contains("lacks the recorded GOVERNANCE retention");
|
||||
storage.heads.get(key).withLockMode(first.getValueString("lock_mode"));
|
||||
audit(2);
|
||||
QRecord second = seal();
|
||||
tamper(WormAuditTables.EXPORT, first.getValueLong("id"), "manifest_sha256", "f".repeat(64));
|
||||
assertThat(findings(second)).contains("prevManifestSha256 does not match");
|
||||
assertThat(findings(STORE.get(WormAuditTables.EXPORT, first.getValueLong("id")))).contains("manifest.json hash does not match");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise exactLiveDifferencesMissingRowsAndPurgedCopies.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void exactLiveDifferencesMissingRowsAndPurgedCopies() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
audit(2);
|
||||
QRecord row = seal();
|
||||
tamper("auditDetail", 1, "newValue", "CANARY-PRIVATE");
|
||||
assertThat(findings(row)).contains("link 1 (audit 1): live content differs from the sealed copy (sealed 2026-10-14)").doesNotContain("CANARY");
|
||||
remove(WormAuditTables.CHAIN, 2);
|
||||
assertThat(findings(row)).contains("link 2 is missing from the live chain");
|
||||
tamper(WormAuditTables.CHAIN, 1, "kind", "LATE_DETAIL");
|
||||
assertThat(findings(row)).contains("live chain kind differs");
|
||||
remove("audit", 1);
|
||||
assertThat(findings(row)).contains("live audit header is missing");
|
||||
String head = (String) ExportFiles.parse(storage.objects.get(row.getValueString("manifest_key"))).get("headHash");
|
||||
STORE.insert(WormAuditTables.PURGE, List.of(new QRecord().withValue("outcome", "PURGED").withValue("anchor_link_id", 2L).withValue("anchor_hash", head)));
|
||||
remove(WormAuditTables.CHAIN, 1);
|
||||
remove("audit", 2);
|
||||
var report = verify(row);
|
||||
assertThat(report.result().passed()).isTrue();
|
||||
assertThat(report.purged()).isEqualTo(2);
|
||||
var load = new ExportProcesses.VerifyLoad(storage);
|
||||
load.runOnePage(new RunBackendStepInput().withRecords(List.of(row)), new RunBackendStepOutput());
|
||||
assertThat(load.getProcessSummary(new RunBackendStepOutput(), true).toString()).contains("INFO", "2 links verified from the sealed copy only");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise refusesBadSourceAndDetectsMissingLiveDetails.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void refusesBadSourceAndDetectsMissingLiveDetails() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
new WormAuditChainer(config).chainOnce(NOW);
|
||||
tamper("auditDetail", 1, "newValue", "changed");
|
||||
assertThat(new SealedExporter(config, storage).run("USER", NOW).message()).contains("content differs at link 1");
|
||||
assertThat(storage.objects).isEmpty();
|
||||
tamper("auditDetail", 1, "newValue", "new");
|
||||
QRecord row = seal();
|
||||
remove("auditDetail", 1);
|
||||
assertThat(findings(row)).contains("link 1 (audit 1): live content is missing or invalid");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise spansPagesLateDetailsAndCheckpointBoundaries.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void spansPagesLateDetailsAndCheckpointBoundaries() throws Exception
|
||||
{
|
||||
config.withBatchSize(1100).withCheckpointEveryLinks(500);
|
||||
for(long id = 1; id <= 1100; id++)
|
||||
{
|
||||
audit(id);
|
||||
}
|
||||
new WormAuditChainer(config).chainOnce(NOW);
|
||||
// The final 100 links need the hourly checkpoint.
|
||||
new WormAuditChainer(config).chainOnce(NOW.plusSeconds(3601));
|
||||
QRecord row = seal();
|
||||
assertThat(row.getValueLong("to_link_id")).isEqualTo(1100);
|
||||
assertThat(verify(row).result().getLinksChecked()).isEqualTo(1100);
|
||||
detail(1, 1);
|
||||
new WormAuditChainer(config).chainOnce(NOW.plusSeconds(7202));
|
||||
new WormAuditChainer(config).chainOnce(NOW.plusSeconds(10803));
|
||||
QRecord late = seal();
|
||||
assertThat(late.getValueLong("from_link_id")).isEqualTo(1101);
|
||||
assertThat(verify(late).result().passed()).isTrue();
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise reviewLoadSelectionAndSchedulesUseStandardEtl.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void reviewLoadSelectionAndSchedulesUseStandardEtl() throws Exception
|
||||
{
|
||||
var instance = com.kingsrook.qqq.backend.core.context.QContext.getQInstance();
|
||||
assertThat(config.getExportBucketAlias()).isEqualTo("worm_audit");
|
||||
assertThat(config.getExportPrefix()).isEqualTo("worm-audit");
|
||||
assertThat(instance.getProcess(ExportProcesses.EXPORT).getTableName()).isEqualTo(WormAuditTables.EXPORT);
|
||||
assertThat(instance.getProcess(ExportProcesses.VERIFY).getMinInputRecords()).isEqualTo(1);
|
||||
assertThat(instance.getProcess(ExportProcesses.DAILY).getSchedule().getCronTimeZoneId()).isEqualTo("UTC");
|
||||
assertThat(instance.getProcess(ExportProcesses.DAILY).getSchedule().getCronExpression()).isEqualTo("0 0 2 * * ?");
|
||||
var output = new RunBackendStepOutput();
|
||||
var input = new RunBackendStepInput().withRecords(List.of(new QRecord().withValue("id", 0L)));
|
||||
var extract = new ExportProcesses.ExportExtract();
|
||||
assertThat(extract.doCount(input)).isEqualTo(1);
|
||||
extract.setRecordPipe(new com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe());
|
||||
extract.run(input, output);
|
||||
var review = new ExportProcesses.ExportReview();
|
||||
review.runOnePage(input, output);
|
||||
assertThat(review.getProcessSummary(output, false).toString()).contains("Nothing to seal");
|
||||
audit(1);
|
||||
new WormAuditChainer(config).chainOnce(NOW);
|
||||
review.runOnePage(input, output);
|
||||
assertThat(review.getProcessSummary(output, false).toString()).contains("1 chain links will be sealed", "1 checkpoints");
|
||||
assertThat(STORE.all(WormAuditTables.EXPORT)).isEmpty();
|
||||
var load = new ExportProcesses.ExportLoad(storage);
|
||||
assertThat(load.getProcessSummary(output, true)).isEmpty();
|
||||
load.runOnePage(input, output);
|
||||
load.postRun(new BackendStepPostRunInput(input), new BackendStepPostRunOutput(output));
|
||||
assertThat(load.getProcessSummary(output, true).toString()).contains("1 links sealed in export");
|
||||
var rows = STORE.all(WormAuditTables.EXPORT);
|
||||
var verifyInput = new RunBackendStepInput().withRecords(rows).addValue("recordIds", rows.getFirst().getValueString("id"));
|
||||
assertThat(new ExportProcesses.VerifyExtract().getQueryFilter(verifyInput).getCriteria()).isNotEmpty();
|
||||
var verifyReview = new ExportProcesses.VerifyReview();
|
||||
verifyReview.runOnePage(verifyInput, output);
|
||||
assertThat(verifyReview.getProcessSummary(output, false).toString()).contains("1 sealed exports");
|
||||
var verifyLoad = new ExportProcesses.VerifyLoad(storage);
|
||||
verifyLoad.runOnePage(verifyInput, output);
|
||||
assertThat(verifyLoad.getProcessSummary(output, true).toString()).contains("1 links verified");
|
||||
new ExportProcesses.Daily().run(input, output);
|
||||
assertThat(output.getValueString("outcome")).isEqualTo("PASS");
|
||||
audit(2);
|
||||
new WormAuditChainer(config).chainOnce(NOW);
|
||||
new ExportProcesses.Daily().run(input, output);
|
||||
assertThat(output.getValueString("outcome")).isEqualTo("FAIL");
|
||||
assertThat(ProcessesAndAlertsTest.Sink.beats).contains("wormaudit.export.daily");
|
||||
new ExportProcesses.ExportLoad();
|
||||
new ExportProcesses.VerifyLoad();
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise malformedJsonIsValueFreeAndConfigRejectsUnsafePrefixes.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void malformedJsonIsValueFreeAndConfigRejectsUnsafePrefixes() throws Exception
|
||||
{
|
||||
assertThatThrownBy(() -> ExportFiles.parse("{\"x\":\"CANARY\",\"x\":2}".getBytes(StandardCharsets.UTF_8)))
|
||||
.hasMessage("Invalid sealed JSON").hasNoCause();
|
||||
assertThatThrownBy(() -> config.withExportPrefix("../escape").validate()).isInstanceOf(IllegalArgumentException.class);
|
||||
config.withExportPrefix("evidence/audit").withExportBucketAlias("backup").withExportCron("0 5 2 * * ?").validate();
|
||||
assertThat(config.getExportBucketAlias()).isEqualTo("backup");
|
||||
try(var files = new ExportFiles())
|
||||
{
|
||||
assertThatThrownBy(() -> files.file("../escape")).isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Re-hash a tampered object and catalog to reach the independent signature checks.
|
||||
*******************************************************************************/
|
||||
QRecord rewrite(QRecord row, String name, byte[] bytes) throws Exception
|
||||
{
|
||||
String prefix = row.getValueString("object_prefix");
|
||||
storage.objects.put(prefix + name, bytes);
|
||||
Map<String, Object> manifest = ExportFiles.parse(storage.objects.get(prefix + "manifest.json"));
|
||||
Map<String, Object> hashes = new LinkedHashMap<>((Map<String, Object>) manifest.get("files"));
|
||||
hashes.put(name, java.util.HexFormat.of().formatHex(Hashes.digest(bytes)));
|
||||
manifest.put("files", hashes);
|
||||
byte[] encoded = Jcs.bytes(manifest);
|
||||
storage.objects.put(prefix + "manifest.json", encoded);
|
||||
tamper(WormAuditTables.EXPORT, row.getValueLong("id"), "manifest_sha256", java.util.HexFormat.of().formatHex(Hashes.digest(encoded)));
|
||||
return (STORE.get(WormAuditTables.EXPORT, row.getValueLong("id")));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise independentSignaturesMerkleAndSealedContentChecks.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void independentSignaturesMerkleAndSealedContentChecks() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
QRecord row = seal();
|
||||
String prefix = row.getValueString("object_prefix");
|
||||
byte[] originalCheckpoint = storage.objects.get(prefix + "checkpoints.jsonl");
|
||||
Map<String, Object> checkpoint = ExportFiles.parse(originalCheckpoint);
|
||||
checkpoint.put("signature", java.util.Base64.getEncoder().encodeToString(new byte[64]));
|
||||
row = rewrite(row, "checkpoints.jsonl", (Jcs.string(checkpoint) + "\n").getBytes(StandardCharsets.UTF_8));
|
||||
assertThat(findings(row)).contains("checkpoint 1: signature does not verify");
|
||||
checkpoint = ExportFiles.parse(originalCheckpoint);
|
||||
checkpoint.put("merkle_root", "a".repeat(64));
|
||||
row = rewrite(row, "checkpoints.jsonl", (Jcs.string(checkpoint) + "\n").getBytes(StandardCharsets.UTF_8));
|
||||
assertThat(findings(row)).contains("Merkle root or head hash differs");
|
||||
row = rewrite(row, "checkpoints.jsonl", originalCheckpoint);
|
||||
Map<String, Object> document = ExportFiles.parse(storage.objects.get(prefix + "links.jsonl"));
|
||||
Map<String, Object> content = (Map<String, Object>) document.get("content");
|
||||
((Map<String, Object>) content.get("audit")).put("message", "CANARY-SEALED");
|
||||
row = rewrite(row, "links.jsonl", (Jcs.string(document) + "\n").getBytes(StandardCharsets.UTF_8));
|
||||
assertThat(findings(row)).contains("sealed content differs from the chained hash").doesNotContain("CANARY");
|
||||
storage.objects.remove(prefix + "links.jsonl");
|
||||
assertThat(findings(row)).contains("cannot be read or has invalid structure").doesNotContain("CANARY");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise concurrentExportersDoNotOverlapAndMultipleSelectionsPersistSeparately.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void concurrentExportersDoNotOverlapAndMultipleSelectionsPersistSeparately() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
new WormAuditChainer(config).chainOnce(NOW);
|
||||
var instance = com.kingsrook.qqq.backend.core.context.QContext.getQInstance();
|
||||
var workers = java.util.concurrent.Executors.newFixedThreadPool(2);
|
||||
try
|
||||
{
|
||||
var jobs = new java.util.ArrayList<java.util.concurrent.Future<SealedExporter.Result>>();
|
||||
for(int i = 0; i < 2; i++)
|
||||
{
|
||||
jobs.add(workers.submit(() ->
|
||||
{
|
||||
com.kingsrook.qqq.backend.core.context.QContext.init(instance, new com.kingsrook.qqq.backend.core.model.session.QSession());
|
||||
try
|
||||
{
|
||||
return (new SealedExporter(config, storage).run("USER", NOW));
|
||||
}
|
||||
finally
|
||||
{
|
||||
com.kingsrook.qqq.backend.core.context.QContext.clear();
|
||||
}
|
||||
}));
|
||||
}
|
||||
int sealed = 0;
|
||||
for(var job : jobs)
|
||||
{
|
||||
var result = job.get(10, java.util.concurrent.TimeUnit.SECONDS);
|
||||
assertThat(result.failed()).as(result.message()).isFalse();
|
||||
sealed += result.row() == null ? 0 : 1;
|
||||
}
|
||||
assertThat(sealed).isEqualTo(1);
|
||||
}
|
||||
finally
|
||||
{
|
||||
workers.shutdownNow();
|
||||
}
|
||||
audit(2);
|
||||
seal();
|
||||
var load = new ExportProcesses.VerifyLoad(storage);
|
||||
load.runOnePage(new RunBackendStepInput().withRecords(STORE.all(WormAuditTables.EXPORT)), new RunBackendStepOutput());
|
||||
assertThat(STORE.all(WormAuditTables.VERIFICATION)).hasSize(2)
|
||||
.allMatch(r -> "PASS".equals(r.getValueString("outcome")));
|
||||
assertThat(STORE.all(WormAuditTables.VERIFICATION)).extracting(r -> r.getValueLong("export_id")).doesNotHaveDuplicates();
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise h2TransactionRoundTripAndFailedPutHasNoCatalogRow.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void h2TransactionRoundTripAndFailedPutHasNoCatalogRow() throws Exception
|
||||
{
|
||||
String url = "jdbc:h2:mem:exports_" + java.util.UUID.randomUUID() + ";DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE";
|
||||
var instance = RdbmsTestSupport.instance(url, "sa", "", config, false);
|
||||
try(var connection = java.sql.DriverManager.getConnection(url, "sa", ""))
|
||||
{
|
||||
RdbmsTestSupport.createTables(connection, instance, false);
|
||||
com.kingsrook.qqq.backend.core.context.QContext.init(instance, new com.kingsrook.qqq.backend.core.model.session.QSession());
|
||||
WormAuditTestSupport.lookups();
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
QRecord row = seal();
|
||||
assertThat(verify(row).result().passed()).isTrue();
|
||||
assertThat(verify(STORE.get(WormAuditTables.EXPORT, row.getValueLong("id"))).result().passed()).isTrue();
|
||||
connection.createStatement().execute("UPDATE audit_detail SET new_value='changed' WHERE id=1");
|
||||
assertThat(findings(row)).contains("link 1 (audit 1): live content differs");
|
||||
audit(2);
|
||||
new WormAuditChainer(config).chainOnce(NOW);
|
||||
storage.failPut = storage.objects.size() + 1;
|
||||
assertThat(new SealedExporter(config, storage).run("USER", NOW).failed()).isTrue();
|
||||
assertThat(STORE.all(WormAuditTables.EXPORT)).hasSize(1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise realAdapterRequiresBackupRoleAndPositiveCapability.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void realAdapterRequiresBackupRoleAndPositiveCapability() throws Exception
|
||||
{
|
||||
var bucket = new com.kingsrook.qbits.objectstorage.config.StorageBucketConfig("worm_audit")
|
||||
.withConnection("test").withBucketName("test-bucket").withIndexed(false)
|
||||
.withRole(com.kingsrook.qbits.objectstorage.config.BucketRole.BACKUP);
|
||||
System.setProperty("worm.unit.access", "unit-access");
|
||||
System.setProperty("worm.unit.secret", "unit-secret");
|
||||
var server = com.sun.net.httpserver.HttpServer.create(new java.net.InetSocketAddress("127.0.0.1", 0), 0);
|
||||
var responseVersion = new java.util.concurrent.atomic.AtomicReference<>("sealed-version");
|
||||
var requested = new java.util.concurrent.CopyOnWriteArrayList<String>();
|
||||
server.createContext("/test-bucket/links.jsonl", exchange ->
|
||||
{
|
||||
requested.add(exchange.getRequestURI().getRawQuery());
|
||||
exchange.getResponseHeaders().add("x-amz-version-id", responseVersion.get());
|
||||
byte[] bytes = "sealed fixture".getBytes(StandardCharsets.UTF_8);
|
||||
exchange.sendResponseHeaders(200, bytes.length);
|
||||
try(var output = exchange.getResponseBody())
|
||||
{
|
||||
output.write(bytes);
|
||||
}
|
||||
});
|
||||
server.start();
|
||||
try
|
||||
{
|
||||
com.kingsrook.qbits.objectstorage.api.ObjectStorage.configureStandalone(
|
||||
new com.kingsrook.qbits.objectstorage.ObjectStorageQBitConfig().withBackendName("memory")
|
||||
.withAllowedSecretRefPatterns(List.of("prop.worm.unit.*"))
|
||||
.withConnection(new com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig("test")
|
||||
.withEndpoint("http://127.0.0.1:" + server.getAddress().getPort()).withRegion("us-east-1")
|
||||
.withAddressingStyle(com.kingsrook.qbits.objectstorage.config.AddressingStyle.PATH)
|
||||
.withProviderType(com.kingsrook.qbits.objectstorage.config.ProviderType.MINIO)
|
||||
.withAccessKeyRef("${prop.worm.unit.access}").withSecretKeyRef("${prop.worm.unit.secret}"))
|
||||
.withBucket(bucket));
|
||||
var adapter = new com.kingsrook.qbits.wormaudit.export.ObjectStorageSealedStore("worm_audit", "unit");
|
||||
assertThat(adapter.supportsLock()).isFalse();
|
||||
com.kingsrook.qbits.objectstorage.internal.CapabilityStore.record(bucket,
|
||||
com.kingsrook.qbits.objectstorage.api.Capability.OBJECT_LOCK,
|
||||
com.kingsrook.qbits.objectstorage.api.CapabilityStatus.SUPPORTED, "unit fixture");
|
||||
assertThat(adapter.supportsLock()).isTrue();
|
||||
try(var input = adapter.getVersion("links.jsonl", "sealed-version"))
|
||||
{
|
||||
assertThat(new String(input.readAllBytes(), StandardCharsets.UTF_8)).isEqualTo("sealed fixture");
|
||||
}
|
||||
responseVersion.set("different-version");
|
||||
assertThatThrownBy(() -> adapter.getVersion("links.jsonl", "sealed-version"))
|
||||
.hasMessage("Storage returned a different object version");
|
||||
assertThatThrownBy(() -> adapter.getVersion("links.jsonl", null)).hasMessage("Missing sealed object version");
|
||||
assertThat(requested).containsExactly("versionId=sealed-version", "versionId=sealed-version");
|
||||
bucket.withRole(com.kingsrook.qbits.objectstorage.config.BucketRole.PRIVATE);
|
||||
assertThat(adapter.supportsLock()).isFalse();
|
||||
}
|
||||
finally
|
||||
{
|
||||
server.stop(0);
|
||||
com.kingsrook.qbits.objectstorage.internal.ObjectStorageRuntime.reset();
|
||||
System.clearProperty("worm.unit.access");
|
||||
System.clearProperty("worm.unit.secret");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QAuthenticationException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
|
||||
import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleInterface;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test authentication module that behaves like the token-based core modules
|
||||
** (OAuth2, Auth0): a QSystemUserSession is always valid, a session it created
|
||||
** is valid, and any other session has no token and fails validation with the
|
||||
** same message the OAuth2 module produces (JWTDecodeException: The token is
|
||||
** null).
|
||||
*******************************************************************************/
|
||||
public class TokenRequiredAuthenticationModule implements QAuthenticationModuleInterface
|
||||
{
|
||||
private static final Set<String> ISSUED = ConcurrentHashMap.newKeySet();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Issue a session that carries a token.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QSession createSession(QInstance qInstance, Map<String, String> context) throws QAuthenticationException
|
||||
{
|
||||
QSession session = new QSession();
|
||||
ISSUED.add(session.getUuid());
|
||||
return (session);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Accept the system user and issued sessions; reject tokenless sessions.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean isSessionValid(QInstance instance, QSession session)
|
||||
{
|
||||
if(session instanceof QSystemUserSession || ISSUED.contains(session.getUuid()))
|
||||
{
|
||||
return (true);
|
||||
}
|
||||
throw (new IllegalStateException("The token is null"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.ExportProcesses;
|
||||
import com.kingsrook.qbits.wormaudit.processes.WormAuditProcesses;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Regression: production must work on a host whose default authentication
|
||||
** rejects tokenless sessions (kof22 uses OAuth2). produce() reads the settings
|
||||
** singleton for the startup schedules; under a plain new QSession() that read
|
||||
** failed with "The token is null" and the host could not start.
|
||||
*******************************************************************************/
|
||||
class TokenRequiredHostTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Clear the context and the memory store.
|
||||
*******************************************************************************/
|
||||
@AfterEach
|
||||
void tearDown()
|
||||
{
|
||||
MemoryRecordStore.fullReset();
|
||||
QContext.clear();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The test module really rejects a tokenless session, so the tests below
|
||||
** prove the qbit does not use one.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void moduleRejectsTokenlessSession() throws Exception
|
||||
{
|
||||
MemoryRecordStore.fullReset();
|
||||
QInstance instance = tokenRequiredHost(new WormAuditQBitConfig());
|
||||
QContext.init(instance, new QSession());
|
||||
assertThatThrownBy(() -> new AuditStore(null).get(WormAuditTables.SETTINGS, 1))
|
||||
.hasStackTraceContaining("The token is null");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** produce() succeeds and still honors the persisted settings row for the
|
||||
** startup schedules, read as the system user.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void produceReadsPersistedSettingsAsSystemUser() throws Exception
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// first start: an anonymous host stores an edited settings row //
|
||||
////////////////////////////////////////////////////////////////////
|
||||
MemoryRecordStore.fullReset();
|
||||
WormAuditQBitConfig seedConfig = new WormAuditQBitConfig();
|
||||
QInstance seedHost = WormAuditTestSupport.core(memory());
|
||||
new WormAuditQBitProducer().withQBitConfig(seedConfig.withBackendName("memory")).produce(seedHost);
|
||||
QContext.init(seedHost, new QSession());
|
||||
QRecord row = WormAuditSettings.defaults(seedConfig)
|
||||
.withValue("chain_interval_seconds", 45)
|
||||
.withValue("verify_cron", "0 15 4 * * ?")
|
||||
.withValue("export_cron", "0 45 1 * * ?");
|
||||
new AuditStore(null).insert(WormAuditTables.SETTINGS, List.of(row));
|
||||
QContext.clear();
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// restart on a token-required host, with no session in context, as //
|
||||
// when a host builds its instance before any request arrives //
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
QInstance instance = tokenRequiredHost(new WormAuditQBitConfig());
|
||||
|
||||
assertThat(instance.getProcess(WormAuditProcesses.CHAIN).getSchedule().getRepeatSeconds()).isEqualTo(45);
|
||||
assertThat(instance.getProcess(WormAuditProcesses.DAILY).getSchedule().getCronExpression()).isEqualTo("0 15 4 * * ?");
|
||||
assertThat(instance.getProcess(ExportProcesses.DAILY).getSchedule().getCronExpression()).isEqualTo("0 45 1 * * ?");
|
||||
assertThat(QContext.getQSession()).isNull();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** With no settings row, production falls back to the configured values.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void produceFallsBackToConfigWithoutRow() throws Exception
|
||||
{
|
||||
MemoryRecordStore.fullReset();
|
||||
QInstance instance = tokenRequiredHost(new WormAuditQBitConfig().withChainIntervalSeconds(17));
|
||||
assertThat(instance.getProcess(WormAuditProcesses.CHAIN).getSchedule().getRepeatSeconds()).isEqualTo(17);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build a host whose default authentication requires a token, and produce.
|
||||
*******************************************************************************/
|
||||
private static QInstance tokenRequiredHost(WormAuditQBitConfig config) throws Exception
|
||||
{
|
||||
QInstance instance = WormAuditTestSupport.core(memory());
|
||||
instance.withInstanceDefaultAuthentication(new QAuthenticationMetaData()
|
||||
.withName("token-required")
|
||||
.withType(QAuthenticationType.MOCK)
|
||||
.withModuleCodeReference(new QCodeReference(TokenRequiredAuthenticationModule.class)));
|
||||
new WormAuditQBitProducer().withQBitConfig(config.withBackendName("memory")).produce(instance);
|
||||
return (instance);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The memory backend.
|
||||
*******************************************************************************/
|
||||
private static QBackendMetaData memory()
|
||||
{
|
||||
return (new QBackendMetaData().withName("memory").withBackendType(MemoryBackendModule.class));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,366 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import java.sql.DriverManager;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
|
||||
import com.kingsrook.qbits.wormaudit.app.WormAuditApp;
|
||||
import com.kingsrook.qbits.wormaudit.app.WormAuditHomeData;
|
||||
import com.kingsrook.qbits.wormaudit.app.WormAuditHomeView;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.demo.MemorySealedStore;
|
||||
import com.kingsrook.qbits.wormaudit.demo.WormAuditDemoServer;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedExportVerifier;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedExporter;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.VerificationRuns;
|
||||
import com.kingsrook.qbits.wormaudit.verify.VerificationResult;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
|
||||
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
|
||||
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.OLD;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.STORE;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.detail;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.lookups;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.tamper;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Home acceptance: truthful summaries, permissions, privacy and real demo data.
|
||||
*******************************************************************************/
|
||||
class WormAuditHomeTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Empty tables do not imply successful verification or export coverage.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void emptyStatesAndWarnings() throws Exception
|
||||
{
|
||||
var config = new WormAuditQBitConfig();
|
||||
init(config);
|
||||
var data = WormAuditHomeData.load(config, new MemorySealedStore(), OLD);
|
||||
assertThat(data.links()).isZero();
|
||||
assertThat(data.backlog()).isZero();
|
||||
assertThat(data.settling()).isZero();
|
||||
assertThat(data.anchor()).isNull();
|
||||
assertThat(data.oldestAudit()).isNull();
|
||||
assertThat(data.nextPurge()).isNull();
|
||||
assertThat(data.exportAgeHours()).isNull();
|
||||
assertThat(data.warnings()).contains("No signing key configured", "No trusted public keys configured", "Daily verification has never run");
|
||||
String html = WormAuditHomeView.render(data, "/app");
|
||||
assertThat(html).contains("Chain status", "Export status", "Retention", "Recent purges", "Setup warnings",
|
||||
"No links chained yet", "No verification has run yet", "Unknown until verification runs", "No sealed exports yet",
|
||||
"No sealed ranges to check", "No live audits", "No purge requests yet");
|
||||
assertThat(WormAuditHomeView.renderHealthCard(data, "/app/worm_audit")).contains("Not verified yet");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Strict settle boundary and AUDIT anti-join, including late-detail links.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void chainBacklogRetentionAndRealFailureRecovery() throws Exception
|
||||
{
|
||||
var config = ChainVerificationTest.signed();
|
||||
init(config);
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
Instant now = OLD.plusSeconds(120);
|
||||
new WormAuditChainer(config).chainOnce(now);
|
||||
audit(2);
|
||||
audit(3);
|
||||
tamper("audit", 3, "timestamp", now.minusSeconds(60));
|
||||
var data = WormAuditHomeData.load(config, new MemorySealedStore(), now);
|
||||
assertThat(data.links()).isEqualTo(1);
|
||||
assertThat(data.lastLinkAt()).isEqualTo(now);
|
||||
assertThat(data.backlog()).isEqualTo(1);
|
||||
assertThat(data.settling()).isEqualTo(1);
|
||||
assertThat(data.oldestAudit()).isEqualTo(OLD.plusSeconds(1));
|
||||
assertThat(data.nextPurge()).isEqualTo(OLD.plusSeconds(1).atZone(ZoneOffset.UTC).plusYears(6).toInstant());
|
||||
new WormAuditChainer(config).chainOnce(now.plusSeconds(61));
|
||||
tamper("auditDetail", 1, "newValue", "forced mismatch");
|
||||
var failure = new WormAuditVerifier(config).verify(null, now.plusSeconds(61));
|
||||
QRecord failed = VerificationRuns.save(failure, "SCHEDULE", null, now.plusSeconds(62));
|
||||
assertThat(failure.passed()).isFalse();
|
||||
data = WormAuditHomeData.load(config, new MemorySealedStore(), now.plusSeconds(63));
|
||||
assertThat(data.openBreak().getValueLong("id")).isEqualTo(failed.getValueLong("id"));
|
||||
String html = WormAuditHomeView.render(data, "/app/admin/worm");
|
||||
assertThat(html).contains("/app/admin/worm/worm_audit_verification/1", "/app/admin/worm/worm_audit_chain/1", "content differs from the chained hash");
|
||||
assertThat(WormAuditHomeView.renderHealthCard(data, "/app/worm_audit")).contains("Chain breaks reported");
|
||||
// A passing selected run must not clear a full-chain failure.
|
||||
VerificationRuns.save(new WormAuditVerifier(config).verify(Set.of(2L), now.plusSeconds(64)), "USER", null, now.plusSeconds(65));
|
||||
assertThat(WormAuditHomeData.load(config, new MemorySealedStore(), now.plusSeconds(66)).openBreak()).isNotNull();
|
||||
tamper("auditDetail", 1, "newValue", "new");
|
||||
VerificationRuns.save(new WormAuditVerifier(config).verify(null, now.plusSeconds(67)), "SCHEDULE", null, now.plusSeconds(68));
|
||||
data = WormAuditHomeData.load(config, new MemorySealedStore(), now.plusSeconds(69));
|
||||
assertThat(data.openBreak()).isNull();
|
||||
assertThat(data.warnings()).hasSize(2);
|
||||
assertThat(WormAuditHomeView.render(data, "/app")).contains("Setup warnings", "GOVERNANCE", "30 days");
|
||||
assertThat(WormAuditHomeView.renderHealthCard(data, "/app/worm_audit")).contains("Setup needs attention");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Genuine exports, trailing lag and a corruption fixture for a coverage hole.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void sealedRangesLagGapsAndPurges() throws Exception
|
||||
{
|
||||
var storage = new MemorySealedStore();
|
||||
var config = ChainVerificationTest.signed().withSealedStoreSupplier(() -> storage);
|
||||
init(config);
|
||||
for(long id = 1; id <= 4; id++)
|
||||
{
|
||||
audit(id);
|
||||
}
|
||||
Instant now = OLD.plusSeconds(180);
|
||||
new WormAuditChainer(config).chainOnce(now);
|
||||
var export = new SealedExporter(config).run("USER", now);
|
||||
assertThat(export.failed()).isFalse();
|
||||
assertThat(new SealedExportVerifier(config).verify(export.row(), "USER").result().passed()).isTrue();
|
||||
audit(5);
|
||||
new WormAuditChainer(config).chainOnce(now.plusSeconds(120));
|
||||
var data = WormAuditHomeData.load(config, null, now.plus(Duration.ofHours(3)));
|
||||
assertThat(data.exportLag()).isEqualTo(1);
|
||||
assertThat(data.exportAgeHours()).isEqualTo(3);
|
||||
assertThat(data.exportGaps()).isEmpty();
|
||||
assertThat(data.lockCapability()).isEqualTo("SUPPORTED");
|
||||
assertThat(data.bucketAlias()).isEqualTo("worm_audit");
|
||||
assertThat(data.lastExport().getValueString("lock_mode")).isEqualTo("GOVERNANCE");
|
||||
assertThat(WormAuditHomeView.render(data, "/app")).contains("/app/worm_audit_export/1", "GOVERNANCE", "Retain until");
|
||||
// Corruption fixture only: the real exporter never creates this hole.
|
||||
tamper(WormAuditTables.EXPORT, 1, "from_link_id", 3L);
|
||||
assertThat(WormAuditHomeData.load(config, storage, now).exportGaps()).containsExactly("1–2");
|
||||
for(long id = 1; id <= 12; id++)
|
||||
{
|
||||
STORE.insert(WormAuditTables.PURGE, List.of(new QRecord().withValue("id", id).withValue("requested_at", now.plusSeconds(id))
|
||||
.withValue("requested_by", "<operator>").withValue("cutoff", OLD).withValue("outcome", id == 1 ? "PURGED" : "REFUSED")
|
||||
.withValue("reason", "Retention not elapsed").withValue("anchor_link_id", 2L).withValue("anchor_hash", "abcd".repeat(16))
|
||||
.withValue("audits_deleted", 0L).withValue("details_deleted", 0L).withValue("links_deleted", 0L)));
|
||||
}
|
||||
data = WormAuditHomeData.load(config, storage, now.plusSeconds(20));
|
||||
assertThat(data.anchor().getValueLong("anchor_link_id")).isEqualTo(2);
|
||||
assertThat(data.recentPurges()).hasSize(10).extracting(r -> r.getValueLong("id")).containsExactly(12L, 11L, 10L, 9L, 8L, 7L, 6L, 5L, 4L, 3L);
|
||||
assertThat(data.exportGaps()).isEmpty();
|
||||
assertThat(WormAuditHomeView.render(data, "/app")).contains("abcdabcdabcd…", "<operator>", "REFUSED").doesNotContain("abcd".repeat(16));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unknown/unsupported capability and stale schedule diagnostics stay value-free.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void capabilityAndDailyWarnings() throws Exception
|
||||
{
|
||||
var config = ChainVerificationTest.signed();
|
||||
init(config);
|
||||
var unsupported = new MemorySealedStore()
|
||||
{
|
||||
@Override
|
||||
public boolean supportsLock()
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
};
|
||||
var unknown = new MemorySealedStore()
|
||||
{
|
||||
@Override
|
||||
public CapabilityStatus lockCapability()
|
||||
{
|
||||
return (CapabilityStatus.UNKNOWN);
|
||||
}
|
||||
};
|
||||
var error = new MemorySealedStore()
|
||||
{
|
||||
@Override
|
||||
public boolean supportsLock()
|
||||
{
|
||||
throw (new IllegalStateException("CANARY-provider-secret"));
|
||||
}
|
||||
};
|
||||
assertThat(WormAuditHomeData.load(config, unsupported, OLD).lockCapability()).isEqualTo("UNSUPPORTED");
|
||||
assertThat(WormAuditHomeData.load(config, unknown, OLD).lockCapability()).isEqualTo("UNKNOWN");
|
||||
String html = WormAuditHomeView.render(WormAuditHomeData.load(config, error, OLD), "/app");
|
||||
assertThat(html).contains("UNKNOWN", "Object Lock capability check failed").doesNotContain("CANARY", "provider-secret");
|
||||
VerificationRuns.save(new VerificationResult(OLD), "SCHEDULE", null, OLD);
|
||||
assertThat(WormAuditHomeData.load(config, unsupported, OLD.plus(Duration.ofHours(26))).warnings()).doesNotContain("Daily verification last ran more than 26 hours ago");
|
||||
var stale = WormAuditHomeData.load(config, unsupported, OLD.plus(Duration.ofHours(26)).plusSeconds(1));
|
||||
assertThat(stale.warnings()).contains("Daily verification last ran more than 26 hours ago");
|
||||
assertThat(WormAuditHomeView.renderHealthCard(stale, "/app/worm_audit")).contains("Setup needs attention");
|
||||
config.withTrustedPublicKeys(Map.of());
|
||||
assertThat(WormAuditHomeData.load(config, unsupported, OLD).warnings()).contains("No trusted public keys configured");
|
||||
// USER and SEALED_EXPORT runs cannot refresh the daily CHAIN heartbeat.
|
||||
VerificationRuns.save(new VerificationResult(OLD.plus(Duration.ofDays(2))), "USER", null, OLD.plus(Duration.ofDays(2)));
|
||||
VerificationRuns.save(new VerificationResult(OLD.plus(Duration.ofDays(2))), "SCHEDULE", 1L, OLD.plus(Duration.ofDays(2)));
|
||||
assertThat(WormAuditHomeData.load(config, unsupported, OLD.plus(Duration.ofDays(2))).warnings()).contains("Daily verification last ran more than 26 hours ago");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** No portion of raw audit content belongs in either HTML widget.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void canaryNeverAppearsAndWidgetsRequireReadPermission() throws Exception
|
||||
{
|
||||
var config = ChainVerificationTest.signed().withAppName("evidence").withAppLabel("Evidence").withTablePathPrefix("/app");
|
||||
var instance = init(config);
|
||||
instance.addApp(new QAppMetaData().withName("admin").withLabel("Admin"));
|
||||
config.withParentAppName("admin");
|
||||
instance.getApp("evidence").setParentAppName("admin");
|
||||
String canary = "CANARY-PHI-" + UUID.randomUUID();
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
tamper("audit", 1, "message", canary);
|
||||
tamper("auditDetail", 1, "message", canary);
|
||||
tamper("auditDetail", 1, "oldValue", canary);
|
||||
tamper("auditDetail", 1, "newValue", canary);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
var input = new RenderWidgetInput().withWidgetMetaData(instance.getWidget(WormAuditApp.HOME_WIDGET));
|
||||
assertThatThrownBy(() -> new WormAuditApp.HomeRenderer().render(input)).isInstanceOf(QPermissionDeniedException.class);
|
||||
assertThatThrownBy(() -> new WormAuditApp.HealthCardRenderer().render(input)).isInstanceOf(QPermissionDeniedException.class);
|
||||
QContext.getQSession().withPermissions(WormAuditTables.PURGE + ".read");
|
||||
String home = ((RawHTML) new WormAuditApp.HomeRenderer().render(input).getWidgetData()).getHtml();
|
||||
String card = ((RawHTML) new WormAuditApp.HealthCardRenderer().render(input).getWidgetData()).getHtml();
|
||||
for(int start = 0; start <= canary.length() - 5; start++)
|
||||
{
|
||||
assertThat(home + card).doesNotContain(canary.substring(start, start + 5));
|
||||
}
|
||||
assertThat(home + card).contains("overflow-wrap:anywhere", "color:inherit").doesNotContain("overflow-x", "<table", "#fff");
|
||||
assertThat(card).contains("href=\"/app/admin/evidence\"");
|
||||
assertThat(instance.getApp("evidence").getLabel()).isEqualTo("Evidence");
|
||||
assertThat(instance.getApp("evidence").getParentAppName()).isEqualTo("admin");
|
||||
assertThat(instance.getApp("evidence").getWidgets()).containsExactly(WormAuditApp.HOME_WIDGET);
|
||||
assertThat(instance.getApp("evidence").getChildren()).extracting(c -> c.getName()).containsExactlyElementsOf(WormAuditTables.NAV);
|
||||
assertThat(instance.getApp("evidence").getChildren()).extracting(c -> c.getLabel())
|
||||
.containsExactly("Verifications", "Sealed Exports", "Checkpoints", "Audit Chain", "Detail Coverage", "Purges", "Settings");
|
||||
assertThat(instance.getWidget(WormAuditApp.HOME_WIDGET).getGridColumns()).isEqualTo(12);
|
||||
assertThat(instance.getWidget(WormAuditApp.HOME_WIDGET).getIsCard()).isFalse();
|
||||
assertThat(instance.getWidget(WormAuditApp.HEALTH_CARD_WIDGET).getIsCard()).isTrue();
|
||||
new QInstanceValidator().validate(instance);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Embedded widgets retain direct authorization without a standalone app.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void embeddedWidgetsRequireReadPermission() throws Exception
|
||||
{
|
||||
var config = ChainVerificationTest.signed().withIncludeApp(false).withAppName("audit").withAppLabel("Audit");
|
||||
var instance = init(config);
|
||||
assertThat(instance.getApp("worm_audit")).isNull();
|
||||
assertThat(instance.getApp("audit")).isNull();
|
||||
var input = new RenderWidgetInput().withWidgetMetaData(instance.getWidget(WormAuditApp.HEALTH_CARD_WIDGET));
|
||||
QContext.getQSession().withPermissions("admin.hasAccess");
|
||||
assertThatThrownBy(() -> new WormAuditApp.HealthCardRenderer().render(input)).isInstanceOf(QPermissionDeniedException.class);
|
||||
assertThatThrownBy(() -> new WormAuditApp.HomeRenderer().render(input)).isInstanceOf(QPermissionDeniedException.class);
|
||||
QContext.getQSession().withPermissions(WormAuditTables.CHAIN + ".read");
|
||||
String card = ((RawHTML) new WormAuditApp.HealthCardRenderer().render(input).getWidgetData()).getHtml();
|
||||
assertThat(card).contains("href=\"/app/audit\"");
|
||||
assertThat(new WormAuditApp.HomeRenderer().render(input).getWidgetData()).isInstanceOf(RawHTML.class);
|
||||
QContext.setQSession(new QSession());
|
||||
assertThatThrownBy(() -> new WormAuditApp.HealthCardRenderer().render(input)).isInstanceOf(QPermissionDeniedException.class);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise the real SQL anti-join without Docker, including its exact boundary.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void sqlSummaryUsesTheWriterAntiJoin() throws Exception
|
||||
{
|
||||
String url = "jdbc:h2:mem:home_" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE;INIT=SET TIME ZONE 'UTC'";
|
||||
var config = ChainVerificationTest.signed();
|
||||
var instance = RdbmsTestSupport.instance(url, "sa", "", config, false);
|
||||
try(var connection = DriverManager.getConnection(url, "sa", ""))
|
||||
{
|
||||
RdbmsTestSupport.createTables(connection, instance, false);
|
||||
QContext.init(instance, new QSession());
|
||||
lookups();
|
||||
audit(1);
|
||||
new WormAuditChainer(config).chainOnce(OLD.plusSeconds(100));
|
||||
audit(2);
|
||||
audit(3);
|
||||
var data = WormAuditHomeData.load(config, new MemorySealedStore(), OLD.plusSeconds(63));
|
||||
assertThat(data.links()).isEqualTo(1);
|
||||
assertThat(data.backlog()).isEqualTo(1);
|
||||
assertThat(data.settling()).isEqualTo(1);
|
||||
assertThat(data.warnings()).contains("Daily verification has never run");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The demo's history comes from real engines, including FAIL, recovery and PHI tokens.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void demoSeedsRealHistory() throws Exception
|
||||
{
|
||||
MemoryRecordStore.fullReset();
|
||||
var application = new WormAuditDemoServer();
|
||||
var instance = application.defineQInstance();
|
||||
new QInstanceValidator().validate(instance);
|
||||
application.seed(instance);
|
||||
QContext.init(instance, new QSession());
|
||||
var data = WormAuditHomeData.load();
|
||||
assertThat(data.links()).isGreaterThanOrEqualTo(350);
|
||||
assertThat(data.lastVerification().getValueString("outcome")).isEqualTo("PASS");
|
||||
assertThat(data.openBreak()).isNull();
|
||||
assertThat(data.exportGaps()).isEmpty();
|
||||
assertThat(data.exportLag()).isEqualTo(5);
|
||||
assertThat(data.backlog()).isEqualTo(2);
|
||||
assertThat(data.settling()).isEqualTo(1);
|
||||
assertThat(data.warnings()).hasSize(2);
|
||||
assertThat(data.recentPurges()).extracting(r -> r.getValueString("outcome")).containsExactly("REFUSED", "PURGED");
|
||||
assertThat(STORE.where(WormAuditTables.VERIFICATION, "outcome", "FAIL")).hasSize(1).allSatisfy(row ->
|
||||
{
|
||||
assertThat(row.getValueLong("first_break_link_id")).isPositive();
|
||||
assertThat(row.getValueLong("first_break_audit_id")).isPositive();
|
||||
});
|
||||
assertThat(STORE.all("auditDetail")).filteredOn(r -> "diagnosis".equals(r.getValueString("fieldName")))
|
||||
.isNotEmpty().allSatisfy(row -> assertThat(row.getValueString("newValue")).startsWith("hmac:v1:test:"));
|
||||
var config = VerificationRuns.config();
|
||||
// Settle and chain the final backlog before checking the entire live suffix.
|
||||
new WormAuditChainer(config).chainOnce(Instant.now().plusSeconds(120));
|
||||
assertThat(new WormAuditVerifier(config).verify().passed()).isTrue();
|
||||
assertThat(new SealedExportVerifier(config).verify(STORE.last(WormAuditTables.EXPORT), "USER").result().passed()).isTrue();
|
||||
}
|
||||
}
|
||||
@@ -21,10 +21,15 @@
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.wormaudit.app.WormAuditApp;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditHelp;
|
||||
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.WormAuditProcesses;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
@@ -43,6 +48,7 @@ class WormAuditQBitProducerTest
|
||||
{
|
||||
var config = new WormAuditQBitConfig();
|
||||
QInstance instance = WormAuditTestSupport.init(config);
|
||||
assertThat(config.getIncludeApp()).isTrue();
|
||||
assertThat(config.getRetentionYears()).isEqualTo(6);
|
||||
assertThat(config.getTableMetaDataCustomizer()).isNull();
|
||||
assertThat(config.getSigningKey()).isNull();
|
||||
@@ -160,4 +166,36 @@ class WormAuditQBitProducerTest
|
||||
() -> WormAuditDdl.resolveNames(new WormAuditQBitConfig(), new QInstance()))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Embedded production keeps the host app intact and registers all evidence metadata.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void embeddedMetadataPreservesHostApp() throws Exception
|
||||
{
|
||||
var config = new WormAuditQBitConfig().withBackendName("memory").withIncludeApp(false).withAppName("audit");
|
||||
var instance = WormAuditTestSupport.core(new QBackendMetaData().withName("memory").withBackendType(MemoryBackendModule.class));
|
||||
var host = new QAppMetaData().withName("audit").withLabel("Combined Audit").withWidgets(List.of("hostSummary"));
|
||||
instance.addApp(host);
|
||||
new WormAuditQBitProducer().withQBitConfig(config).produce(instance);
|
||||
assertThat(config.getIncludeApp()).isFalse();
|
||||
assertThat(instance.getApp("worm_audit")).isNull();
|
||||
assertThat(instance.getApp("audit")).isSameAs(host);
|
||||
assertThat(host.getLabel()).isEqualTo("Combined Audit");
|
||||
assertThat(host.getWidgets()).containsExactly("hostSummary");
|
||||
assertThat(host.getHelpContent()).isNullOrEmpty();
|
||||
assertThat(instance.getWidget(WormAuditApp.HEALTH_CARD_WIDGET)).isNotNull();
|
||||
assertThat(instance.getWidget(WormAuditApp.HOME_WIDGET)).isNotNull();
|
||||
for(String name : WormAuditTables.ALL)
|
||||
{
|
||||
assertThat(instance.getTable(name).getHelpContent()).containsKey(WormAuditHelp.SLOT);
|
||||
assertThat(instance.getTable(name).getPermissionRules().getPermissionBaseName()).isEqualTo(name);
|
||||
}
|
||||
assertThat(instance.getTable(WormAuditTables.CHAIN).getDisabledCapabilities()).contains(Capability.TABLE_UPDATE, Capability.TABLE_DELETE, Capability.TABLE_INSERT);
|
||||
assertThat(instance.getTable("audit").getDisabledCapabilities()).contains(Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
|
||||
assertThat(instance.getProcess(WormAuditProcesses.VERIFY)).isNotNull();
|
||||
assertThat(instance.getProcess(WormAuditProcesses.CHAIN).getSchedule().getRepeatSeconds()).isEqualTo(config.getChainIntervalSeconds());
|
||||
assertThat(instance.getProcess(WormAuditProcesses.DAILY).getSchedule().getCronExpression()).isEqualTo(config.getVerifyCron());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.Statement;
|
||||
import java.util.UUID;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The settings row the shipped database script seeds (ddl/postgres/worm-audit.sql,
|
||||
** changeset worm-audit:10, rendered with the host's configured values).
|
||||
** The whole script is PostgreSQL-only (triggers, roles, plpgsql), so it cannot
|
||||
** run on H2; this runs the settings table's CREATE and its seed statement as
|
||||
** the script renders them, on H2 in PostgreSQL mode: the row appears with the
|
||||
** host's values, a second run inserts nothing, and an edit survives.
|
||||
** WormAuditSettings.seed() is deprecated (nothing in src/main calls it) but still
|
||||
** works for memory and H2 hosts; WormAuditSettings.read() falls back to the
|
||||
** configured values while the row is absent.
|
||||
*******************************************************************************/
|
||||
class WormAuditSettingsSeedTest
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
** Fresh table, seed twice, edit survives, values follow the host config.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void seedStatementIsInsertOnlyAndUsesHostValues() throws Exception
|
||||
{
|
||||
var config = new WormAuditQBitConfig().withRetentionYears(8).withExportRetentionDays(45).withCheckpointEveryLinks(12);
|
||||
String ddl = WormAuditDdl.postgres(config);
|
||||
String create = statement(ddl, "CREATE TABLE IF NOT EXISTS public.worm_audit_settings");
|
||||
String seed = statement(ddl, "INSERT INTO public.worm_audit_settings");
|
||||
|
||||
try(Connection connection = DriverManager.getConnection("jdbc:h2:mem:" + UUID.randomUUID() + ";MODE=PostgreSQL;DB_CLOSE_DELAY=-1");
|
||||
Statement sql = connection.createStatement())
|
||||
{
|
||||
sql.execute(create.replace("timestamptz", "timestamp with time zone").replace("now()", "CURRENT_TIMESTAMP").replace("current_user", "'app'"));
|
||||
sql.execute(seed);
|
||||
assertThat(scalar(sql, "SELECT COUNT(*) FROM public.worm_audit_settings")).isEqualTo("1");
|
||||
assertThat(scalar(sql, "SELECT audit_retention_years FROM public.worm_audit_settings")).isEqualTo("8");
|
||||
assertThat(scalar(sql, "SELECT export_retention_days FROM public.worm_audit_settings")).isEqualTo("45");
|
||||
assertThat(scalar(sql, "SELECT checkpoint_every_links FROM public.worm_audit_settings")).isEqualTo("12");
|
||||
assertThat(scalar(sql, "SELECT export_lock_mode FROM public.worm_audit_settings")).isEqualTo(config.getExportLockMode());
|
||||
assertThat(scalar(sql, "SELECT export_cron FROM public.worm_audit_settings")).isEqualTo(config.getExportCron());
|
||||
|
||||
sql.execute("UPDATE public.worm_audit_settings SET export_retention_days = 90 WHERE id = 1");
|
||||
sql.execute(seed);
|
||||
assertThat(scalar(sql, "SELECT COUNT(*) FROM public.worm_audit_settings")).isEqualTo("1");
|
||||
assertThat(scalar(sql, "SELECT export_retention_days FROM public.worm_audit_settings")).isEqualTo("90");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The deprecated public seed() still creates row 1 once and keeps edits.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
@SuppressWarnings("deprecation")
|
||||
void deprecatedSeedStillWorksForMemoryHosts() throws Exception
|
||||
{
|
||||
var config = new WormAuditQBitConfig().withRetentionYears(8);
|
||||
WormAuditTestSupport.init(config);
|
||||
WormAuditSettings.seed(config);
|
||||
assertThat(WormAuditTestSupport.STORE.get("worm_audit_settings", 1).getValueInteger("audit_retention_years")).isEqualTo(8);
|
||||
WormAuditSettings.seed(config);
|
||||
assertThat(WormAuditTestSupport.STORE.get("worm_audit_settings", 1)).isNotNull();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The script seeds nothing but the settings table: its only top-level INSERT is the
|
||||
** settings seed (the indented purge INSERTs are the purge function writing its log).
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void onlySettingsAreSeeded()
|
||||
{
|
||||
String ddl = WormAuditDdl.postgres(new WormAuditQBitConfig());
|
||||
Matcher matcher = Pattern.compile("(?m)^INSERT INTO\\s+(\\S+)").matcher(ddl);
|
||||
while(matcher.find())
|
||||
{
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// worm-audit:13 backfills detail coverage from links that already exist: //
|
||||
// derived from the chain, never seeded content. //
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
assertThat(matcher.group(1)).as("every top-level INSERT in the script targets the settings table, not a log")
|
||||
.isIn("public.worm_audit_settings", "public.worm_audit_chain_detail(detail_id,");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** From the start text up to the statement's closing semicolon.
|
||||
*******************************************************************************/
|
||||
private static String statement(String ddl, String start)
|
||||
{
|
||||
int from = ddl.indexOf(start);
|
||||
assertThat(from).as(start).isGreaterThanOrEqualTo(0);
|
||||
return (ddl.substring(from, ddl.indexOf(";", from)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static String scalar(Statement sql, String query) throws Exception
|
||||
{
|
||||
try(ResultSet result = sql.executeQuery(query))
|
||||
{
|
||||
assertThat(result.next()).isTrue();
|
||||
return (result.getString(1));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,307 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneId;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.wormaudit.app.WormAuditHomeData;
|
||||
import com.kingsrook.qbits.wormaudit.app.WormAuditHomeView;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedExporter;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedExportVerifier;
|
||||
import com.kingsrook.qbits.wormaudit.model.SettingsCustomizer;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.WormAuditProcesses;
|
||||
import com.kingsrook.qbits.wormaudit.processes.ExportProcesses;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.*;
|
||||
import static org.assertj.core.api.Assertions.*;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Editable settings, bounded cache, audited changes and historical lock modes.
|
||||
*******************************************************************************/
|
||||
class WormAuditSettingsTest
|
||||
{
|
||||
static final String TABLE = WormAuditTables.SETTINGS;
|
||||
|
||||
/*******************************************************************************
|
||||
** Deterministic clock for the exact 30-second cache boundary.
|
||||
*******************************************************************************/
|
||||
static class MutableClock extends Clock
|
||||
{
|
||||
Instant now = Instant.parse("2026-10-14T02:00:00Z");
|
||||
/*******************************************************************************
|
||||
** Verify get zone.
|
||||
*******************************************************************************/
|
||||
public ZoneId getZone()
|
||||
{
|
||||
return (ZoneOffset.UTC);
|
||||
}
|
||||
/*******************************************************************************
|
||||
** Verify with zone.
|
||||
*******************************************************************************/
|
||||
public Clock withZone(ZoneId zone)
|
||||
{
|
||||
return (this);
|
||||
}
|
||||
/*******************************************************************************
|
||||
** Verify instant.
|
||||
*******************************************************************************/
|
||||
public Instant instant()
|
||||
{
|
||||
return (now);
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify editor.
|
||||
*******************************************************************************/
|
||||
static void editor()
|
||||
{
|
||||
QContext.getQInstance().withEnforcePermissionsInCoreActions(true);
|
||||
QContext.getQSession().withPermissions(Set.of(TABLE + ".read", TABLE + ".edit", "worm_audit.settings"));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify update.
|
||||
*******************************************************************************/
|
||||
static void update(String field, java.io.Serializable value) throws Exception
|
||||
{
|
||||
var result = new UpdateAction().execute(new UpdateInput(TABLE).withRecords(List.of(new QRecord().withValue("id", 1).withValue(field, value))));
|
||||
assertThat(result.getRecords()).allSatisfy(row -> assertThat(row.getErrors()).isNullOrEmpty());
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify fallback overrides cache and insert only seed.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void fallbackOverridesCacheAndInsertOnlySeed() throws Exception
|
||||
{
|
||||
var config = new WormAuditQBitConfig().withRetentionYears(8).withExportRetentionDays(45).withCheckpointEveryLinks(12);
|
||||
var instance = init(config);
|
||||
var clock = new MutableClock();
|
||||
var reader = new WormAuditSettings(config, clock);
|
||||
assertThat(reader.read().audit_retention_years()).isEqualTo(8);
|
||||
assertThat(reader.read().export_retention_days()).isEqualTo(45);
|
||||
WormAuditTestSupport.seedSettings(config);
|
||||
editor();
|
||||
update("export_retention_days", 90);
|
||||
assertThat(reader.read().export_retention_days()).isEqualTo(90);
|
||||
update("export_retention_days", 120);
|
||||
clock.now = clock.now.plusSeconds(29);
|
||||
assertThat(reader.read().export_retention_days()).isEqualTo(90);
|
||||
clock.now = clock.now.plusSeconds(1);
|
||||
assertThat(reader.read().export_retention_days()).isEqualTo(120);
|
||||
WormAuditTestSupport.seedSettings(config);
|
||||
assertThat(STORE.get(TABLE, 1).getValueInteger("export_retention_days")).isEqualTo(120);
|
||||
assertThat(reader.read().checkpoint_every_links()).isEqualTo(12);
|
||||
instance.getTables().remove(TABLE);
|
||||
reader.invalidate();
|
||||
assertThat(reader.read().export_retention_days()).isEqualTo(45);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify partial null row uses config and invalid persisted settings fail closed.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void partialNullRowUsesConfigAndInvalidPersistedSettingsFailClosed() throws Exception
|
||||
{
|
||||
var config = new WormAuditQBitConfig().withExportRetentionDays(42);
|
||||
init(config);
|
||||
com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore.getInstance().insert(
|
||||
new InsertInput(TABLE).withRecord(new QRecord().withValue("id", 1).withValue("export_lock_mode", "COMPLIANCE")), false);
|
||||
var reader = new WormAuditSettings(config);
|
||||
assertThat(reader.read().export_retention_days()).isEqualTo(42);
|
||||
assertThat(reader.read().export_lock_mode()).isEqualTo("COMPLIANCE");
|
||||
com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore.getInstance().update(
|
||||
new UpdateInput(TABLE).withRecords(List.of(new QRecord().withValue("id", 1).withValue("export_retention_days", 0))), false);
|
||||
reader.invalidate();
|
||||
assertThatThrownBy(reader::read).hasMessageContaining("Export retention days");
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify missing physical table falls back.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void missingPhysicalTableFallsBack() throws Exception
|
||||
{
|
||||
var config = new WormAuditQBitConfig().withExportRetentionDays(81);
|
||||
var instance = RdbmsTestSupport.instance("jdbc:h2:mem:settings_missing;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE", "sa", "", config, false);
|
||||
QContext.init(instance, new QSession());
|
||||
assertThat(config.effectiveSettings().export_retention_days()).isEqualTo(81);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify validation and capabilities.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void validationAndCapabilities() throws Exception
|
||||
{
|
||||
var config = new WormAuditQBitConfig();
|
||||
var instance = init(config);
|
||||
var table = instance.getTable(TABLE);
|
||||
assertThat(table.getLabel()).isEqualTo("WORM Audit Settings");
|
||||
assertThat(table.getAuditRules().getAuditLevel()).isEqualTo(AuditLevel.FIELD);
|
||||
assertThat(table.isCapabilityEnabled(instance.getBackendForTable(TABLE), Capability.TABLE_UPDATE)).isTrue();
|
||||
assertThat(table.isCapabilityEnabled(instance.getBackendForTable(TABLE), Capability.TABLE_INSERT)).isFalse();
|
||||
assertThat(table.isCapabilityEnabled(instance.getBackendForTable(TABLE), Capability.TABLE_DELETE)).isFalse();
|
||||
WormAuditTestSupport.seedSettings(config);
|
||||
editor();
|
||||
assertThatThrownBy(() -> new DeleteAction().execute(new DeleteInput(TABLE).withPrimaryKeys(List.of(1)))).isInstanceOf(Exception.class);
|
||||
assertThatThrownBy(() -> new InsertAction().execute(new InsertInput(TABLE).withRecord(new QRecord().withValue("id", 2)))).isInstanceOf(Exception.class);
|
||||
assertThatThrownBy(() -> new SettingsCustomizer().preInsert(new InsertInput(TABLE), List.of(new QRecord().withValue("id", 2)), false)).hasMessageContaining("one row only (ID 1)");
|
||||
assertThatThrownBy(() -> new SettingsCustomizer().preInsert(new InsertInput(TABLE), List.of(WormAuditSettings.defaults(config)), false))
|
||||
.hasMessageContaining("already exists");
|
||||
assertThatThrownBy(() -> new SettingsCustomizer().preDelete(new DeleteInput(TABLE), List.of(STORE.get(TABLE, 1)), false))
|
||||
.hasMessageContaining("cannot be deleted");
|
||||
for(String field : List.of("audit_retention_years", "export_retention_days", "checkpoint_every_links", "checkpoint_every_minutes", "chain_interval_seconds", "late_detail_lookback_hours"))
|
||||
{
|
||||
assertThatThrownBy(() -> WormAuditSettings.validate(new QRecord().withValue(field, 0))).hasMessageContaining("whole number");
|
||||
}
|
||||
assertThatThrownBy(() -> WormAuditSettings.validate(new QRecord().withValue("settle_horizon_seconds", -1))).hasMessageContaining("Settle horizon seconds");
|
||||
assertThatThrownBy(() -> WormAuditSettings.validate(new QRecord().withValue("audit_retention_years", 101))).hasMessageContaining("100");
|
||||
assertThatThrownBy(() -> WormAuditSettings.validate(new QRecord().withValue("export_retention_days", 36501))).hasMessageContaining("36500");
|
||||
assertThatThrownBy(() -> WormAuditSettings.validate(new QRecord().withValue("export_retention_days", "1.5"))).hasMessageContaining("whole number");
|
||||
assertThatThrownBy(() -> WormAuditSettings.validate(new QRecord().withValue("export_lock_mode", "NONE"))).hasMessageContaining("GOVERNANCE or COMPLIANCE");
|
||||
assertThatThrownBy(() -> WormAuditSettings.validate(new QRecord().withValue("export_lock_mode", null))).hasMessageContaining("GOVERNANCE or COMPLIANCE");
|
||||
for(String field : List.of("export_cron", "verify_cron"))
|
||||
{
|
||||
assertThatThrownBy(() -> new SettingsCustomizer().preUpdate(new UpdateInput(TABLE), List.of(new QRecord().withValue("id", 1).withValue(field, "bad")), false, java.util.Optional.empty()))
|
||||
.hasMessageContaining("valid Quartz cron");
|
||||
}
|
||||
assertThatThrownBy(() -> new UpdateAction().execute(new UpdateInput(TABLE).withRecords(List.of(new QRecord().withValue("id", 1).withValue("export_cron", "bad")))))
|
||||
.hasMessageContaining("valid Quartz cron");
|
||||
assertThat(STORE.get(TABLE, 1).getValueString("export_cron")).isEqualTo(config.getExportCron());
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify edit requires both permissions and produces field audit that chains.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void editRequiresBothPermissionsAndProducesFieldAuditThatChains() throws Exception
|
||||
{
|
||||
var config = ChainVerificationTest.signed().withSettleHorizonSeconds(0);
|
||||
init(config);
|
||||
WormAuditTestSupport.seedSettings(config);
|
||||
QContext.getQInstance().withEnforcePermissionsInCoreActions(true);
|
||||
QContext.getQSession().withPermissions(Set.of(TABLE + ".read", TABLE + ".edit"));
|
||||
assertThatThrownBy(() -> update("export_lock_mode", "COMPLIANCE")).hasMessageContaining("worm_audit.settings");
|
||||
QContext.getQSession().setPermissions(new java.util.HashSet<>(Set.of("worm_audit.settings")));
|
||||
assertThatThrownBy(() -> update("export_lock_mode", "COMPLIANCE")).hasMessageContaining("worm_audit_settings.edit");
|
||||
editor();
|
||||
update("export_lock_mode", "COMPLIANCE");
|
||||
assertThat(STORE.all("audit")).hasSize(1);
|
||||
assertThat(STORE.where("auditDetail", "fieldName", "export_lock_mode")).hasSize(1);
|
||||
assertThat(STORE.get(TABLE, 1).getValueInstant("modify_date")).isNotNull();
|
||||
assertThat(STORE.get(TABLE, 1).getValueString("modified_by")).isEqualTo("system");
|
||||
new WormAuditChainer(config).chainOnce(Instant.now().plusSeconds(1));
|
||||
assertThat(STORE.all(WormAuditTables.CHAIN)).hasSize(1);
|
||||
assertThat(new WormAuditVerifier(config).verify(null, Instant.now().plusSeconds(1)).passed()).isTrue();
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify startup schedules and home use effective settings.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void startupSchedulesAndHomeUseEffectiveSettings() throws Exception
|
||||
{
|
||||
var config = ChainVerificationTest.signed();
|
||||
var instance = init(config);
|
||||
WormAuditTestSupport.seedSettings(config);
|
||||
editor();
|
||||
update("chain_interval_seconds", 45);
|
||||
update("export_cron", "0 15 4 * * ?");
|
||||
update("verify_cron", "0 30 5 * * ?");
|
||||
update("audit_retention_years", 7);
|
||||
assertThat(instance.getProcess(WormAuditProcesses.CHAIN).getSchedule().getRepeatSeconds()).isEqualTo(30);
|
||||
var restarted = WormAuditTestSupport.core(new com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData().withName("memory")
|
||||
.withBackendType(com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule.class));
|
||||
new WormAuditQBitProducer().withQBitConfig(config).produce(restarted);
|
||||
assertThat(QContext.getQInstance()).isSameAs(instance);
|
||||
instance = restarted;
|
||||
QContext.init(instance, new QSession());
|
||||
assertThat(instance.getProcess(WormAuditProcesses.CHAIN).getSchedule().getRepeatSeconds()).isEqualTo(45);
|
||||
assertThat(instance.getProcess(ExportProcesses.DAILY).getSchedule().getCronExpression()).isEqualTo("0 15 4 * * ?");
|
||||
assertThat(instance.getProcess(WormAuditProcesses.DAILY).getSchedule().getCronExpression()).isEqualTo("0 30 5 * * ?");
|
||||
assertThat(instance.getProcess(WormAuditProcesses.DAILY).getSchedule().getCronTimeZoneId()).isEqualTo("UTC");
|
||||
audit(50);
|
||||
var home = WormAuditHomeData.load(config, new SealedExportTest.MemorySealedStore(), Instant.now());
|
||||
assertThat(home.nextPurge()).isEqualTo(OLD.plusSeconds(50).atZone(ZoneOffset.UTC).plusYears(7).toInstant());
|
||||
assertThat(home.warnings()).anyMatch(w -> w.contains("differs from") && w.contains("purge interval"));
|
||||
assertThat(WormAuditHomeView.render(home, "/app")).contains("GOVERNANCE", "s3:BypassGovernanceRetention", "Switch to COMPLIANCE", "30 days", "Export retention is shorter");
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify switch affects new exports only and old checkpoint cadence remains valid.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void switchAffectsNewExportsOnlyAndOldCheckpointCadenceRemainsValid() throws Exception
|
||||
{
|
||||
var config = ChainVerificationTest.signed().withCheckpointEveryLinks(2).withSettleHorizonSeconds(0);
|
||||
init(config);
|
||||
WormAuditTestSupport.seedSettings(config);
|
||||
var storage = new SealedExportTest.MemorySealedStore();
|
||||
audit(1);
|
||||
audit(2);
|
||||
Instant now = Instant.now().plusSeconds(1);
|
||||
new WormAuditChainer(config).chainOnce(now);
|
||||
var exporter = new SealedExporter(config, storage);
|
||||
var first = exporter.run("USER", now).row();
|
||||
assertThat(first).isNotNull();
|
||||
assertThat(first.getValueString("lock_mode")).isEqualTo("GOVERNANCE");
|
||||
assertThat(first.getValueInstant("retain_until")).isEqualTo(now.truncatedTo(java.time.temporal.ChronoUnit.MILLIS).plusSeconds(30 * 86400L));
|
||||
assertThat(storage.heads.values()).allSatisfy(h -> assertThat(h.getLockMode()).isEqualTo("GOVERNANCE"));
|
||||
var verifier = new SealedExportVerifier(config, storage);
|
||||
assertThat(verifier.verify(first, "USER").result().passed()).isTrue();
|
||||
editor();
|
||||
update("export_lock_mode", "COMPLIANCE");
|
||||
update("checkpoint_every_links", 1);
|
||||
new WormAuditChainer(config).chainOnce(Instant.now().plusSeconds(2));
|
||||
var second = exporter.run("USER", Instant.now().plusSeconds(2)).row();
|
||||
assertThat(second.getValueString("lock_mode")).isEqualTo("COMPLIANCE");
|
||||
assertThat(verifier.verify(first, "USER").result().passed()).isTrue();
|
||||
assertThat(verifier.verify(second, "USER").result().passed()).isTrue();
|
||||
assertThat(new WormAuditVerifier(config).verify(null, Instant.now().plusSeconds(2)).passed()).isTrue();
|
||||
String key = first.getValueString("object_prefix") + "links.jsonl";
|
||||
storage.heads.get(key).withLockMode("COMPLIANCE");
|
||||
assertThat(verifier.verify(first, "USER").result().getBreaks()).anyMatch(b -> b.reason().contains("lacks the recorded GOVERNANCE retention"));
|
||||
storage.heads.remove(key);
|
||||
assertThat(verifier.verify(first, "USER").result().getBreaks()).anyMatch(b -> b.reason().contains(key) && b.reason().contains("GOVERNANCE bypass removal"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Stream;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditHelp;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Guardrail (owner requirement 2026-10-14: "all tables should be populated and
|
||||
** the qbit should be fully using them"). For every worm_audit_* table:
|
||||
** - runtime code (not just its model, metadata or home screen) reads or writes
|
||||
** it;
|
||||
** - it is either seeded by the shipped database script, or listed here with
|
||||
** why it starts empty;
|
||||
** - it has bundle help that says what it is for and how its rows get created
|
||||
** (shown on the table screen and when it is empty).
|
||||
** A new table fails this test until it is wired, seeded or explained.
|
||||
*******************************************************************************/
|
||||
class WormAuditTablesGuardrailTest
|
||||
{
|
||||
private static final Path MAIN = Path.of("src", "main", "java", "com", "kingsrook", "qbits", "wormaudit");
|
||||
|
||||
/*******************************************************************************
|
||||
** Tables the shipped database script (ddl/postgres/worm-audit.sql) seeds.
|
||||
*******************************************************************************/
|
||||
static final List<String> SEEDED = List.of(WormAuditTables.SETTINGS);
|
||||
|
||||
/*******************************************************************************
|
||||
** Tables that legitimately start empty, and where their rows come from.
|
||||
*******************************************************************************/
|
||||
static final Map<String, String> STARTS_EMPTY = Map.of(
|
||||
WormAuditTables.CHAIN, "log: the chain job links each settled audit record",
|
||||
WormAuditTables.CHAIN_DETAIL, "log: the chain trigger (or writer) records each detail a new link covers",
|
||||
WormAuditTables.CHECKPOINT, "log: the chain job signs a range every N links or minutes",
|
||||
WormAuditTables.EXPORT, "log: the daily export or Export now seals a range",
|
||||
WormAuditTables.VERIFICATION, "log: every verification run, manual or daily",
|
||||
WormAuditTables.PURGE, "log: the database owner's purge function, after retention");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every table is used by runtime code.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void everyTableIsUsedAtRuntime() throws Exception
|
||||
{
|
||||
List<String> runtimeSource;
|
||||
try(Stream<Path> files = Files.walk(MAIN))
|
||||
{
|
||||
runtimeSource = files.filter(p -> p.toString().endsWith(".java"))
|
||||
.filter(p -> !p.startsWith(MAIN.resolve("model")) && !p.startsWith(MAIN.resolve("app")) && !p.startsWith(MAIN.resolve("ddl")))
|
||||
.filter(p -> !p.endsWith("WormAuditQBitProducer.java"))
|
||||
.map(WormAuditTablesGuardrailTest::read).toList();
|
||||
}
|
||||
for(String table : WormAuditTables.ALL)
|
||||
{
|
||||
String constant = "WormAuditTables." + table.substring("worm_audit_".length()).toUpperCase();
|
||||
assertThat(runtimeSource.stream().anyMatch(source -> source.contains(constant)))
|
||||
.as(table + " (" + constant + ") must be read or written by runtime code, not only declared").isTrue();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every table is seeded by the script or explained.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void everyTableIsSeededOrExplained() throws Exception
|
||||
{
|
||||
String ddl = WormAuditDdl.postgres(new WormAuditQBitConfig());
|
||||
for(String table : WormAuditTables.ALL)
|
||||
{
|
||||
boolean seeded = SEEDED.contains(table);
|
||||
assertThat(seeded || STARTS_EMPTY.containsKey(table)).as(table + " must be seeded by the database script or listed in STARTS_EMPTY").isTrue();
|
||||
assertThat(seeded && STARTS_EMPTY.containsKey(table)).as(table + " cannot be both seeded and starting empty").isFalse();
|
||||
assertThat(ddl.contains("INSERT INTO public." + table + "\n") || ddl.contains("INSERT INTO public." + table + " ")).as(table + " seeded iff the script inserts it").isEqualTo(seeded);
|
||||
}
|
||||
assertThat(STARTS_EMPTY.keySet()).allMatch(WormAuditTables.ALL::contains, "STARTS_EMPTY names only real tables");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every table and the app have bundle help that says how rows get created.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void everyTableHasEmptyStateHelp() throws Exception
|
||||
{
|
||||
QInstance instance = WormAuditTestSupport.init(new WormAuditQBitConfig());
|
||||
assertThat(WormAuditHelp.entries()).allSatisfy((key, entry) -> assertThat(entry.path("summary").asText().length()).as(key).isLessThanOrEqualTo(120));
|
||||
for(String table : WormAuditTables.ALL)
|
||||
{
|
||||
List<QHelpContent> help = instance.getTable(table).getHelpContent() == null ? null : instance.getTable(table).getHelpContent().get(WormAuditHelp.SLOT);
|
||||
assertThat(help).as(table + " table help").isNotEmpty();
|
||||
assertThat(help.get(0).getContent()).as(table + " help says how rows get created").contains("How rows get created");
|
||||
}
|
||||
assertThat(instance.getApp("worm_audit").getHelpContent()).containsKey(WormAuditHelp.SLOT);
|
||||
for(Map.Entry<String, JsonNode> entry : WormAuditHelp.entries().entrySet())
|
||||
{
|
||||
String key = entry.getKey();
|
||||
assertThat(key.startsWith("app:") || WormAuditTables.ALL.contains(key.substring(6, key.indexOf(';')))).as("help names only real tables: " + key).isTrue();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A host's own help is never replaced.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void hostHelpIsKept() throws Exception
|
||||
{
|
||||
QInstance instance = WormAuditTestSupport.init(new WormAuditQBitConfig());
|
||||
instance.getTable(WormAuditTables.CHAIN).getHelpContent().put(WormAuditHelp.SLOT, List.of(new QHelpContent().withContentAsMarkdown("host text")));
|
||||
WormAuditHelp.apply(instance, new WormAuditQBitConfig());
|
||||
assertThat(instance.getTable(WormAuditTables.CHAIN).getHelpContent().get(WormAuditHelp.SLOT)).hasSize(1);
|
||||
assertThat(instance.getTable(WormAuditTables.CHAIN).getHelpContent().get(WormAuditHelp.SLOT).get(0).getContent()).isEqualTo("host text");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static String read(Path path)
|
||||
{
|
||||
try
|
||||
{
|
||||
return (Files.readString(path, StandardCharsets.UTF_8));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw (new IllegalStateException(e));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -37,7 +37,7 @@ import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.Mem
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
|
||||
class WormAuditTestSupport {
|
||||
public class WormAuditTestSupport {
|
||||
static final Instant OLD = Instant.parse("2020-01-02T03:04:05.123456Z");
|
||||
static final AuditStore STORE = new AuditStore(null);
|
||||
|
||||
@@ -130,6 +130,16 @@ class WormAuditTestSupport {
|
||||
false);
|
||||
}
|
||||
|
||||
/** What the shipped database script does for the singleton: insert row 1 from the host config, only when absent. */
|
||||
public static void seedSettings(WormAuditQBitConfig config) throws Exception {
|
||||
if (STORE.get(com.kingsrook.qbits.wormaudit.model.WormAuditTables.SETTINGS, 1) == null) {
|
||||
STORE.insert(
|
||||
com.kingsrook.qbits.wormaudit.model.WormAuditTables.SETTINGS,
|
||||
List.of(WormAuditSettings.defaults(config)));
|
||||
}
|
||||
config.invalidateSettings();
|
||||
}
|
||||
|
||||
static void remove(String table, long id) throws Exception {
|
||||
MemoryRecordStore.getInstance().delete(new DeleteInput(table).withPrimaryKeys(List.of(id)));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.chain;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Floors move only after complete sweeps, only to ids observed before the
|
||||
** horizon, and reset when the tables shrink below them.
|
||||
*******************************************************************************/
|
||||
class SweepFloorsTest
|
||||
{
|
||||
private static final Instant T0 = Instant.parse("2026-10-15T00:00:00Z");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Floors follow the newest observation at or before the horizon.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void floorsAdvanceOnlyPastSettledObservationsOfCompleteSweeps()
|
||||
{
|
||||
SweepFloors floors = SweepFloors.of(new WormAuditQBitConfig());
|
||||
var first = floors.observe(T0, 10, 100);
|
||||
floors.swept(first, T0.minusSeconds(1), true, true, null);
|
||||
assertThat(floors.audit()).as("nothing observed before the horizon yet").isZero();
|
||||
|
||||
var second = floors.observe(T0.plusSeconds(30), 12, 130);
|
||||
floors.swept(second, T0.plusSeconds(10), false, true, null);
|
||||
assertThat(floors.audit()).as("the header sweep hit its limit").isZero();
|
||||
assertThat(floors.detail()).isEqualTo(100);
|
||||
|
||||
var third = floors.observe(T0.plusSeconds(60), 15, 160);
|
||||
floors.swept(third, T0.plusSeconds(45), true, false, null);
|
||||
assertThat(floors.audit()).isEqualTo(12);
|
||||
assertThat(floors.detail()).as("the late-detail sweep hit its limit").isEqualTo(100);
|
||||
|
||||
var fourth = floors.observe(T0.plusSeconds(90), 15, 160);
|
||||
floors.swept(fourth, T0.plusSeconds(90), true, true, null);
|
||||
assertThat(floors.audit()).isEqualTo(15);
|
||||
assertThat(floors.detail()).isEqualTo(160);
|
||||
|
||||
var shrunk = floors.observe(T0.plusSeconds(120), 3, 7);
|
||||
assertThat(floors.audit()).as("tables emptied below the floor: start over").isZero();
|
||||
assertThat(floors.detail()).isZero();
|
||||
floors.swept(shrunk, T0.plusSeconds(100), true, true, null);
|
||||
assertThat(floors.audit()).isZero();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A full sweep is due first, then hourly after each complete one.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void fullSweepIsDueFirstAndHourly()
|
||||
{
|
||||
SweepFloors floors = SweepFloors.of(new WormAuditQBitConfig());
|
||||
assertThat(floors.fullSweepDue(T0)).isTrue();
|
||||
var first = floors.observe(T0, 1, 1);
|
||||
floors.swept(first, T0, true, false, T0);
|
||||
assertThat(floors.fullSweepDue(T0.plusSeconds(30))).as("an incomplete full sweep repeats").isTrue();
|
||||
var second = floors.observe(T0.plusSeconds(30), 1, 1);
|
||||
floors.swept(second, T0, true, true, T0.plusSeconds(30));
|
||||
assertThat(floors.fullSweepDue(T0.plusSeconds(60))).isFalse();
|
||||
assertThat(floors.fullSweepDue(T0.plusSeconds(30).plus(SweepFloors.FULL_SWEEP_EVERY))).isTrue();
|
||||
assertThat(floors.fullSweepDue(T0)).as("clock moved backwards").isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Each installed config has its own floors.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void floorsArePerConfig()
|
||||
{
|
||||
var config = new WormAuditQBitConfig();
|
||||
assertThat(SweepFloors.of(config)).isSameAs(SweepFloors.of(config)).isNotSameAs(SweepFloors.of(new WormAuditQBitConfig()));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.demo;
|
||||
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Instant;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.objectstorage.api.ListRequest;
|
||||
import com.kingsrook.qbits.objectstorage.api.ObjectSummary;
|
||||
import com.kingsrook.qbits.objectstorage.api.Page;
|
||||
import com.kingsrook.qbits.objectstorage.api.StoredObject;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedStore;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Tiny test-scope immutable store with version and COMPLIANCE retention echoes.
|
||||
*******************************************************************************/
|
||||
public class MemorySealedStore implements SealedStore
|
||||
{
|
||||
private final Map<String, byte[]> objects = new LinkedHashMap<>();
|
||||
private final Map<String, StoredObject> heads = new LinkedHashMap<>();
|
||||
|
||||
/*******************************************************************************
|
||||
** Demo capability.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean supportsLock()
|
||||
{
|
||||
return (true);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Refuse overwrites and retain the exact bytes.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String put(String key, Path file, com.kingsrook.qbits.objectstorage.api.LockMode mode, Instant until) throws Exception
|
||||
{
|
||||
if(objects.containsKey(key))
|
||||
{
|
||||
throw (new IllegalStateException("Demo sealed object already exists"));
|
||||
}
|
||||
String version = "v" + (objects.size() + 1);
|
||||
objects.put(key, Files.readAllBytes(file));
|
||||
heads.put(key, new StoredObject().withKey(key).withVersionId(version).withLockMode(mode.name()).withLockRetainUntil(until));
|
||||
return (version);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exact retention metadata.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public StoredObject head(String key)
|
||||
{
|
||||
return (heads.get(key));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exact version read.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public InputStream getVersion(String key, String versionId)
|
||||
{
|
||||
if(!heads.containsKey(key) || !heads.get(key).getVersionId().equals(versionId))
|
||||
{
|
||||
throw (new IllegalArgumentException("Demo sealed version is missing"));
|
||||
}
|
||||
return (new ByteArrayInputStream(objects.get(key)));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The small demo fits one page.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public Page<ObjectSummary> list(ListRequest request)
|
||||
{
|
||||
return (new Page<ObjectSummary>().withItems(objects.keySet().stream().filter(k -> k.startsWith(request.getPrefix()))
|
||||
.map(k -> new ObjectSummary().withKey(k)).toList()));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.demo;
|
||||
|
||||
|
||||
import java.security.KeyPairGenerator;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.wormaudit.PhiCanaryTest;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitProducer;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedExporter;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.VerificationRuns;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.actions.audits.AuditActionCustomizerInterface;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.instances.AbstractQQQApplication;
|
||||
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.audits.AuditsMetaDataProvider;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
|
||||
import com.kingsrook.qqq.middleware.javalin.QApplicationJavalinServer;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Local demo: synthetic notes, test HMAC tokens, real chain, checkpoints,
|
||||
** verifications and sealed exports over a moving clock. Memory storage and its
|
||||
** simulated purge are not production enforcement.
|
||||
**
|
||||
** <pre>
|
||||
** ./mvnw -o test-compile exec:java -Dexec.classpathScope=test -Dexec.mainClass=com.kingsrook.qbits.wormaudit.demo.WormAuditDemoServer -Dexec.args=8099
|
||||
** </pre>
|
||||
*******************************************************************************/
|
||||
public class WormAuditDemoServer extends AbstractQQQApplication
|
||||
{
|
||||
private static Clock clock = Clock.systemUTC();
|
||||
private final MemorySealedStore storage = new MemorySealedStore();
|
||||
private WormAuditQBitConfig config;
|
||||
|
||||
/*******************************************************************************
|
||||
** The PhiCanaryTest stand-in, with a moving timestamp before audit insertion.
|
||||
*******************************************************************************/
|
||||
public static class DemoTokenHook extends PhiCanaryTest.TokenHook
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Tokenize through the same core customization point as the canary test.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void customizeInput(AuditSingleInput input)
|
||||
{
|
||||
input.withTimestamp(clock.instant());
|
||||
super.customizeInput(input);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fully anonymous demo with a FIELD-audited synthetic note table.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QInstance defineQInstance() throws QException
|
||||
{
|
||||
try
|
||||
{
|
||||
var keys = KeyPairGenerator.getInstance("Ed25519").generateKeyPair();
|
||||
config = new WormAuditQBitConfig().withBackendName("memory").withCheckpointEveryLinks(20).withCheckpointEveryMinutes(1)
|
||||
.withSigningKey(() -> new WormAuditQBitConfig.SigningKey("demo", keys.getPrivate())).withTrustedPublicKeys(Map.of("demo", keys.getPublic()))
|
||||
.withSealedStoreSupplier(() -> storage)
|
||||
// Synthetic local demo only: anonymous readers can view its tables.
|
||||
.withTableMetaDataCustomizer((instance, table) -> table.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED)));
|
||||
QInstance instance = new QInstance().withInstanceDefaultAuthentication(new QAuthenticationMetaData()
|
||||
.withName("anonymous").withType(QAuthenticationType.FULLY_ANONYMOUS));
|
||||
instance.addBackend(new QBackendMetaData().withName("memory").withBackendType(MemoryBackendModule.class));
|
||||
new AuditsMetaDataProvider().defineAll(instance, "memory", null);
|
||||
instance.addTable(new QTableMetaData().withName("patient_note").withLabel("Synthetic patient note").withBackendName("memory")
|
||||
.withPrimaryKeyField("id").withRecordLabelFields("id").withRecordLabelFormat("Note #%s")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("diagnosis", QFieldType.STRING).withLabel("Diagnosis").withSupplementalMetaData(new PhiCanaryTest.Phi()))
|
||||
.withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD)));
|
||||
instance.addSupplementalCustomizer(AuditActionCustomizerInterface.CUSTOMIZER_TYPE, new QCodeReference(DemoTokenHook.class));
|
||||
new WormAuditQBitProducer().withQBitConfig(config).produce(instance);
|
||||
instance.setHomeAppName(config.getAppName());
|
||||
return (instance);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw (new QException("Cannot define WORM Audit demo", e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Write through core DML and the test token hook at the current demo time.
|
||||
*******************************************************************************/
|
||||
private void notes(int count) throws QException
|
||||
{
|
||||
for(int i = 0; i < count; i++)
|
||||
{
|
||||
new InsertAction().execute(new InsertInput("patient_note").withRecord(new QRecord().withValue("diagnosis", "SYNTHETIC-DEMO-NOTE-" + i)));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Persist a real verifier result with the historical run time.
|
||||
*******************************************************************************/
|
||||
private void verify() throws QException
|
||||
{
|
||||
var result = new WormAuditVerifier(config).verify(null, clock.instant());
|
||||
VerificationRuns.save(result, "SCHEDULE", null, clock.instant());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Use the production exporter and configured storage boundary.
|
||||
*******************************************************************************/
|
||||
private void seal() throws QException
|
||||
{
|
||||
var result = new SealedExporter(config).run("SCHEDULE", clock.instant());
|
||||
if(result.failed() || result.row() == null)
|
||||
{
|
||||
throw (new QException("Demo seeding failed to seal its checkpointed range"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A fortnight of activity and a genuinely expired, sealed prefix.
|
||||
*******************************************************************************/
|
||||
public void seed(QInstance instance) throws Exception
|
||||
{
|
||||
QContext.init(instance, new QSession());
|
||||
com.kingsrook.qbits.wormaudit.WormAuditTestSupport.seedSettings(config);
|
||||
AuditStore store = new AuditStore(null);
|
||||
Instant today = Instant.now().minus(Duration.ofHours(2));
|
||||
try
|
||||
{
|
||||
Instant expired = today.atZone(ZoneOffset.UTC).minusYears(7).toInstant();
|
||||
clock = Clock.fixed(expired, ZoneOffset.UTC);
|
||||
notes(3);
|
||||
clock = Clock.fixed(expired.plusSeconds(120), ZoneOffset.UTC);
|
||||
new WormAuditChainer(config).chainOnce(clock.instant());
|
||||
clock = Clock.fixed(expired.plusSeconds(240), ZoneOffset.UTC);
|
||||
new WormAuditChainer(config).chainOnce(clock.instant());
|
||||
verify();
|
||||
seal();
|
||||
QRecord anchor = store.last(WormAuditTables.CHAIN);
|
||||
long auditCount = store.count("audit", new QQueryFilter());
|
||||
long detailCount = store.count("auditDetail", new QQueryFilter());
|
||||
// Demo data: memory has no purge function. Remove the expired, sealed
|
||||
// prefix and append the receipt the PostgreSQL function would write.
|
||||
for(String table : List.of("auditDetail", "audit", WormAuditTables.CHAIN))
|
||||
{
|
||||
for(QRecord row : store.all(table))
|
||||
{
|
||||
MemoryRecordStore.getInstance().delete(new DeleteInput(table).withPrimaryKeys(List.of(row.getValue("id"))));
|
||||
}
|
||||
}
|
||||
store.insert(WormAuditTables.PURGE, List.of(new QRecord().withValue("requested_at", today.minus(Duration.ofDays(15)))
|
||||
.withValue("requested_by", "demo-purger").withValue("cutoff", expired.plusSeconds(1)).withValue("retention_interval", "6 years")
|
||||
.withValue("outcome", "PURGED").withValue("reason", "Expired sealed prefix removed (demo memory purge)")
|
||||
.withValue("audits_deleted", auditCount).withValue("details_deleted", detailCount).withValue("links_deleted", anchor.getValueLong("id"))
|
||||
.withValue("anchor_link_id", anchor.getValueLong("id")).withValue("anchor_hash", anchor.getValueString("link_hash"))));
|
||||
for(int day = 13; day >= 0; day--)
|
||||
{
|
||||
Instant time = today.minus(Duration.ofDays(day));
|
||||
clock = Clock.fixed(time, ZoneOffset.UTC);
|
||||
notes(25);
|
||||
clock = Clock.fixed(time.plusSeconds(120), ZoneOffset.UTC);
|
||||
new WormAuditChainer(config).chainOnce(clock.instant());
|
||||
clock = Clock.fixed(time.plusSeconds(240), ZoneOffset.UTC);
|
||||
new WormAuditChainer(config).chainOnce(clock.instant());
|
||||
if(day == 7)
|
||||
{
|
||||
// Demo seeding only: temporarily tamper a memory detail, persist a
|
||||
// genuine FAIL naming that link, then restore before later PASSes.
|
||||
QRecord detail = store.all("auditDetail").getFirst();
|
||||
var original = detail.getValue("newValue");
|
||||
memoryValue(detail, "hmac:v1:demo:tampered");
|
||||
verify();
|
||||
memoryValue(detail, original);
|
||||
}
|
||||
else
|
||||
{
|
||||
verify();
|
||||
}
|
||||
seal();
|
||||
}
|
||||
// Demo REFUSED receipt: too-recent cutoff; no rows removed.
|
||||
store.insert(WormAuditTables.PURGE, List.of(new QRecord().withValue("requested_at", today.plusSeconds(300))
|
||||
.withValue("requested_by", "demo-purger").withValue("cutoff", today).withValue("retention_interval", "6 years")
|
||||
.withValue("outcome", "REFUSED").withValue("reason", "Cutoff is inside the retention interval")
|
||||
.withValue("audits_deleted", 0L).withValue("details_deleted", 0L).withValue("links_deleted", 0L)));
|
||||
// Real unsealed suffix and unchained backlog; no invented export gap.
|
||||
clock = Clock.fixed(today.plusSeconds(600), ZoneOffset.UTC);
|
||||
notes(5);
|
||||
new WormAuditChainer(config).chainOnce(today.plusSeconds(720));
|
||||
clock = Clock.fixed(Instant.now().minusSeconds(120), ZoneOffset.UTC);
|
||||
notes(2);
|
||||
clock = Clock.systemUTC();
|
||||
notes(1);
|
||||
}
|
||||
finally
|
||||
{
|
||||
clock = Clock.systemUTC();
|
||||
QContext.clear();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Deliberate memory-only tamper used solely while seeding the FAIL receipt.
|
||||
*******************************************************************************/
|
||||
private static void memoryValue(QRecord detail, java.io.Serializable value) throws QException
|
||||
{
|
||||
MemoryRecordStore.getInstance().update(new UpdateInput("auditDetail").withRecords(List.of(new QRecord()
|
||||
.withValue("id", detail.getValue("id")).withValue("newValue", value))), false);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Serve the standard widget API and packaged Next UI, reusing the seeded keys.
|
||||
*******************************************************************************/
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
int port = args.length == 0 ? 8099 : Integer.parseInt(args[0]);
|
||||
WormAuditDemoServer application = new WormAuditDemoServer();
|
||||
QInstance instance = application.defineQInstance();
|
||||
new QInstanceValidator().validate(instance);
|
||||
application.seed(instance);
|
||||
new QApplicationJavalinServer(new AbstractQQQApplication()
|
||||
{
|
||||
@Override
|
||||
public QInstance defineQInstance()
|
||||
{
|
||||
return (instance);
|
||||
}
|
||||
}).withPort(port).withServeFrontendNext(true).start();
|
||||
System.out.println("WORM Audit demo on http://localhost:" + port + "/app/worm_audit");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.verify;
|
||||
|
||||
|
||||
import java.security.KeyPairGenerator;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.canonical.AuditCanonical;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.chain.Hashes;
|
||||
import com.kingsrook.qbits.wormaudit.checkpoint.Checkpoints;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Instrument the storage boundary independently of the memory backend: no
|
||||
** unbounded reads are allowed, including portable completeness checks.
|
||||
*******************************************************************************/
|
||||
class PagedVerificationTest
|
||||
{
|
||||
private static final int LINKS = 25000;
|
||||
private static final int PAGE = 137;
|
||||
private static final int RANGE = 997;
|
||||
private static final Instant NOW = Instant.parse("2026-10-13T00:00:00Z");
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Cross many non-aligned page/checkpoint boundaries with genuine signatures.
|
||||
*******************************************************************************/
|
||||
@ParameterizedTest
|
||||
@ValueSource(booleans = { true, false })
|
||||
void fullRunBoundsPagesAndCheckpointRanges(boolean sql) throws Exception
|
||||
{
|
||||
var keys = KeyPairGenerator.getInstance("Ed25519").generateKeyPair();
|
||||
var config = new WormAuditQBitConfig().withCheckpointEveryLinks(RANGE)
|
||||
.withSigningKey(() -> new WormAuditQBitConfig.SigningKey("test", keys.getPrivate()))
|
||||
.withTrustedPublicKeys(Map.of("test", keys.getPublic()));
|
||||
var store = new InstrumentedStore(config, sql);
|
||||
var result = new WormAuditVerifier(config, PAGE).verify(store, null, NOW);
|
||||
assertThat(result.getBreaks()).isEmpty();
|
||||
assertThat(result.getLinksChecked()).isEqualTo(LINKS);
|
||||
assertThat(result.getFromLinkId()).isEqualTo(1);
|
||||
assertThat(result.getToLinkId()).isEqualTo(LINKS);
|
||||
assertThat(store.maxPage).isEqualTo(PAGE);
|
||||
assertThat(store.maxRange).isEqualTo(RANGE);
|
||||
assertThat(store.checkpointsRead).isEqualTo(26);
|
||||
assertThat(store.completenessVisited).isTrue();
|
||||
if(!sql)
|
||||
{
|
||||
store.links.get(0).withValue("detail_ids", "1,1,2");
|
||||
store.links.get(1).withValue("detail_ids", "2");
|
||||
store.links.get(2).withValue("detail_ids", "malformed");
|
||||
store.links.get(3).withValue("audit_id", 1L);
|
||||
store.audits.get(2).withValue("timestamp", null);
|
||||
store.audits.get(4).withValue("timestamp", NOW);
|
||||
store.details.addAll(List.of(new QRecord().withValue("id", 1L).withValue("auditId", 1L),
|
||||
new QRecord().withValue("id", 2L).withValue("auditId", 2L),
|
||||
new QRecord().withValue("id", 3L).withValue("auditId", 99999L),
|
||||
new QRecord().withValue("id", 4L).withValue("auditId", 3L)));
|
||||
var broken = new WormAuditVerifier(config, PAGE).verify(store, null, NOW);
|
||||
assertThat(broken.getBreaks()).extracting(finding -> finding.reason()).contains(
|
||||
"audit 1 is covered twice or more by the chain", "audit 4 is not covered by the chain",
|
||||
"audit 3 has no timestamp", "audit_detail 1 is covered twice or more by the chain",
|
||||
"audit_detail 2 is covered twice or more by the chain", "audit_detail 3 is not covered by the chain",
|
||||
"audit_detail 4 is not covered by the chain");
|
||||
}
|
||||
assertThatThrownBy(() -> new WormAuditVerifier(config, 0)).isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatThrownBy(() -> new WormAuditVerifier(config, 1001)).isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fixture storage may retain its data; the verifier can only fetch bounded
|
||||
** pages. Track the total hashes read between successive checkpoint requests.
|
||||
*******************************************************************************/
|
||||
private static class InstrumentedStore extends AuditStore
|
||||
{
|
||||
private final List<QRecord> audits = new ArrayList<>();
|
||||
private final List<QRecord> details = new ArrayList<>();
|
||||
private final List<QRecord> links = new ArrayList<>();
|
||||
private final List<QRecord> checkpoints = new ArrayList<>();
|
||||
private final boolean sql;
|
||||
private int maxPage;
|
||||
private int maxRange;
|
||||
private int rangeRead;
|
||||
private int checkpointsRead;
|
||||
private boolean inCheckpoints;
|
||||
private boolean completenessVisited;
|
||||
private boolean mainScanFinished;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Generate signed, valid content without involving the backend under test.
|
||||
*******************************************************************************/
|
||||
InstrumentedStore(WormAuditQBitConfig config, boolean sql) throws Exception
|
||||
{
|
||||
super(null);
|
||||
this.sql = sql;
|
||||
String previous = Hashes.GENESIS;
|
||||
for(long id = 1; id <= LINKS; id++)
|
||||
{
|
||||
QRecord audit = new QRecord().withValue("id", id).withValue("timestamp", NOW.minusSeconds(120))
|
||||
.withValue("auditTableId", 1).withValue("auditUserId", 1).withValue("recordId", id);
|
||||
audits.add(audit);
|
||||
String hash = Hashes.link(previous, AuditCanonical.bytes(audit, "fixture", "fixture", List.of()));
|
||||
links.add(new QRecord().withValue("id", id).withValue("audit_id", id).withValue("kind", "AUDIT")
|
||||
.withValue("detail_ids", "").withValue("prev_hash", previous).withValue("link_hash", hash).withValue("chained_at", NOW));
|
||||
previous = hash;
|
||||
}
|
||||
for(int start = 0; start < LINKS; start += RANGE)
|
||||
{
|
||||
checkpoints.add(Checkpoints.sign(links.subList(start, Math.min(start + RANGE, LINKS)), config, NOW)
|
||||
.withValue("id", (long) checkpoints.size() + 1));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** No full-table read is ever permitted through this boundary.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> all(String table)
|
||||
{
|
||||
throw (new AssertionError("Unbounded verifier read: " + table));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A fresh copied page prevents subList from retaining the fixture chain.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> page(String table, long after, int limit)
|
||||
{
|
||||
assertThat(limit).isBetween(1, PAGE);
|
||||
maxPage = Math.max(maxPage, limit);
|
||||
List<QRecord> source;
|
||||
if(table.equals(WormAuditTables.CHECKPOINT))
|
||||
{
|
||||
assertThat(mainScanFinished).isTrue();
|
||||
assertThat(limit).isEqualTo(1);
|
||||
maxRange = Math.max(maxRange, rangeRead);
|
||||
assertThat(rangeRead).isLessThanOrEqualTo(RANGE);
|
||||
rangeRead = 0;
|
||||
inCheckpoints = after < checkpoints.size();
|
||||
if(inCheckpoints)
|
||||
{
|
||||
checkpointsRead++;
|
||||
}
|
||||
source = checkpoints;
|
||||
}
|
||||
else if(table.equals(WormAuditTables.CHAIN))
|
||||
{
|
||||
source = links;
|
||||
if(after == LINKS && !inCheckpoints)
|
||||
{
|
||||
mainScanFinished = true;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
completenessVisited = true;
|
||||
source = table.equals("audit") ? audits : details;
|
||||
}
|
||||
int start = (int) Math.min(after, source.size());
|
||||
var page = new ArrayList<>(source.subList(start, Math.min(start + limit, source.size())));
|
||||
if(inCheckpoints && table.equals(WormAuditTables.CHAIN))
|
||||
{
|
||||
rangeRead += page.size();
|
||||
}
|
||||
return (page);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Batch only the audit headers belonging to the current chain page.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> in(String table, String field, Collection<Long> ids)
|
||||
{
|
||||
assertThat(ids.size()).isLessThanOrEqualTo(PAGE);
|
||||
return (ids.stream().distinct().map(id -> (table.equals("audit") ? audits : details).get(id.intValue() - 1)).toList());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve detail parents without collecting audit history.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QRecord get(String table, long id)
|
||||
{
|
||||
return (id > audits.size() ? null : audits.get((int) id - 1));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stable lookup values match the fixture canonical documents.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String name(String table, Long id)
|
||||
{
|
||||
return ("fixture");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Start at genesis.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QRecord anchor()
|
||||
{
|
||||
return (new QRecord().withValue("anchor_link_id", 0L).withValue("anchor_hash", Hashes.GENESIS));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise both SQL dispatch and portable non-memory completeness.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean isRdbms()
|
||||
{
|
||||
return (sql);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Never permit the memory fallback in this test.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean isMemory()
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** SQL completeness has separate acceptance tests; verify dispatch here.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void completeness(Instant cutoff, VerificationResult result)
|
||||
{
|
||||
completenessVisited = true;
|
||||
}
|
||||
}
|
||||
}
|
||||