Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f14a24fdbf
|
||
|
|
0fb9e8912c | ||
|
|
626e153841 | ||
|
|
85de1daf79 | ||
|
|
be26a370ac | ||
|
|
3c6e7d57a2 | ||
|
|
4b6717ebdd
|
||
|
|
9c2766e324
|
||
|
|
19ac187886
|
||
|
|
f380b9b2a1
|
||
|
|
906df343b0 | ||
|
|
d25df1de0a
|
||
|
|
bbac29e4d0
|
||
|
|
a6f4eb79f6 | ||
|
|
8ab4fb686d
|
||
|
|
1212aedca6 | ||
|
|
87c171387a |
No files matched your search
@@ -1,14 +1,65 @@
|
||||
version: 2.1
|
||||
setup: true
|
||||
|
||||
orbs:
|
||||
qqq-orb: kingsrook/qqq-orb@0.8
|
||||
qqq-orb: kingsrook/qqq-orb@0.6.9
|
||||
|
||||
commands:
|
||||
configure-package-install:
|
||||
steps:
|
||||
- run:
|
||||
name: Keep package installation unattended
|
||||
command: |
|
||||
# These build machines are discarded after the job.
|
||||
sudo install -d /etc/needrestart/conf.d
|
||||
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
|
||||
|
||||
workflows:
|
||||
setup:
|
||||
|
||||
test_only:
|
||||
jobs:
|
||||
- qqq-orb/generate_pipeline:
|
||||
orb_version: '0.8'
|
||||
filters:
|
||||
branches:
|
||||
ignore: automation/qbit-rc-pending
|
||||
tags:
|
||||
ignore: /.*/
|
||||
- qqq-orb/mvn_test_only:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [ qqq-maven-registry-credentials ]
|
||||
filters:
|
||||
branches:
|
||||
ignore: /(develop|release/.*|integration.*)/
|
||||
tags:
|
||||
only: []
|
||||
|
||||
publish_snapshot:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [ qqq-maven-registry-credentials ]
|
||||
branch_type: snapshot
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- develop
|
||||
|
||||
publish_release_candidate:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [ qqq-maven-registry-credentials ]
|
||||
branch_type: release_candidate
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- /release\/.*/
|
||||
|
||||
publish_release:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [ qqq-maven-registry-credentials ]
|
||||
branch_type: release
|
||||
filters:
|
||||
branches:
|
||||
ignore: /.*/
|
||||
tags:
|
||||
only: [ /^v\d+\.\d+\.\d+$/ ]
|
||||
@@ -0,0 +1,20 @@
|
||||
# QRun-IO qbit .editorconfig: the Kingsrook layout (3-space indentation, never tabs).
|
||||
root = true
|
||||
|
||||
[*]
|
||||
charset = utf-8
|
||||
end_of_line = lf
|
||||
insert_final_newline = true
|
||||
trim_trailing_whitespace = true
|
||||
indent_style = space
|
||||
indent_size = 3
|
||||
tab_width = 3
|
||||
|
||||
[*.md]
|
||||
trim_trailing_whitespace = false
|
||||
|
||||
[*.{yml,yaml}]
|
||||
indent_size = 2
|
||||
|
||||
[*.{cmd,bat}]
|
||||
end_of_line = crlf
|
||||
@@ -0,0 +1,3 @@
|
||||
* text=auto eol=lf
|
||||
*.cmd text eol=crlf
|
||||
mvnw text eol=lf
|
||||
@@ -1,25 +0,0 @@
|
||||
# Standard QRun release caller (v2, thin contract): every push and manual dispatch delegates to the protected qrun-release-v1 channel. Do not edit per repository.
|
||||
name: qrun-release
|
||||
on:
|
||||
push: {}
|
||||
workflow_dispatch: {}
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
jobs:
|
||||
release:
|
||||
uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/java-docker.yaml@qrun-release-v1
|
||||
with:
|
||||
runner: linux-small
|
||||
docker-runner: container-build
|
||||
browsers: true
|
||||
browser-runner: browser-build
|
||||
postpublish:
|
||||
if: ${{ needs.release.outputs.capture == 'true' }}
|
||||
needs: release
|
||||
permissions:
|
||||
contents: write
|
||||
actions: write
|
||||
uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/qrun-source-version.yaml@qrun-release-v1
|
||||
with:
|
||||
runner: linux-small
|
||||
@@ -0,0 +1,25 @@
|
||||
# Weekly, grouped dependency updates. Minor and patch updates arrive as one PR per ecosystem;
|
||||
# major updates stay separate so each gets its own review. At most three version-update PRs per
|
||||
# ecosystem are open at once. Security updates are enabled in the repository settings and are
|
||||
# not limited by this schedule.
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: maven
|
||||
directory: /
|
||||
target-branch: develop
|
||||
schedule:
|
||||
interval: weekly
|
||||
open-pull-requests-limit: 3
|
||||
groups:
|
||||
maven-minor-and-patch:
|
||||
update-types: [minor, patch]
|
||||
|
||||
- package-ecosystem: github-actions
|
||||
directory: /
|
||||
target-branch: develop
|
||||
schedule:
|
||||
interval: weekly
|
||||
open-pull-requests-limit: 3
|
||||
groups:
|
||||
github-actions:
|
||||
update-types: [minor, patch]
|
||||
@@ -0,0 +1,31 @@
|
||||
# GitHub Release Configuration
|
||||
# Auto-generates release notes from PR labels
|
||||
|
||||
changelog:
|
||||
exclude:
|
||||
labels:
|
||||
- ignore-for-release
|
||||
- duplicate
|
||||
- invalid
|
||||
authors:
|
||||
- dependabot
|
||||
- dependabot[bot]
|
||||
categories:
|
||||
- title: Breaking Changes
|
||||
labels:
|
||||
- breaking-change
|
||||
- breaking
|
||||
- title: New Features
|
||||
labels:
|
||||
- enhancement
|
||||
- feature
|
||||
- title: Bug Fixes
|
||||
labels:
|
||||
- bug
|
||||
- fix
|
||||
- title: Dependencies
|
||||
labels:
|
||||
- dependencies
|
||||
- title: Other Changes
|
||||
labels:
|
||||
- "*"
|
||||
@@ -1,23 +0,0 @@
|
||||
name: CodeQL
|
||||
'on':
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- develop
|
||||
- release/**
|
||||
- hotfix/**
|
||||
pull_request: {}
|
||||
schedule:
|
||||
- cron: 41 16 * * 0
|
||||
workflow_dispatch: {}
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
security-events: write
|
||||
packages: read
|
||||
concurrency:
|
||||
group: codeql-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
jobs:
|
||||
analyze:
|
||||
uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22
|
||||
@@ -0,0 +1,25 @@
|
||||
# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
|
||||
# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
|
||||
# the release line and the release/* push runs CircleCI publish_release_candidate.
|
||||
name: Release candidate
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: Print the plan without merging or creating branches
|
||||
type: boolean
|
||||
default: false
|
||||
pull_request:
|
||||
types: [closed]
|
||||
branches: [develop]
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
release-rc:
|
||||
if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
|
||||
uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
|
||||
with:
|
||||
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
|
||||
@@ -1,20 +1,21 @@
|
||||
# Maven
|
||||
target/
|
||||
.env
|
||||
.idea
|
||||
*.iml
|
||||
*.class
|
||||
*.log
|
||||
*.jar
|
||||
*.war
|
||||
*.nar
|
||||
*.ear
|
||||
|
||||
# IDE
|
||||
.idea/
|
||||
*.iml
|
||||
.vscode/
|
||||
.project
|
||||
.classpath
|
||||
.settings/
|
||||
|
||||
# OS
|
||||
*.zip
|
||||
*.tar.gz
|
||||
*.rar
|
||||
hs_err_pid*
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
*.sw*
|
||||
.flattened-pom.xml
|
||||
dependency-reduced-pom.xml
|
||||
/.env.local
|
||||
/.cache/
|
||||
.claude/settings.local.json
|
||||
@@ -0,0 +1,3 @@
|
||||
wrapperVersion=3.3.4
|
||||
distributionType=only-script
|
||||
distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.11/apache-maven-3.9.11-bin.zip
|
||||
@@ -1,15 +0,0 @@
|
||||
profile: qbit
|
||||
orb_version: '0.8'
|
||||
default_branch: develop
|
||||
java_version: '21'
|
||||
browsers: true
|
||||
api_checks: false
|
||||
docs_only: false
|
||||
release_on_main: false
|
||||
release_tag_policy: none
|
||||
test_branch_policy: all
|
||||
publish:
|
||||
snapshot: false
|
||||
release_candidate: false
|
||||
release: false
|
||||
asciidoc: false
|
||||
@@ -0,0 +1,51 @@
|
||||
# AGENTS.md
|
||||
|
||||
Guidance for coding agents working in this repository.
|
||||
|
||||
## Overview
|
||||
|
||||
QBit WORM Audit makes QQQ audit records write-once: database-enforced append-only audit tables (Postgres triggers plus an INSERT/SELECT-only app role), a hash chain over every audit row with signed checkpoints, a verification action and daily job, and sealed JSONL exports to an S3 Object Lock (COMPLIANCE) bucket through qbit-object-storage. Design: `docs/DESIGN.md`.
|
||||
|
||||
## Build commands
|
||||
|
||||
```bash
|
||||
./mvnw compile
|
||||
./mvnw test
|
||||
./mvnw verify # Checkstyle; JaCoCo: 70% instructions, 90% classes
|
||||
```
|
||||
|
||||
Use the Maven 3.9.11 wrapper (Java 21); Maven 3.10 cannot read the parent POM. On Windows use `./mvnw.cmd`.
|
||||
|
||||
## Architecture
|
||||
|
||||
Package: `com.kingsrook.qbits.wormaudit`. Entry point: `WormAuditQBitProducer implements QBitMetaDataProducer<WormAuditQBitConfig>`. Config requires the host `backendName`; core audit tables must exist before production.
|
||||
|
||||
- `ddl`: reviewed Liquibase PostgreSQL template and safe renderer; public sidecar tables and owner-only retention purge.
|
||||
- `guard`: immutable QQQ capabilities and value-free SQLSTATE mapping.
|
||||
- `canonical`, `chain`, `checkpoint`: strict JCS v1, transaction/advisory-locked writer, explicit detail coverage, pinned Ed25519 ranges.
|
||||
- `verify`: snapshot checks of content, continuity, coverage and checkpoints, starting at the purge anchor.
|
||||
- `processes`: standard streamed ETL action, scheduled chainer and daily QAlerts producer; never app navigation entries.
|
||||
- `model`: read-only tables, range record references and standard record child lists. No app-home widgets or export execution in phase (a).
|
||||
|
||||
PostgreSQL uses a transaction advisory lock; other backends require a single writer JVM.
|
||||
Use `./mvnw.cmd -o verify` offline. PostgreSQL tests skip without Docker locally, but fail whenever `CI` is set.
|
||||
Core RC.5 has no assistant deny API (reflective registration). Alert types use the
|
||||
`wormaudit.*` namespace and are declared through core alert-type metadata.
|
||||
|
||||
## Invariants and naming
|
||||
|
||||
- Audit rows are never updated or deleted by the application; the only removal path is the audited retention purge through a separate database role, after the retention date.
|
||||
- The chain and the sealed exports hash exactly what core wrote. Never add raw PHI to chain rows, checkpoints, exports, logs or alerts.
|
||||
- Tables use lowercase snake_case with the `worm_audit_` prefix; backend names equal table names. Permissions use the full `worm_audit` prefix. Processes are table or record actions, never navigation entries.
|
||||
|
||||
## Code style
|
||||
|
||||
3-space indent, next-line braces, `return (x);`, flower-box Javadoc on classes and methods, fluent `withX()` setters. Follow `checkstyle/` and `CODE_STYLE.md`.
|
||||
|
||||
## Dependencies
|
||||
|
||||
Parent: `com.kingsrook:qbit-build-parent:2.0.0`. QQQ BOM: `4.1.0-RC.5` (default-active `qqq-snapshot` profile).
|
||||
|
||||
## Release flow
|
||||
|
||||
CircleCI uses `kingsrook/qqq-orb@0.6.8`: develop publishes SNAPSHOTs, release/* publishes RCs, and only `vX.Y.Z` GA tags publish releases. Hotfix branches run tests only; never push `hotfix/*` or `release/*` by hand. RC automation uses workflow_dispatch or a merged develop PR labelled `release:rc`. Feature branches start from develop and return by PR. Never publish manually.
|
||||
@@ -0,0 +1,28 @@
|
||||
# Changelog
|
||||
|
||||
All notable changes are documented here, following [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
|
||||
- RC.2 settings: editable, FIELD-audited singleton with insert-only defaults, validation, separate edit permission and a 30-second effective-settings cache. Schedules load at startup.
|
||||
- GOVERNANCE exports with 30-day default retention; per-export retention verification, administrator-bypass diagnostics, home warnings and documented COMPLIANCE transition.
|
||||
- New `worm-audit:10` settings migration; historical changesets remain unchanged.
|
||||
|
||||
- Phase (c): configurable WORM Audit app home, full-width summary widget and optional host health card; permission-aware navigation and content-free, responsive light/dark summaries.
|
||||
- Chain backlog/settling counts, verification and export record links, sealed-range gaps/lag, retention eligibility and recent purge/refusal history.
|
||||
- PostgreSQL guard/ownership/purge-function diagnostics, signing/trust warnings, tri-state Object Lock capability and stale daily-verification warnings.
|
||||
- Test-scope Next UI demo with synthetic tokenized patient notes, genuine verification failure/recovery, signed checkpoints, sealed memory exports and documented simulated purges.
|
||||
- Home privacy/permission/empty-state tests, memory/H2 summary tests and Docker-gated PostgreSQL setup-warning acceptance.
|
||||
|
||||
- Phase (a): PostgreSQL append-only tables/triggers, safe DDL renderer, retention purge with immutable refusal/anchor records, and runtime grants.
|
||||
- Canonical audit v1 documents, contiguous hash chain with late-detail coverage, pinned Ed25519 checkpoints and binary Merkle roots.
|
||||
- Snapshot verification, streamed ETL Review/Result action, scheduled chaining/daily verification, and core QAlerts raise/resolve/heartbeat integration.
|
||||
- Read-only metadata, WORM Audit table navigation, record range references and child lists; reflective RC.6 assistant deny registration.
|
||||
- Offline memory/H2 acceptance, real pre-insert PHI canary hook, and PostgreSQL Testcontainers acceptance with CI-strict Docker availability.
|
||||
- Adoption, app-home, demo and operations documentation.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Keep runtime signing/trust objects and storage suppliers out of serialized metadata so signed-chain record child lists render successfully.
|
||||
@@ -1,15 +1,51 @@
|
||||
# qbit-worm-audit
|
||||
# CLAUDE.md
|
||||
|
||||
## Knowledge base
|
||||
Guidance for coding agents working in this repository.
|
||||
|
||||
Reviewed knowledge for this repo and the wider QQQ platform lives in the second-brain vault:
|
||||
## Overview
|
||||
|
||||
- Platform hub: `R:/Git.Local/KofTwentyTwo/second-brain/knowledge/qqq/qqq-hub.md`
|
||||
- This repo's dossier: `R:/Git.Local/KofTwentyTwo/second-brain/knowledge/qqq/repos/qbit-worm-audit.md`
|
||||
(reviewed at commit `294cd30f2d91`, branch `main`, 2026-07-04)
|
||||
QBit WORM Audit makes QQQ audit records write-once: database-enforced append-only audit tables (Postgres triggers plus an INSERT/SELECT-only app role), a hash chain over every audit row with signed checkpoints, a verification action and daily job, and sealed JSONL exports to an S3 Object Lock (COMPLIANCE) bucket through qbit-object-storage. Design: `docs/DESIGN.md`.
|
||||
|
||||
Key facts from the review: this repo is an **un-customized instantiation** of the
|
||||
`qbit-template-extension` GitHub template — no WORM-audit code exists yet; Maven coordinates are
|
||||
still `com.kingsrook.qbits:qbit-example-extension`; qqq pin is pre-4.0 (`qqq-backend-core 0.35.0`
|
||||
on main, `0.40.0-SNAPSHOT` on develop); README's "AGPL-3.0" line contradicts the Apache-2.0
|
||||
LICENSE. See the dossier's "v4.0 impact" section before starting implementation work here.
|
||||
## Build commands
|
||||
|
||||
```bash
|
||||
./mvnw compile
|
||||
./mvnw test
|
||||
./mvnw verify # Checkstyle; JaCoCo: 70% instructions, 90% classes
|
||||
```
|
||||
|
||||
Use the Maven 3.9.11 wrapper (Java 21); Maven 3.10 cannot read the parent POM. On Windows use `./mvnw.cmd`.
|
||||
|
||||
## Architecture
|
||||
|
||||
Package: `com.kingsrook.qbits.wormaudit`. Entry point: `WormAuditQBitProducer implements QBitMetaDataProducer<WormAuditQBitConfig>`. Config requires the host `backendName`; core audit tables must exist before production.
|
||||
|
||||
- `ddl`: reviewed Liquibase PostgreSQL template and safe renderer; public sidecar tables and owner-only retention purge.
|
||||
- `guard`: immutable QQQ capabilities and value-free SQLSTATE mapping.
|
||||
- `canonical`, `chain`, `checkpoint`: strict JCS v1, transaction/advisory-locked writer, explicit detail coverage, pinned Ed25519 ranges.
|
||||
- `verify`: snapshot checks of content, continuity, coverage and checkpoints, starting at the purge anchor.
|
||||
- `processes`: standard streamed ETL action, scheduled chainer and daily QAlerts producer; never app navigation entries.
|
||||
- `model`: read-only tables, range record references and standard record child lists. No app-home widgets or export execution in phase (a).
|
||||
|
||||
PostgreSQL uses a transaction advisory lock; other backends require a single writer JVM.
|
||||
Use `./mvnw.cmd -o verify` offline. PostgreSQL tests skip without Docker locally, but fail whenever `CI` is set.
|
||||
Core RC.5 has no assistant deny API (reflective registration). Alert types use the
|
||||
`wormaudit.*` namespace and are declared through core alert-type metadata.
|
||||
|
||||
## Invariants and naming
|
||||
|
||||
- Audit rows are never updated or deleted by the application; the only removal path is the audited retention purge through a separate database role, after the retention date.
|
||||
- The chain and the sealed exports hash exactly what core wrote. Never add raw PHI to chain rows, checkpoints, exports, logs or alerts.
|
||||
- Tables use lowercase snake_case with the `worm_audit_` prefix; backend names equal table names. Permissions use the full `worm_audit` prefix. Processes are table or record actions, never navigation entries.
|
||||
|
||||
## Code style
|
||||
|
||||
3-space indent, next-line braces, `return (x);`, flower-box Javadoc on classes and methods, fluent `withX()` setters. Follow `checkstyle/` and `CODE_STYLE.md`.
|
||||
|
||||
## Dependencies
|
||||
|
||||
Parent: `com.kingsrook:qbit-build-parent:2.0.0`. QQQ BOM: `4.1.0-RC.5` (default-active `qqq-snapshot` profile).
|
||||
|
||||
## Release flow
|
||||
|
||||
CircleCI uses `kingsrook/qqq-orb@0.6.8`: develop publishes SNAPSHOTs, release/* publishes RCs, and only `vX.Y.Z` GA tags publish releases. Hotfix branches run tests only; never push `hotfix/*` or `release/*` by hand. RC automation uses workflow_dispatch or a merged develop PR labelled `release:rc`. Feature branches start from develop and return by PR. Never publish manually.
|
||||
@@ -0,0 +1,3 @@
|
||||
# Code of Conduct
|
||||
|
||||
See Project level [Code of Conduct](https://github.com/QRun-IO/qqq/blob/develop/CODE_OF_CONDUCT.md)
|
||||
@@ -0,0 +1,3 @@
|
||||
# Code Style Guide
|
||||
|
||||
* See Project [Code Style Guide](https://github.com/QRun-IO/qqq/blob/develop/CODE_STYLE.md)
|
||||
@@ -0,0 +1,6 @@
|
||||
# Contributing to QBit Secured Data
|
||||
|
||||
* See Project [Contributing](https://github.com/QRun-IO/qqq/blob/develop/CONTRIBUTING.md)
|
||||
* Branches follow GitFlow: `feature/*` from `develop`, pull requests to `develop`, `release/*` for release candidates, `vX.Y.Z` tags for releases.
|
||||
* Commits are signed and follow Conventional Commits.
|
||||
* Build and test locally with `./mvnw verify` (Java 21).
|
||||
@@ -176,7 +176,7 @@
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
Copyright 2022-2025 QRunIO, LLC
|
||||
Copyright 2024 Kingsrook, LLC
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
qBit Template - Extension
|
||||
Copyright 2022-2025 QRunIO, LLC
|
||||
QBit WORM Audit
|
||||
Copyright 2026 Kingsrook, LLC
|
||||
|
||||
This product includes software developed at QRunIO, LLC (https://qrun.io/).
|
||||
This product includes software developed at Kingsrook, LLC (https://kingsrook.com/).
|
||||
|
||||
Licensed under the Apache License, Version 2.0.
|
||||
@@ -1,48 +1,388 @@
|
||||
# QBit Template: Extension
|
||||
# QBit WORM Audit
|
||||
|
||||
Template repository for creating QQQ Extension QBits - infrastructure extensions, customizers, and action handlers.
|
||||
Phases (a)–(c) provide PostgreSQL append-only enforcement, an asynchronous audit hash chain,
|
||||
Ed25519 checkpoints, live and sealed-export verification actions, and daily jobs with core QAlerts signals.
|
||||
It hashes exactly the audit values core stored; it does not read application records or
|
||||
secured-data keys. Sealed JSONL exports use qbit-object-storage 0.1.0-RC.2 and configurable
|
||||
Object Lock (GOVERNANCE for 30 days by default). The WORM Audit app home summarizes chain health, exports, retention and setup.
|
||||
|
||||
## What is an Extension QBit?
|
||||
Requires Java 21, Maven wrapper 3.9.11 and QQQ 4.1.0-RC.5. PostgreSQL provides enforcement;
|
||||
memory/H2 support tamper-evidence tests without database enforcement.
|
||||
|
||||
Extension QBits extend or enhance how QQQ operates at the infrastructure/framework level. They provide:
|
||||
- New backend implementations
|
||||
- Authentication mechanisms
|
||||
- Audit capabilities
|
||||
- Table customizers
|
||||
- Action handlers
|
||||
## Tables and permissions
|
||||
|
||||
## Quick Start
|
||||
All sidecar tables use the configured host backend, snake_case fields, BIGINT identifiers,
|
||||
record sections, and full table-name permissions. Immutable sidecars do not audit themselves.
|
||||
The editable Settings singleton uses an INTEGER ID and FIELD-level core audit. The app lists
|
||||
Verifications, Sealed Exports, Checkpoints, Audit Chain, Purges, and Settings in that order.
|
||||
|
||||
1. Click "Use this template" to create your QBit
|
||||
2. Run `scripts/customize_template.py` to rename the example
|
||||
3. Implement your extension logic
|
||||
4. Add tests
|
||||
5. Publish to Maven repository
|
||||
| Table | Purpose | Permission |
|
||||
|---|---|---|
|
||||
| `worm_audit_verification` | Immutable USER/SCHEDULE results | `worm_audit_verification.read` |
|
||||
| `worm_audit_export` | Successfully sealed export catalog | `worm_audit_export.read` |
|
||||
| `worm_audit_checkpoint` | Signed ranges and Merkle roots | `worm_audit_checkpoint.read` |
|
||||
| `worm_audit_chain` | Contiguous links and explicit detail coverage | `worm_audit_chain.read` |
|
||||
| `worm_audit_purge` | Refusals, removal counts and retention anchors | `worm_audit_purge.read` |
|
||||
| `worm_audit_settings` | Editable singleton; every edit is audited and chained | `worm_audit_settings.read`; editing needs `worm_audit_settings.edit` **and** `worm_audit.settings` |
|
||||
|
||||
## Structure
|
||||
`worm_audit.verify` authorizes the standard streamed ETL **Verify audit chain** action.
|
||||
It also authorizes **Verify sealed export** on selected export records. `worm_audit.export`
|
||||
authorizes **Export now** on the export table. Processes are table/record actions or schedules;
|
||||
none is a navigation entry.
|
||||
Insert/update/delete capabilities are absent on immutable sidecar tables; Settings allows updates only; update/delete capabilities
|
||||
are removed from all four core audit tables. Internal appends use an explicit permission bypass.
|
||||
Checkpoint, export and verification record screens have a bounded chain-link child list;
|
||||
range endpoints and export/checkpoint references link to their records.
|
||||
|
||||
```
|
||||
src/main/java/com/kingsrook/qbits/example/
|
||||
├── ExampleExtensionQBitConfig.java # Configuration
|
||||
├── ExampleExtensionQBitProducer.java # Entry point
|
||||
├── customizers/
|
||||
│ └── ExampleTableCustomizer.java # Table customization
|
||||
└── actions/
|
||||
└── ExampleActionCustomizer.java # Action handling
|
||||
The table metadata customizer can configure host ESB publications. Quick Search registration
|
||||
is host-owned through `QuickSearchQBitConfig.searchableTables`; add the desired tables.
|
||||
Verification and export inserts use QQQ's record-change listener SPI,
|
||||
so an installed ESB publisher receives those events when the host configures its destinations
|
||||
on these two tables. The qbit defines no destination and configures no chain-link publication.
|
||||
|
||||
## Configuration
|
||||
|
||||
| Field | Default / contract |
|
||||
|---|---|
|
||||
| `backendName` | Required; core and sidecar audit tables must use this backend |
|
||||
| `appName` / `appLabel` | `worm_audit` / `WORM Audit` |
|
||||
| `parentAppName` | Optional host app, for example `admin` |
|
||||
| `tablePathPrefix` | `/app`; standard QQQ table/record route prefix |
|
||||
| `sealedStoreSupplier` | Optional host/test storage boundary; defaults to qbit-object-storage |
|
||||
| `retentionYears` | 6; compiled into the migration, not changed by a runtime flag |
|
||||
| `signingKey` | Optional `Supplier<SigningKey>`; `SigningKey(keyId, PrivateKey)` |
|
||||
| `trustedPublicKeys` | `Map<String, PublicKey>`; must contain the signing key's public key |
|
||||
| `chainIntervalSeconds` | 30 |
|
||||
| `settleHorizonSeconds` | 60 (zero allowed) |
|
||||
| `exportBucketAlias` | `worm_audit`; qbit-object-storage BACKUP bucket alias |
|
||||
| `exportPrefix` | `worm-audit`; slash-separated letters, digits, underscores or hyphens |
|
||||
| `exportCron` | `0 0 2 * * ?`; 02:00 UTC on the configured scheduler |
|
||||
| `batchSize` | 1,000 headers per tick / links per insert batch |
|
||||
| `lateDetailLookbackHours` | 24; positive hours, older omissions checked by full verification |
|
||||
| `checkpointEveryLinks` | 10,000; maximum links in each checkpoint |
|
||||
| `checkpointEveryMinutes` | 60; requires at least one uncheckpointed link |
|
||||
| `verifyCron` | `0 0 3 * * ?` (03:00 UTC, Quartz syntax) |
|
||||
| `schedulerName` | `worm_audit`; uses the host scheduler if present, otherwise registers Quartz |
|
||||
| `coreAuditTableNames` | Overrides keyed by `audit`, `auditDetail`, `auditTable`, `auditUser` |
|
||||
| `appRole` | `app` |
|
||||
| `tableMetaDataCustomizer` | Optional; append-only capability guard runs afterward |
|
||||
|
||||
A missing signing key produces no checkpoints and adds the warning **no signing key configured**
|
||||
to verification. It does not itself break the chain. Signing requires a matching public-key pin.
|
||||
Rotate by adding a new pin and changing the supplier; retain old pins for old checkpoints.
|
||||
Never store the private key in a database row or checked-in configuration.
|
||||
|
||||
## Settings
|
||||
|
||||
Open **WORM Audit → Settings**, record **1** (`worm_audit_settings`, labelled WORM Audit Settings).
|
||||
This is a standard editable QQQ table. New records and deletion are disabled; a pre-insert
|
||||
check and database constraint require ID 1. Every edit uses core FIELD-level auditing, so
|
||||
the change enters the audit chain. `modify_date` and `modified_by` are stamped by the server.
|
||||
Keep notes free of sensitive data; notes are audited too.
|
||||
|
||||
| Field | Default | Validation / effect |
|
||||
|---|---|---|
|
||||
| `audit_retention_years` | 6 | 1–100; home next eligible purge date |
|
||||
| `export_lock_mode` | GOVERNANCE | GOVERNANCE or COMPLIANCE; new exports |
|
||||
| `export_retention_days` | 30 | 1–36500; new exports |
|
||||
| `checkpoint_every_links` | 10000 | Positive integer; new checkpoint ranges |
|
||||
| `checkpoint_every_minutes` | 60 | Positive integer; checkpoint cadence |
|
||||
| `chain_interval_seconds` | 30 | Positive integer; **takes effect after restart** |
|
||||
| `settle_horizon_seconds` | 60 | Nonnegative integer; chaining and verification |
|
||||
| `late_detail_lookback_hours` | 24 | Positive integer; late-detail polling |
|
||||
| `export_cron` | `0 0 2 * * ?` | Quartz cron, UTC; **takes effect after restart** |
|
||||
| `verify_cron` | `0 0 3 * * ?` | Quartz cron, UTC; **takes effect after restart** |
|
||||
| `notes` | Empty | Administrative notes |
|
||||
| `modify_date`, `modified_by` | Server supplied | Last edit time and user |
|
||||
|
||||
Config values supply defaults for missing rows or null fields; a missing settings table also
|
||||
falls back to config. Other database failures fail closed. `exportLockMode` and
|
||||
`exportRetentionDays` are the corresponding config defaults. Runtime reads cache the row
|
||||
for at most 30 seconds per application instance; local QQQ edits invalidate that cache.
|
||||
Each run reads effective settings. Startup schedules read the existing row, or config when
|
||||
absent. Historical checkpoints remain valid after changing their creation cadence.
|
||||
|
||||
Changeset `worm-audit:10` creates and seeds the table with `INSERT ... SELECT ... WHERE NOT
|
||||
EXISTS`; rerunning it never overwrites an administrator change. The renderer seeds from the
|
||||
host config. Memory hosts and the demo can call `WormAuditSettings.seed(config)` for the same
|
||||
insert-only behavior during single-threaded startup.
|
||||
|
||||
`audit_retention_years` does **not** rewrite the database purge function. The home warns if
|
||||
it differs from the compiled interval; changing that interval requires a reviewed migration.
|
||||
The home also warns when export retention is shorter than audit retention and while
|
||||
GOVERNANCE permits administrator bypass deletion. The runtime role receives SELECT, INSERT
|
||||
and UPDATE on Settings, with no DELETE or TRUNCATE grant and no append-only triggers.
|
||||
|
||||
## Switching sealed exports to COMPLIANCE
|
||||
|
||||
Edit `export_lock_mode` to **COMPLIANCE** in Settings. New exports use COMPLIANCE once the
|
||||
settings cache refreshes (within 30 seconds); existing GOVERNANCE objects keep their mode
|
||||
unless an administrator explicitly extends their retention and upgrades their mode. The
|
||||
verifier continues to compare each object with its export row's recorded mode and date, so
|
||||
an out-of-band mode change is reported as a mismatch. The bucket default retention should
|
||||
match the chosen mode and retention days, although the exporter always sends explicit values.
|
||||
The settings change is audited and chained. **COMPLIANCE cannot be undone**: protected
|
||||
versions cannot be deleted or have retention shortened until expiry, including by administrators.
|
||||
|
||||
## App home
|
||||
|
||||
Open `/app/worm_audit` (or `/app/admin/worm_audit` with `parentAppName="admin"`).
|
||||
The full-width `wormAuditHome` HTML widget uses the standard `/qqq/v1/widget` API.
|
||||
Hosts can add `wormAuditHealthCard` to their own dashboard. Both widgets and the app
|
||||
require read permission on at least one WORM table, using the host's table permission rules.
|
||||
The aggregate home shows only identifiers, counts, times, shortened hashes and reasons;
|
||||
it never renders audit messages or field values. Record links retain the target table's permissions.
|
||||
|
||||
- Chain status: live link count, latest link time, settled unchained backlog and settling
|
||||
count at the configured horizon, latest verification and first open chain break.
|
||||
A subsequent full scheduled CHAIN PASS clears a reported break; a selected USER PASS does not.
|
||||
- Export status: current lock mode and retention days, latest sealed range and its recorded retention, link/time lag, coverage holes
|
||||
between the purge anchor and sealed head, and bucket Object Lock capability. The unsealed
|
||||
trailing suffix is lag, not a gap. Provider failures show UNKNOWN with a value-free reason.
|
||||
- Retention: configured years, purge anchor, oldest live audit and its next eligible purge date.
|
||||
The date indicates retention eligibility; the database purge function still checks the sealed prefix.
|
||||
- Recent purges: the last ten requests, including refusals.
|
||||
- Setup warnings: missing signing/trusted keys, unsupported or unknown Object Lock,
|
||||
daily CHAIN verification never run or older than 26 hours. PostgreSQL also checks every
|
||||
protected table's row/truncate guards, runtime ownership and the purge function.
|
||||
|
||||
Sections and definition-list rows wrap at narrow widths and inherit the Next UI's light/dark colors.
|
||||
Empty states explicitly distinguish missing history from a passing check.
|
||||
|
||||
### Local demo
|
||||
|
||||
The test-scope server serves the API and bundled Next UI with a memory backend:
|
||||
|
||||
```powershell
|
||||
./mvnw.cmd -o test-compile exec:java "-Dexec.classpathScope=test" "-Dexec.mainClass=com.kingsrook.qbits.wormaudit.demo.WormAuditDemoServer" "-Dexec.args=8099"
|
||||
```
|
||||
|
||||
## Key Characteristics
|
||||
On macOS/Linux use `./mvnw` with the same arguments. Open <http://localhost:8099/app/worm_audit>.
|
||||
No Docker or credentials are required. This anonymous demo uses synthetic `patient_note`
|
||||
records audited at FIELD level through the `PhiCanaryTest` HMAC stand-in. It seeds two weeks
|
||||
of actual chainer ticks, signed checkpoints, daily verifier results and immutable memory exports.
|
||||
One temporary detail tamper produces a genuine FAIL before restoration and later PASSes.
|
||||
The PURGED/REFUSED receipts and memory-only prefix removal are explicitly marked demo data;
|
||||
PostgreSQL enforcement and real provider durability are not simulated production guarantees.
|
||||
|
||||
- **No QAppSection** - Pure infrastructure
|
||||
- **No user-facing tables** (or minimal operational)
|
||||
- **Wraps existing tables/processes** with new behavior
|
||||
- **Validates dependencies** on other QQQ components
|
||||
The demo includes a small unsealed suffix and unchained backlog, with no invented export gap.
|
||||
The final settling count naturally ages into the backlog while the demo is open.
|
||||
|
||||
## Documentation
|
||||
## Adopting in another app
|
||||
|
||||
- [Getting Started](docs/00-getting-started.md)
|
||||
- [Extension Patterns](docs/01-extension-patterns.md)
|
||||
Add `com.kingsrook.qbits:qbit-worm-audit:0.1.0-SNAPSHOT`. Define the host's core audit tables
|
||||
with `AuditsMetaDataProvider` first, then install the qbit:
|
||||
|
||||
```java
|
||||
WormAuditQBitConfig config = new WormAuditQBitConfig()
|
||||
.withBackendName("main")
|
||||
.withAppRole("app")
|
||||
.withSigningKey(() -> new WormAuditQBitConfig.SigningKey("2026-10", privateKey))
|
||||
.withTrustedPublicKeys(Map.of("2026-10", publicKey));
|
||||
new WormAuditQBitProducer().withQBitConfig(config).produce(qInstance);
|
||||
String migration = WormAuditDdl.postgres(config);
|
||||
```
|
||||
|
||||
1. Render and review the migration, then copy it into the host Liquibase changelog.
|
||||
The [default rendering](docs/ddl/worm-audit-postgres.sql) uses `audit`, `audit_detail`,
|
||||
`audit_table`, `audit_user`, runtime role `app`, and six years. The renderer resolves table
|
||||
backend names during production; explicit overrides win. Standalone rendering uses those
|
||||
defaults. Identifiers are validated and quoted; schema-qualified core table names are supported.
|
||||
2. Apply the DDL as the schema owner. It creates the five sidecar tables in `public`, functions,
|
||||
triggers, the NOLOGIN `worm_audit_purger` role, and runtime grants. Core column names must use
|
||||
the usual snake_case mappings (`audit_id`, `audit_table_id`, etc.). The migration is idempotent. Provision the purger role beforehand or give the migration role permission to create it.
|
||||
3. Run as a separate runtime role with SELECT/INSERT and sequence USAGE. Do not make it a member
|
||||
of the purger role or a schema/table owner. Use the host's PostgreSQL RDBMS strategy (RC.5's
|
||||
base strategy uses MySQL quoting; see the test host strategy in `RdbmsTestSupport`).
|
||||
4. Start QQQ's standard scheduler with its system session and grant users only the required read
|
||||
and verification permissions. Point the existing alerts implementation at core `QAlerts`.
|
||||
5. Configure ESB publications on verification/export in the table customizer. Register all five
|
||||
tables in the host Quick Search config with `SearchableTableConfig` / `SearchableFieldConfig`.
|
||||
Index identifiers and status fields; set basepull timestamps to `chained_at`, `created_at`,
|
||||
`sealed_at`, `finished_at`, and `requested_at` respectively (these tables have no `modifyDate`).
|
||||
|
||||
The RC.6 assistant deny API is registered reflectively when present; absence in RC.5 is a no-op.
|
||||
The qbit declares `wormaudit.chain.break` (CRITICAL), `wormaudit.export.failed` (ERROR),
|
||||
and the daily verification/export heartbeat types through core alert-type metadata.
|
||||
Daily heartbeat monitors expect a 24-hour interval with 2 hours of grace (overdue at 26 hours).
|
||||
`QAlerts` dispatches to the installed sink or its logging fallback. Every scheduled verification sends
|
||||
`wormaudit.verify.daily`; FAIL raises with the first break and verification record reference;
|
||||
PASS resolves dedup key `chain`.
|
||||
|
||||
## Chain and verification format
|
||||
|
||||
- JCS v1 uses UTF-16 key ordering, strict surrogate validation and UTC microsecond timestamps.
|
||||
IDs are integer longs except canonical `recordId`, which is a string. No floating-point values.
|
||||
- Link hashes concatenate the predecessor's **64 lowercase ASCII hex bytes**, one NUL byte,
|
||||
and canonical UTF-8 bytes. Genesis is 64 ASCII zeroes.
|
||||
- Every link names the detail IDs it covered, at most 250. A PostgreSQL insert guard and GIN
|
||||
overlap index reject duplicate coverage under the same advisory lock. Overflow and later details use
|
||||
`LATE_DETAIL` links with the same canonical header-and-detail-subset document. The canonical
|
||||
JSON is never persisted. A v1 content mismatch can identify the link, audit and covered detail
|
||||
IDs, but cannot identify one changed field among multiple details without original content.
|
||||
- PostgreSQL chaining uses transaction-scoped advisory lock `86753090421` shared with purge.
|
||||
Other backends use a JVM-wide lock and require one writer JVM. SQL backends select settled
|
||||
unchained headers with an anti-join; late commits are picked up on a later tick.
|
||||
- Merkle leaves are decoded 32-byte link hashes; parents hash the concatenated binary children;
|
||||
an odd node is promoted unchanged. Ed25519 signs JCS of the seven specified range/trust fields.
|
||||
- SQL verification uses a repeatable-read snapshot. Full runs check settled header/detail
|
||||
completeness; selected or filtered runs check their links, predecessor boundaries and
|
||||
intersecting checkpoints.
|
||||
|
||||
## Retention operations
|
||||
|
||||
Only the ops login in `worm_audit_purger` may execute:
|
||||
|
||||
```sql
|
||||
SELECT public.worm_audit_purge(now() - interval '6 years');
|
||||
```
|
||||
|
||||
A recent/null cutoff or an incomplete expired prefix records REFUSED and returns normally.
|
||||
Purge locks the audit tables against concurrent inserts, removes only a prefix closed over
|
||||
all links and details of its audits, and records its last link id/hash. Detail and chain deletion
|
||||
is allowed only under the function-owner identity, transaction-local purge flag, and compiled
|
||||
retention cutoff. An app session setting that flag gets no deletion authority.
|
||||
|
||||
Lookups cannot change id/name or be deleted/truncated. The runtime role may update only the
|
||||
lookup label/dates through column grants; table-level UPDATE stays revoked. With normal grants,
|
||||
PostgreSQL rejects prohibited runtime writes as SQLSTATE 42501 before executing a trigger.
|
||||
The second layer (triggers) returns `P0001` / `worm_audit:` even if someone grants wider DML
|
||||
rights. The qbit maps those errors to a value-free user-facing message.
|
||||
|
||||
Verification starts from the latest successful purge anchor. Checkpoint signatures still verify
|
||||
after purge, but Merkle recomputation of a purged/partly purged checkpoint requires its sealed
|
||||
export and is reported as a warning. Seal and verify exports before exercising retention
|
||||
removal in production. Superusers/owners can disable triggers;
|
||||
protect signing keys and keep public-key pins outside the database.
|
||||
|
||||
## Sealed exports
|
||||
|
||||
Install qbit-object-storage and configure `exportBucketAlias` (default `worm_audit`) as a
|
||||
`BucketRole.BACKUP` bucket. **Object Lock must be enabled when the bucket is created**.
|
||||
The exporter requires the bucket's `OBJECT_LOCK` capability to be `SUPPORTED`; run the
|
||||
object-storage read-only capability probe during setup. Unknown, partial and unsupported
|
||||
capabilities all refuse sealing. Every object receives the effective `export_lock_mode` and retention until sealing time plus
|
||||
`export_retention_days` (GOVERNANCE and 30 days by default), checked by HEAD after PUT. The
|
||||
export row records both values. Verification checks that historical mode and retain-until,
|
||||
regardless of subsequent settings changes. Missing GOVERNANCE objects are reported as breaks
|
||||
with a possible administrator bypass-removal explanation.
|
||||
|
||||
The runtime IAM policy needs `s3:PutObject`, `s3:GetObject`, `s3:ListBucket`,
|
||||
`s3:GetObjectRetention`, `s3:PutObjectRetention`, and
|
||||
`s3:GetBucketObjectLockConfiguration`. Grant no delete or governance-bypass access.
|
||||
The storage bucket's upload size limit must accommodate the largest daily JSONL file.
|
||||
Retention protects object versions; protect the storage volumes and maintain off-site copies too.
|
||||
|
||||
At 02:00 UTC, or through **Export now**, the exporter takes the chainer's writer lock and
|
||||
seals links after the previous export through the latest checkpoint. The first export starts
|
||||
after the purge anchor, or at genesis. No new checkpoint produces an explicit "Nothing to
|
||||
seal" Result. If retention has removed an unexported gap, or a range cannot be proven by
|
||||
complete signed checkpoints, sealing refuses rather than recording incomplete evidence.
|
||||
Other database backends require a single writer JVM, as in phase (a).
|
||||
|
||||
Keys use `<exportPrefix>/<yyyy>/<MM>/<dd>/<from>-<to>/` with a UTC date:
|
||||
|
||||
- `links.jsonl`: JCS lines containing `link` identifiers/hashes and the embedded canonical
|
||||
`content` document exactly as hashed.
|
||||
- `checkpoints.jsonl`: JCS checkpoint envelopes including signatures and public keys.
|
||||
- `manifest.json`: JCS `worm-audit-export/1` with `fromLinkId`, `toLinkId`, `linkCount`,
|
||||
`checkpointCount`, `files` (filename to SHA-256), `headHash`, `prevManifestSha256`,
|
||||
`qbitVersion`, and `createdAt`.
|
||||
|
||||
Files are streamed through temporary disk storage with 1,000-link pages and batched content
|
||||
fetches, then removed. Only ids, hashes and object metadata enter the export catalog.
|
||||
The `object_versions` text column pins each upload's returned version id in a JCS JSON
|
||||
object keyed by `checkpoints.jsonl`, `links.jsonl`, and `manifest.json`. Sealing refuses
|
||||
missing version ids or a different version in the retention HEAD response, leaving no
|
||||
catalog row and raising the export failure alert.
|
||||
Partial upload failures leave no catalog row; retained orphan objects may remain. Retrying
|
||||
can create new locked versions under the same keys. Failures raise `wormaudit.export.failed`
|
||||
with dedup key `export`; success resolves it. Every scheduled run emits
|
||||
`wormaudit.export.daily`.
|
||||
|
||||
**Verify sealed export** accepts one or several selected records. It downloads and checks
|
||||
the object hashes, retention, manifest chain, link hashes, pinned checkpoint signatures and
|
||||
Merkle roots, then compares the sealed content and link metadata against a live database
|
||||
snapshot. Differences name exact link/audit ids without printing audit values. Each export
|
||||
gets its own `SEALED_EXPORT` verification row. Purged prefixes verify from the sealed copy
|
||||
alone and receive an INFO line. Verification does not write restored audit rows.
|
||||
|
||||
Verification compares every current HEAD version to the catalog and reports an ERROR for
|
||||
any overwrite, even when the replacement bytes are identical. It then reads the recorded
|
||||
sealed versions, so retained evidence remains verifiable after an overwrite; the overall
|
||||
verification still fails because the overwrite is a break. Missing catalog versions fail
|
||||
closed. RC.2's `StorageBucket.get` has no version parameter, so the adapter uses the public
|
||||
`ObjectStorageRuntime.connection(...).client()` accessor for an explicit S3 version GET,
|
||||
reusing its connection, key normalization and rate limiter and checking the response version.
|
||||
No object-storage change is required; this adapter depends on RC.2's public internal runtime
|
||||
accessors. The storage credentials need `s3:GetObjectVersion` in addition to the existing
|
||||
upload, HEAD and listing permissions.
|
||||
|
||||
## Build and acceptance
|
||||
|
||||
```powershell
|
||||
./mvnw.cmd -o verify
|
||||
```
|
||||
|
||||
Checkstyle and JaCoCo gates (70% instructions, 90% classes) run without Docker. Memory and H2
|
||||
tests cover canonical vectors, signatures, rollback, exact coverage, late arrivals/details,
|
||||
verification findings, PHI tokenization, process summaries, permissions, and the alerts SPI.
|
||||
The PHI stand-in uses RC.5 `AuditActionCustomizerInterface.customizeInput` before insertion;
|
||||
classification is test-local because RC.5 lacks `DataClassification`.
|
||||
|
||||
PostgreSQL acceptance uses `postgres:16-alpine`, applies the DDL twice and tests grants,
|
||||
triggers, purge, real QQQ RDBMS chaining, superuser tamper, and 8 writers / 3 chainers.
|
||||
It skips automatically without Docker locally and fails if **any** `CI` environment value is set.
|
||||
On a Docker-enabled coordinator (image already available if running without network):
|
||||
|
||||
```powershell
|
||||
$env:CI = 'true'
|
||||
./mvnw.cmd -o '-Dtest=PostgresAcceptanceTest' test
|
||||
./mvnw.cmd -o verify
|
||||
```
|
||||
|
||||
MinIO export acceptance uses the same `RequiresDockerCondition`. It checks GOVERNANCE
|
||||
with 30-day retention and a COMPLIANCE export, denied version deletion (without bypass for GOVERNANCE), overwrite version preservation, export verification,
|
||||
exact live-tamper findings through the record action, and refusal of a bucket without lock:
|
||||
|
||||
```powershell
|
||||
$env:CI = 'true'
|
||||
./mvnw.cmd -o '-Dtest=MinioExportAcceptanceTest' test
|
||||
```
|
||||
|
||||
The default MinIO image is pinned to the same Chainguard digest as object-storage's tests.
|
||||
Override it with `-Dqqq.test.minioImage=<image>` when necessary. Images must already be
|
||||
available for a completely network-free Docker run.
|
||||
|
||||
POSIX equivalents: `CI=true ./mvnw -o -Dtest=MinioExportAcceptanceTest test`,
|
||||
`CI=true ./mvnw -o -Dtest=PostgresAcceptanceTest test` and
|
||||
`CI=true ./mvnw -o verify`. No commits, publishing or deployment are part of these commands.
|
||||
|
||||
## License
|
||||
|
||||
AGPL-3.0 - See [LICENSE](LICENSE)
|
||||
[Apache License 2.0](LICENSE). Copyright Kingsrook, LLC.
|
||||
|
||||
### Physical columns and bounded polling
|
||||
|
||||
`WormAuditDdl.postgres(config)` renders table names only. Core physical columns
|
||||
must be snake_case: `audit.id`, `audit.timestamp`, `audit_detail.id`,
|
||||
`audit_detail.audit_id`, `audit_table.id/name/label/create_date/modify_date`, and
|
||||
`audit_user.id/name/create_date/modify_date`. QInstance field backend-name
|
||||
customizations do not change this migration; adapt the reviewed SQL before
|
||||
applying it if your schema differs. UTC values in `timestamp without time zone`
|
||||
and `timestamptz` are both supported.
|
||||
|
||||
`lateDetailLookbackHours` defaults to **24** (positive hours). Each PostgreSQL
|
||||
poll checks uncovered details whose chained header timestamp lies in
|
||||
`[cutoff - lookback, cutoff)`. The daily full verifier reports omissions outside
|
||||
that window. Initial header details are always chained, including overflow
|
||||
chunks. Checkpoints contain at most `checkpointEveryLinks` links; multiple full
|
||||
ranges can be signed per tick. Verification uses 1,000-link keyset pages.
|
||||
|
||||
Full verification holds at most one 1,000-link page and one checkpoint range of hashes.
|
||||
SQL completeness uses bounded offender queries; other persistent backends use repeated bounded
|
||||
scans. Only the memory backend uses a full coverage index.
|
||||
Full verification retains at most 1,000 exact break diagnostics and 100 warnings,
|
||||
while persisting the total break count. SQL completeness counts all offenders
|
||||
and reports the first 100 header ids and first 100 detail ids. This keeps a badly
|
||||
corrupted chain from exhausting verifier memory.
|
||||
@@ -0,0 +1,4 @@
|
||||
# Security Policy
|
||||
|
||||
* See Project [Security Policy](https://github.com/QRun-IO/qqq/blob/develop/SECURITY.md)
|
||||
* Report suspected vulnerabilities privately through GitHub private vulnerability reporting on this repository, never in a public issue.
|
||||
@@ -0,0 +1,246 @@
|
||||
<?xml version="1.0"?>
|
||||
<!--
|
||||
~ QQQ - Low-code Application Framework for Engineers.
|
||||
~ Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
~ 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
~ contact@kingsrook.com
|
||||
~ https://github.com/Kingsrook/
|
||||
~
|
||||
~ Licensed under the Apache License, Version 2.0 (the "License");
|
||||
~ you may not use this file except in compliance with the License.
|
||||
~ You may obtain a copy of the License at
|
||||
~
|
||||
~ https://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
|
||||
<!DOCTYPE module PUBLIC
|
||||
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
||||
"http://checkstyle.sourceforge.net/dtds/configuration_1_3.dtd">
|
||||
<!--
|
||||
See reference at https://checkstyle.sourceforge.io/checks.html
|
||||
-->
|
||||
<module name = "Checker">
|
||||
<property name="charset" value="UTF-8"/>
|
||||
|
||||
<property name="severity" value="warning"/>
|
||||
|
||||
<property name="fileExtensions" value="java, properties, xml"/>
|
||||
<!-- Checks for whitespace -->
|
||||
<!-- See http://checkstyle.sf.net/config_whitespace.html -->
|
||||
<module name="FileTabCharacter">
|
||||
<property name="eachLine" value="true"/>
|
||||
</module>
|
||||
|
||||
<!--
|
||||
<module name="LineLength">
|
||||
<property name="max" value="100"/>
|
||||
<property name="ignorePattern" value="^package.*|^import.*|a href|href|http://|https://|ftp://"/>
|
||||
</module>
|
||||
-->
|
||||
|
||||
<module name="TreeWalker">
|
||||
<module name="SuppressWarningsHolder"/>
|
||||
<module name="OuterTypeFilename"/>
|
||||
<module name="IllegalTokenText">
|
||||
<property name="tokens" value="STRING_LITERAL, CHAR_LITERAL"/>
|
||||
<property name="format" value="\\u00(09|0(a|A)|0(c|C)|0(d|D)|22|27|5(C|c))|\\(0(10|11|12|14|15|42|47)|134)"/>
|
||||
<property name="message" value="Consider using special escape sequence instead of octal value or Unicode escaped value."/>
|
||||
</module>
|
||||
<module name="AvoidEscapedUnicodeCharacters">
|
||||
<property name="allowEscapesForControlCharacters" value="true"/>
|
||||
<property name="allowByTailComment" value="true"/>
|
||||
<property name="allowNonPrintableEscapes" value="true"/>
|
||||
</module>
|
||||
<module name="AvoidStarImport"/>
|
||||
<module name="OneTopLevelClass"/>
|
||||
<module name="NoLineWrap"/>
|
||||
<module name="EmptyBlock">
|
||||
<property name="option" value="TEXT"/>
|
||||
<property name="tokens" value="LITERAL_TRY, LITERAL_FINALLY, LITERAL_IF, LITERAL_ELSE, LITERAL_SWITCH"/>
|
||||
</module>
|
||||
<module name="NeedBraces"/>
|
||||
<module name="LeftCurly">
|
||||
<property name="option" value="nl"/>
|
||||
</module>
|
||||
<module name="RightCurly">
|
||||
<property name="option" value="alone"/>
|
||||
</module>
|
||||
<!--
|
||||
<module name="WhitespaceAround">
|
||||
<property name="allowEmptyConstructors" value="true"/>
|
||||
<property name="allowEmptyMethods" value="true"/>
|
||||
<property name="allowEmptyTypes" value="true"/>
|
||||
<property name="allowEmptyLoops" value="true"/>
|
||||
<message key="ws.notFollowed"
|
||||
value="WhitespaceAround: ''{0}'' is not followed by whitespace. Empty blocks may only be represented as '{}' when not part of a multi-block statement (4.1.3)"/>
|
||||
<message key="ws.notPreceded"
|
||||
value="WhitespaceAround: ''{0}'' is not preceded with whitespace."/>
|
||||
</module>
|
||||
-->
|
||||
<module name="OneStatementPerLine"/>
|
||||
<module name="MultipleVariableDeclarations"/>
|
||||
<module name="ArrayTypeStyle"/>
|
||||
<module name="MissingSwitchDefault"/>
|
||||
<module name="FallThrough"/>
|
||||
<module name="UpperEll"/>
|
||||
<module name="ModifierOrder"/>
|
||||
<module name="EmptyLineSeparator">
|
||||
<property name="allowNoEmptyLineBetweenFields" value="true"/>
|
||||
</module>
|
||||
<module name="SeparatorWrap">
|
||||
<property name="id" value="SeparatorWrapDot"/>
|
||||
<property name="tokens" value="DOT"/>
|
||||
<property name="option" value="nl"/>
|
||||
</module>
|
||||
<module name="SeparatorWrap">
|
||||
<property name="id" value="SeparatorWrapComma"/>
|
||||
<property name="tokens" value="COMMA"/>
|
||||
<property name="option" value="EOL"/>
|
||||
</module>
|
||||
<module name="SeparatorWrap">
|
||||
<!-- ELLIPSIS is EOL until https://github.com/google/styleguide/issues/258 -->
|
||||
<property name="id" value="SeparatorWrapEllipsis"/>
|
||||
<property name="tokens" value="ELLIPSIS"/>
|
||||
<property name="option" value="EOL"/>
|
||||
</module>
|
||||
<module name="SeparatorWrap">
|
||||
<!-- ARRAY_DECLARATOR is EOL until https://github.com/google/styleguide/issues/259 -->
|
||||
<property name="id" value="SeparatorWrapArrayDeclarator"/>
|
||||
<property name="tokens" value="ARRAY_DECLARATOR"/>
|
||||
<property name="option" value="EOL"/>
|
||||
</module>
|
||||
<module name="SeparatorWrap">
|
||||
<property name="id" value="SeparatorWrapMethodRef"/>
|
||||
<property name="tokens" value="METHOD_REF"/>
|
||||
<property name="option" value="nl"/>
|
||||
</module>
|
||||
<module name="PackageName">
|
||||
<property name="format" value="^[a-z]+(\.[a-z][a-z0-9]*)*$"/>
|
||||
<message key="name.invalidPattern"
|
||||
value="Package name ''{0}'' must match pattern ''{1}''."/>
|
||||
</module>
|
||||
<module name="TypeName">
|
||||
<message key="name.invalidPattern"
|
||||
value="Type name ''{0}'' must match pattern ''{1}''."/>
|
||||
</module>
|
||||
<module name="MemberName">
|
||||
<property name="format" value="^[_a-zA-Z]([a-z0-9A-Z][a-zA-Z0-9]*)?$"/>
|
||||
<message key="name.invalidPattern" value="Member name ''{0}'' must match pattern ''{1}''."/>
|
||||
</module>
|
||||
<module name="ParameterName">
|
||||
<property name="format" value="^[a-zA-Z]([a-z0-9A-Z][a-zA-Z0-9]*)?$"/>
|
||||
<message key="name.invalidPattern" value="Parameter name ''{0}'' must match pattern ''{1}''."/>
|
||||
</module>
|
||||
<module name="CatchParameterName">
|
||||
<property name="format" value="^[a-zA-Z]([a-z0-9A-Z][a-zA-Z0-9]*)?$"/>
|
||||
<message key="name.invalidPattern" value="Catch parameter name ''{0}'' must match pattern ''{1}''."/>
|
||||
</module>
|
||||
<module name="LocalVariableName">
|
||||
<property name="tokens" value="VARIABLE_DEF"/>
|
||||
<property name="format" value="^[a-zA-Z]([a-z0-9A-Z][a-zA-Z0-9]*)?$"/>
|
||||
<message key="name.invalidPattern" value="Local variable name ''{0}'' must match pattern ''{1}''."/>
|
||||
</module>
|
||||
<module name="ClassTypeParameterName">
|
||||
<property name="format" value="(^[A-Z][0-9]?)$|([A-Z][_A-Z0-9]*$)"/>
|
||||
<message key="name.invalidPattern" value="Class type name ''{0}'' must match pattern ''{1}''."/>
|
||||
</module>
|
||||
<module name="MethodTypeParameterName">
|
||||
<property name="format" value="(^[A-Z][0-9]?)$|([A-Z][_A-Z0-9]*$)"/>
|
||||
<message key="name.invalidPattern" value="Method type name ''{0}'' must match pattern ''{1}''."/>
|
||||
</module>
|
||||
<module name="InterfaceTypeParameterName">
|
||||
<property name="format" value="(^[A-Z][0-9]?)$|([A-Z][_A-Z0-9]*$)"/>
|
||||
<message key="name.invalidPattern" value="Interface type name ''{0}'' must match pattern ''{1}''."/>
|
||||
</module>
|
||||
<module name="NoFinalizer"/>
|
||||
<module name="GenericWhitespace">
|
||||
<message key="ws.followed" value="GenericWhitespace ''{0}'' is followed by whitespace."/>
|
||||
<message key="ws.preceded" value="GenericWhitespace ''{0}'' is preceded with whitespace."/>
|
||||
<message key="ws.illegalFollow" value="GenericWhitespace ''{0}'' should followed by whitespace."/>
|
||||
<message key="ws.notPreceded" value="GenericWhitespace ''{0}'' is not preceded with whitespace."/>
|
||||
</module>
|
||||
<module name="Indentation">
|
||||
<property name="basicOffset" value="3"/>
|
||||
<property name="braceAdjustment" value="0"/>
|
||||
<property name="caseIndent" value="3"/>
|
||||
<property name="throwsIndent" value="6"/>
|
||||
<property name="lineWrappingIndentation" value="3"/>
|
||||
<property name="arrayInitIndent" value="6"/>
|
||||
</module>
|
||||
<!--
|
||||
<module name="AbbreviationAsWordInName">
|
||||
<property name="ignoreFinal" value="false"/>
|
||||
<property name="allowedAbbreviationLength" value="1"/>
|
||||
</module>
|
||||
<module name="OverloadMethodsDeclarationOrder"/>
|
||||
<module name="VariableDeclarationUsageDistance"/>
|
||||
<module name="CustomImportOrder">
|
||||
<property name="sortImportsInGroupAlphabetically" value="true"/>
|
||||
<property name="separateLineBetweenGroups" value="true"/>
|
||||
<property name="customImportOrderRules" value="STATIC###THIRD_PARTY_PACKAGE"/>
|
||||
</module>
|
||||
-->
|
||||
<module name="CustomImportOrder">
|
||||
<property name="customImportOrderRules" value="SPECIAL_IMPORTS###STANDARD_JAVA_PACKAGE###THIRD_PARTY_PACKAGE###STATIC"/>
|
||||
<property name="specialImportsRegExp" value="^javax\."/>
|
||||
<property name="standardPackageRegExp" value="^java\."/>
|
||||
<property name="sortImportsInGroupAlphabetically" value="true"/>
|
||||
<property name="separateLineBetweenGroups" value="false"/>
|
||||
</module>
|
||||
<module name="MethodParamPad"/>
|
||||
<module name="NoWhitespaceBefore">
|
||||
<property name="tokens" value="COMMA, SEMI, POST_INC, POST_DEC, DOT, ELLIPSIS, METHOD_REF"/>
|
||||
<property name="allowLineBreaks" value="true"/>
|
||||
</module>
|
||||
<module name="ParenPad"/>
|
||||
<module name="OperatorWrap">
|
||||
<property name="option" value="NL"/>
|
||||
<property name="tokens" value="BAND, BOR, BSR, BXOR, DIV, EQUAL, GE, GT, LAND, LE, LITERAL_INSTANCEOF, LOR, LT, MINUS, MOD, NOT_EQUAL, PLUS, QUESTION, SL, SR, STAR, METHOD_REF "/>
|
||||
</module>
|
||||
<module name="AnnotationLocation">
|
||||
<property name="id" value="AnnotationLocationMostCases"/>
|
||||
<property name="tokens" value="CLASS_DEF, INTERFACE_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF"/>
|
||||
</module>
|
||||
<module name="AnnotationLocation">
|
||||
<property name="id" value="AnnotationLocationVariables"/>
|
||||
<property name="tokens" value="VARIABLE_DEF"/>
|
||||
<property name="allowSamelineMultipleAnnotations" value="true"/>
|
||||
</module>
|
||||
<module name="JavadocMethod">
|
||||
<property name="allowMissingParamTags" value="true"/>
|
||||
<property name="allowMissingReturnTag" value="true"/>
|
||||
<property name="allowedAnnotations" value="Override, Test"/>
|
||||
</module>
|
||||
<module name="MissingJavadocMethod">
|
||||
<property name="scope" value="private"/>
|
||||
</module>
|
||||
<module name="MissingJavadocType">
|
||||
<property name="scope" value="private"/>
|
||||
</module>
|
||||
<module name="MethodName">
|
||||
<property name="format" value="^[a-z][a-zA-Z0-9_]*$"/>
|
||||
<message key="name.invalidPattern"
|
||||
value="Method name ''{0}'' must match pattern ''{1}''."/>
|
||||
</module>
|
||||
<module name="EmptyCatchBlock">
|
||||
<property name="exceptionVariableName" value="expected"/>
|
||||
</module>
|
||||
<module name="CommentsIndentation"/>
|
||||
|
||||
<!-- this would be nice, but requires the the @inheritDoc javadoc tag... -->
|
||||
<module name="MissingOverride"/>
|
||||
|
||||
</module>
|
||||
<module name="Header">
|
||||
<property name="headerFile" value="checkstyle/license.txt"/>
|
||||
<property name="fileExtensions" value="java"/>
|
||||
<property name="ignoreLines" value="3"/>
|
||||
</module>
|
||||
<module name="SuppressWarningsFilter"/>
|
||||
</module>
|
||||
@@ -0,0 +1,19 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
@@ -1,42 +0,0 @@
|
||||
# Getting Started
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Java 17+
|
||||
- Maven 3.8+
|
||||
- QQQ 0.23.0+
|
||||
|
||||
## Creating Your Extension QBit
|
||||
|
||||
1. Click "Use this template" on GitHub
|
||||
2. Clone your new repository
|
||||
3. Run the customization script:
|
||||
```bash
|
||||
python scripts/customize_template.py
|
||||
```
|
||||
4. Enter your QBit name when prompted
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
src/main/java/com/kingsrook/qbits/yourextension/
|
||||
├── YourExtensionQBitConfig.java
|
||||
├── YourExtensionQBitProducer.java
|
||||
├── customizers/
|
||||
└── actions/
|
||||
```
|
||||
|
||||
## Using Your QBit
|
||||
|
||||
```java
|
||||
new YourExtensionQBitProducer()
|
||||
.withConfig(new YourExtensionQBitConfig()
|
||||
.withTargetTableName("order"))
|
||||
.produce(qInstance, "myExtension");
|
||||
```
|
||||
|
||||
## Next Steps
|
||||
|
||||
- Read [Extension Patterns](01-extension-patterns.md)
|
||||
- Implement your customization logic
|
||||
- Add tests
|
||||
@@ -1,58 +0,0 @@
|
||||
# Extension Patterns
|
||||
|
||||
## Table Customizers
|
||||
|
||||
Add behavior to existing tables:
|
||||
|
||||
```java
|
||||
public class AuditTableCustomizer
|
||||
{
|
||||
public void customize(QTableMetaData table)
|
||||
{
|
||||
table.withField(new QFieldMetaData("createdAt", QFieldType.DATE_TIME));
|
||||
table.withField(new QFieldMetaData("modifiedAt", QFieldType.DATE_TIME));
|
||||
|
||||
table.withCustomizer(
|
||||
TableCustomizers.PRE_INSERT_RECORD,
|
||||
new QCodeReference(AuditPreInsert.class));
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Action Customizers
|
||||
|
||||
Wrap actions with additional behavior:
|
||||
|
||||
```java
|
||||
public class LoggingActionWrapper implements PreInsertRecordInterface
|
||||
{
|
||||
@Override
|
||||
public void preInsertRecord(InsertInput input, QRecord record) throws QException
|
||||
{
|
||||
LOG.info("Inserting record", logPair("table", input.getTableName()));
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Config Validation
|
||||
|
||||
Validate that required components exist:
|
||||
|
||||
```java
|
||||
@Override
|
||||
public void validate(QInstance qInstance, List<String> errors)
|
||||
{
|
||||
if(qInstance.getTable(targetTableName) == null)
|
||||
{
|
||||
errors.add("Target table required: " + targetTableName);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## When to Use Extension QBits
|
||||
|
||||
- Adding audit logging to tables
|
||||
- Implementing authentication/authorization
|
||||
- Adding encryption to fields
|
||||
- Custom validation rules
|
||||
- Performance monitoring
|
||||
@@ -0,0 +1,422 @@
|
||||
# QBit WORM Audit: design
|
||||
|
||||
Status: approved. **Owner decisions (2026-10-12): all recommendations accepted.** Every open
|
||||
question in section 12 is decided as recommended.
|
||||
|
||||
## 1. Owner request
|
||||
|
||||
> "we also need to ensure that our auditing qbit supports WORM audits as well. and the phi/secure
|
||||
> data qbit needs to ensure that we don't leak phi into audits"
|
||||
|
||||
Context: HIPAA §164.312(b) audit controls, §164.312(c)(1) integrity, §164.316(b)(2)(i) six-year
|
||||
retention of required documentation (audit records are kept to the same horizon), and a
|
||||
HITRUST-style review that asks three things: (1) can anyone change or delete an audit record,
|
||||
(2) would we know if they did, and (3) is there a copy nobody can alter?
|
||||
|
||||
## 2. Goals and non-goals
|
||||
|
||||
Goals:
|
||||
1. **Append-only at the database level.** No application path, and no ordinary database login,
|
||||
can UPDATE, DELETE or TRUNCATE audit rows. The only removal is an audited retention purge,
|
||||
after the retention date, through a separate database role.
|
||||
2. **Tamper evidence.** Every audit row is linked into a hash chain. Signed checkpoints anchor the
|
||||
chain. A verification action and a daily job report the exact first broken row.
|
||||
3. **Sealed copies.** New audit rows and checkpoints are exported on a schedule to an S3 bucket
|
||||
with Object Lock in COMPLIANCE mode (default six years), and can be verified against the live
|
||||
chain.
|
||||
4. **No PHI in the WORM layer.** The chain and the exports seal exactly what core wrote. With
|
||||
qbit-secured-data that means HMACs for classified values, never plaintext.
|
||||
|
||||
Non-goals: changing what core audits (that is core's and qbit-secured-data's job); protecting
|
||||
against a Postgres superuser *preventing* a change (a superuser can disable triggers; the
|
||||
chain, the signatures and the Object Lock copies make that change *detectable*, see 4.5).
|
||||
|
||||
## 3. Scope: which tables
|
||||
|
||||
| Table (QQQ name / kof22 physical) | Treatment |
|
||||
|---|---|
|
||||
| `audit` / `audit` | Append-only (UPDATE, DELETE, TRUNCATE rejected). Chained. |
|
||||
| `auditDetail` / `audit_detail` | Append-only. Chained with its header. |
|
||||
| `auditTable` / `audit_table` | Lookup. DELETE and TRUNCATE rejected; UPDATE of `id` or `name` rejected (label and dates may change). Names are resolved into the chained content, so a later rename is detected. |
|
||||
| `auditUser` / `audit_user` | As `audit_table`. |
|
||||
| `worm_audit_chain`, `worm_audit_checkpoint`, `worm_audit_export`, `worm_audit_verification`, `worm_audit_purge` | The qbit's own tables: all append-only. |
|
||||
|
||||
Physical names come from the host's backend (kof22 maps camelCase to snake_case). The DDL is a
|
||||
template rendered for the host's names (section 5.4).
|
||||
|
||||
## 4. Append-only enforcement (Postgres first)
|
||||
|
||||
### 4.1 Triggers
|
||||
One function, `worm_audit_reject()`, attached as:
|
||||
- `BEFORE UPDATE OR DELETE ... FOR EACH ROW` on every append-only table;
|
||||
- `BEFORE TRUNCATE ... FOR EACH STATEMENT` on every append-only table;
|
||||
- on the two lookups, a row trigger that rejects DELETE and any UPDATE that changes `id` or
|
||||
`name`.
|
||||
|
||||
The error is explicit and greppable:
|
||||
|
||||
```
|
||||
ERROR: worm_audit: audit rows are write-once; UPDATE of "audit" is not allowed (id=4211). Expired rows are removed only by worm_audit_purge().
|
||||
SQLSTATE: P0001 (HINT: see qbit-worm-audit docs/DESIGN.md section 4)
|
||||
```
|
||||
|
||||
In the application the qbit also removes the update and delete capabilities from these tables'
|
||||
QQQ metadata, so the admin UI and both APIs never offer edit or delete, and a direct call gets
|
||||
QQQ's standard "table does not support update/delete" error before reaching the database. The
|
||||
qbit maps a `P0001` with the `worm_audit:` prefix to a `QUserFacingException`: "Audit records are
|
||||
write-once and cannot be changed or deleted."
|
||||
|
||||
### 4.2 The retention purge (the only removal path)
|
||||
`worm_audit_purge(p_cutoff timestamptz)` is a `SECURITY DEFINER` function owned by the schema
|
||||
owner. `EXECUTE` is revoked from `PUBLIC` and granted only to the role `worm_audit_purger`, which
|
||||
the app role is not a member of.
|
||||
|
||||
1. It refuses (and records a `REFUSED` row in `worm_audit_purge`, then returns; no exception, so
|
||||
the record survives) when `p_cutoff > now() - retention`. The retention interval is a constant
|
||||
compiled into the function by the changeset (default `6 years`), so shortening it is a schema
|
||||
migration, reviewed and itself audited.
|
||||
2. It purges only a **chain prefix**: the audits whose chain links all lie at or before the last
|
||||
link whose audit timestamp is older than the cutoff, together with their details and links.
|
||||
3. It sets a transaction-local flag (`set_config('worm_audit.purging', 'on', true)`). The reject
|
||||
trigger allows DELETE only when that flag is on **and** `current_user` is the function owner
|
||||
**and** the row is older than the retention interval. Setting the flag from another session
|
||||
does nothing, because `current_user` is not the owner.
|
||||
4. It writes a `PURGED` row: who (session_user), cutoff, retention, counts, and the **anchor**
|
||||
(the last purged link's id and hash). Verification continues from the anchor.
|
||||
|
||||
The purge runs from an ops runbook (`psql` as `worm_audit_purger`). Version 0.1 has no UI action;
|
||||
the app home lists purge rows, including refusals.
|
||||
|
||||
### 4.3 GRANT model
|
||||
|
||||
| Role | Owns | Grants on append-only tables |
|
||||
|---|---|---|
|
||||
| `<app>_owner` (Liquibase) | tables, functions, triggers | all (owner) |
|
||||
| `<app>` (runtime) | nothing | `SELECT, INSERT`; `USAGE` on their sequences. No UPDATE, DELETE or TRUNCATE. |
|
||||
| `worm_audit_purger` (ops) | nothing | `EXECUTE worm_audit_purge`; nothing else |
|
||||
|
||||
When a host uses one role for both migrations and runtime (kof22 today), the triggers still stop
|
||||
every UPDATE, DELETE and TRUNCATE from the application, but the role can `ALTER TABLE ... DISABLE
|
||||
TRIGGER`. The docs recommend splitting the roles, and the app home shows a warning when the
|
||||
runtime role owns the audit tables (checked through `pg_tables.tableowner = current_user`).
|
||||
|
||||
### 4.4 Changesets
|
||||
The qbit ships `src/main/resources/com/kingsrook/qbits/wormaudit/ddl/postgres/worm-audit.sql`,
|
||||
a Liquibase formatted-SQL template with placeholders for the four core audit table names, the
|
||||
app role and the retention interval, plus a renderer (`WormAuditDdl.postgres(config)`) and the
|
||||
rendered default (`docs/ddl/worm-audit-postgres.sql`, snake_case names). Core audit physical
|
||||
columns must use snake_case (for example `audit_table_id`, `audit_id`, `new_value`); hosts map
|
||||
QQQ field names to those columns. Hosts copy the rendered
|
||||
file into their changelog (kof22: `changesets/features/YYYY-MM-DD-worm-audit.sql`). It creates the
|
||||
qbit tables, the functions, the triggers, the purger role (`CREATE ROLE ... NOLOGIN` when it is
|
||||
missing) and the grants. It is idempotent (`CREATE OR REPLACE`, `DROP TRIGGER IF EXISTS`).
|
||||
|
||||
### 4.5 Threat model summary
|
||||
|
||||
| Actor | Can modify audit rows? | Detected by |
|
||||
|---|---|---|
|
||||
| App user, admin UI, QQQ APIs, assistant | No (capabilities removed; triggers) | n/a |
|
||||
| SQL injection or a bug as the runtime role | No (triggers; no grant) | n/a |
|
||||
| Owner role (migrations) | Only by disabling triggers | chain verification (daily), alert |
|
||||
| Postgres superuser rewriting rows and recomputing the chain | Yes | checkpoint signatures (no signing key in the DB); sealed exports in Object Lock |
|
||||
| Anyone deleting the sealed copies | No (COMPLIANCE mode: not even the bucket root, until expiry) | n/a |
|
||||
|
||||
### 4.6 Other databases (documented, not shipped in 0.1)
|
||||
- **MySQL/MariaDB:** `BEFORE UPDATE`/`BEFORE DELETE` triggers with `SIGNAL SQLSTATE '45000'`;
|
||||
no TRUNCATE trigger, so revoke `DROP` from the app user; purge through a `SQL SECURITY DEFINER`
|
||||
procedure.
|
||||
- **SQL Server:** `INSTEAD OF UPDATE, DELETE` triggers plus `DENY UPDATE, DELETE` to the app user.
|
||||
- **H2** (tests only): no enforcement; the chain and the exports work on any QQQ RDBMS or memory
|
||||
backend, so tamper *evidence* is database-independent.
|
||||
|
||||
## 5. Tamper evidence
|
||||
|
||||
### 5.1 Canonical content (agreed with qbit-secured-data)
|
||||
One canonical document per audit header: the header plus all of its detail rows, as RFC 8785 JCS
|
||||
JSON (sorted keys, no insignificant whitespace, UTF-8):
|
||||
|
||||
```json
|
||||
{"audit":{"clientChannel":null,"id":4211,"message":"Record was Edited","recordId":"17","table":"cms_page","timestamp":"2026-10-13T21:04:05.123456Z","user":"james"},
|
||||
"details":[{"fieldName":"title","id":9001,"message":"Changed Title from ...","newValue":"...","oldValue":"..."}],
|
||||
"v":1}
|
||||
```
|
||||
|
||||
- Values are exactly as stored: no formatting, no re-masking, no lookups other than resolving
|
||||
`auditTableId`/`auditUserId` to their names. `details` are sorted by id.
|
||||
- With qbit-secured-data's core hook (QRun-IO/qqq#1161, `docs/SECURED-DATA.md` §14.2), PII and
|
||||
PHI values are tokenised inside `DMLAuditAction` before `AuditAction` inserts, in the same
|
||||
transaction as the data change, so the token is the only form ever written. Stored forms (ASCII,
|
||||
at most 200 characters) in `oldValue`/`newValue` are
|
||||
`hmac:v1:<keyId>:<b64url>`, `shred:v1:<auditKeyId>:<b64url>`, `changed:v1`,
|
||||
`wiped:v1:<ruleId>` and `shredded:v1:<ruleId>` (the last is its own detail row, written when
|
||||
keys are destroyed, and is chained like any other row). Detail messages for those fields
|
||||
are replaced with value-free ones ("Changed <Label>"); header messages carry no field values.
|
||||
The canonical form, the chain and the exports therefore contain tokens only, and verifying the
|
||||
chain never needs a key. The WORM layer never reads, joins or exports
|
||||
`secured_data_audit_key` or any other `secured_data_*` table. Crypto-shredding (destroying a
|
||||
record's key) leaves the HMACs and the chain intact; they simply become unlinkable.
|
||||
- The canonical JSON is never stored in the database. The chain stores ids and hashes only, so
|
||||
the qbit adds no new copy of audit content to the database.
|
||||
|
||||
### 5.2 The chain: a single asynchronous chainer (recommended)
|
||||
`link_hash = SHA-256(prev_link_hash || 0x00 || canonical_bytes)`, hex, with the genesis
|
||||
`prev_link_hash` = 64 zeros.
|
||||
|
||||
Core inserts audits from many threads, and headers and details may be inserted in separate
|
||||
autocommit statements (core passes the caller's transaction, which may be null). Computing the
|
||||
chain in core's write path would need a global lock on every audited write, and a trigger-based
|
||||
chain would need the canonical form in SQL. The design instead uses **one chainer**:
|
||||
|
||||
- A scheduled job (every 30 s by default) takes `pg_try_advisory_xact_lock` (any other instance
|
||||
skips that tick; on non-Postgres backends, a single-row lock table with `SELECT ... FOR
|
||||
UPDATE`). It selects audit headers that are **older than a settle horizon** (default 60 s, so
|
||||
their details have been committed) and not yet chained (an anti-join on the unique
|
||||
`worm_audit_chain.audit_id`), in `(timestamp, id)` order. Then it appends links in batches.
|
||||
- Link ids are assigned by the chainer as `max(id) + 1` inside the lock, not from a sequence, so
|
||||
the chain is contiguous (a sequence gaps on rollback).
|
||||
- **Late arrivals** (a header whose id sorts before already-chained ones because its transaction
|
||||
committed late) are chained when they appear. The chain order is chaining order, not audit id
|
||||
order, and each link records its `audit_id`.
|
||||
- **Late details** (a detail inserted after its header was chained) get their own link,
|
||||
`kind = LATE_DETAIL`, covering exactly those detail ids. Verification flags any detail not
|
||||
covered by exactly one link.
|
||||
- Unique constraints back the lock: `UNIQUE (audit_id) WHERE kind = 'AUDIT'`, and the detail
|
||||
coverage.
|
||||
|
||||
Why not per-partition chains: they make each partition's chain shorter but multiply heads,
|
||||
checkpoints and verification state, and they still need a writer per partition. One chainer
|
||||
handles thousands of rows a second in batches, which is well above audit volume, and keeps a
|
||||
single head to sign. The table carries no partition column (YAGNI). A future partitioned mode
|
||||
would add one.
|
||||
|
||||
Window: a row is unchained for at most horizon plus interval (about 90 s). During that window the
|
||||
triggers still protect it. The chain defends against privileged tampering, which the window does
|
||||
not change materially.
|
||||
|
||||
### 5.3 Signed checkpoints (`worm_audit_checkpoint`)
|
||||
Every hour, and every 10,000 links, the chainer writes a checkpoint for the links since the last
|
||||
one: `from_link_id`, `to_link_id`, `head_hash` (the last link hash), `merkle_root` (a binary
|
||||
SHA-256 Merkle tree over the range's link hashes, so a single exported range can be proven
|
||||
without the whole chain), `key_id`, `public_key`, `algorithm = Ed25519` and `signature` over the
|
||||
JCS form of those fields. The signing key comes from config (`WormAuditQBitConfig.signingKey`, a
|
||||
supplier; kof22: env `WORM_AUDIT_SIGNING_KEY` from Infisical) and never touches the database. The
|
||||
public key is stored on each checkpoint and pinned in config (`trustedPublicKeys`), so a forged
|
||||
checkpoint signed with another key fails verification. Rotation adds a new key id. Old
|
||||
checkpoints verify against their own pinned key.
|
||||
|
||||
### 5.4 Verification
|
||||
**"Verify audit chain"**, a table action on `worm_audit_chain` (and on selected links), is a
|
||||
standard streamed ETL:
|
||||
- Extract: the links in scope (all since the purge anchor, or the selection), in id order.
|
||||
- Transform: recompute each link from the live audit rows (`prev` continuity, canonical content,
|
||||
`link_hash`); check checkpoint signatures and Merkle roots; for a full run, check completeness
|
||||
(every settled audit and detail is covered exactly once).
|
||||
- Load: one `worm_audit_verification` row.
|
||||
- Review: "1,204,331 chain links will be verified (links 1–1,204,331, 121 checkpoints)."
|
||||
- Result: "1,204,331 links verified" or, on failure, one line per break, each with the exact row:
|
||||
"Break at link 88,412 (audit 90,117, table cms_page, record 17): content differs from the
|
||||
chained hash" / "audit_detail 9,002 is not covered by the chain" / "checkpoint 41: signature
|
||||
does not verify".
|
||||
|
||||
**Daily verification**: a scheduled job runs the same verifier at 03:00. A failure raises
|
||||
`QAlerts.raise(new AlertSignal("wormaudit.chain.break").withDedupKey("chain")...)` with the first
|
||||
break in the title and the verification record as the record ref; a later pass resolves it. Every
|
||||
run sends `QAlerts.heartbeat("wormaudit.verify.daily")`. `QAlerts` is the core producer SPI
|
||||
(QQQ 4.1.0-RC.5), so the qbit needs no dependency on qbit-alerts. With no alerts implementation
|
||||
installed the alerts go to the log.
|
||||
|
||||
## 6. Sealed exports
|
||||
|
||||
### 6.1 Export job
|
||||
|
||||
**Default superseded by Owner decisions (2026-10-14) below:** lock mode and export retention
|
||||
now come from Settings, defaulting to GOVERNANCE and 30 days. The original decision text is
|
||||
retained for history.
|
||||
Daily at 02:00 (configurable), and by the table action "Export now" on `worm_audit_export`:
|
||||
1. Range: links after the last export's `to_link_id`, up to the latest checkpoint. Exports are
|
||||
contiguous by construction. The app home flags a gap (a range not covered) and lag.
|
||||
2. Objects under `<prefix>/<yyyy>/<mm>/<dd>/<from>-<to>/`:
|
||||
- `links.jsonl`: one line per link: `{"link":{id,kind,auditId,detailIds,prevHash,linkHash},"content":<canonical document>}`;
|
||||
- `checkpoints.jsonl`: the checkpoints in range, with signatures;
|
||||
- `manifest.json`: format version, range, counts, the SHA-256 of each file, `head_hash`,
|
||||
the previous export's manifest SHA-256 (the exports form their own chain), qbit version.
|
||||
3. Each object is put through qbit-object-storage (`SystemStorage`, bucket alias from config,
|
||||
default `worm_audit`) with `LockMode.COMPLIANCE` and `retainUntil = now + retentionYears`. The
|
||||
export refuses to run, and alerts `wormaudit.export.failed`, when the bucket does not report
|
||||
the `OBJECT_LOCK` capability or the put does not echo the lock.
|
||||
4. On success it inserts one `worm_audit_export` row (insert-only; a failed export leaves no row,
|
||||
and its alert carries the reason). Every scheduled run sends `wormaudit.export.daily`.
|
||||
|
||||
### 6.2 Restore and verify a sealed export
|
||||
"Verify sealed export", a record action on `worm_audit_export` (standard ETL), downloads the
|
||||
exact object versions pinned in the catalog's `object_versions` map. A new version under an
|
||||
existing key is an overwrite even when bytes and retention match: verification checks the latest
|
||||
version against the pin and rejects replacements, then reads and checks the pinned version.
|
||||
It checks the manifest hashes, the lock mode and retain-until of each object, the link
|
||||
hashes recomputed from the exported content, the checkpoint signatures, and the export's link to
|
||||
the previous manifest. Then it compares each exported link and content document with the live
|
||||
chain and the live audit rows. Result lines name each difference ("link 88,412: live content
|
||||
differs from the sealed copy (sealed 2026-10-14)"). This is also the restore tool: the exported
|
||||
content is the canonical audit content, so a verified export is evidence of the original rows
|
||||
after a live tamper. A purged range stays verifiable from the sealed copy alone. Writing restored
|
||||
rows back into a live table is out of scope (it would need the purge role and break append-only).
|
||||
|
||||
## 7. Tables
|
||||
|
||||
All tables use the host's RDBMS backend, are append-only (section 4), are audited by nothing
|
||||
(auditing the audit tables would recurse), have full record screens, and are searchable in Quick
|
||||
Search. ESB table events are on for `worm_audit_verification` and `worm_audit_export` (the
|
||||
events integrations care about); chain links are high-volume and publish none.
|
||||
|
||||
- **`worm_audit_chain`** [Audit Chain Link]: `id` (contiguous), `kind` (AUDIT, LATE_DETAIL),
|
||||
`audit_id`, `detail_ids` (comma list, at most 250 per link), `prev_hash`, `link_hash`,
|
||||
`chained_at`. Unique `(audit_id) where kind = 'AUDIT'`.
|
||||
- **`worm_audit_checkpoint`** [Checkpoint]: `id`, `from_link_id`, `to_link_id`, `link_count`,
|
||||
`head_hash`, `merkle_root`, `algorithm`, `key_id`, `public_key`, `signature`, `created_at`.
|
||||
- **`worm_audit_export`** [Sealed Export]: `id`, `from_link_id`, `to_link_id`,
|
||||
`from_checkpoint_id`, `to_checkpoint_id`, `bucket_alias`, `object_prefix`, `object_versions`, `manifest_key`,
|
||||
`manifest_sha256`, `prev_manifest_sha256`, `object_count`, `byte_count`, `lock_mode`,
|
||||
`retain_until`, `sealed_at`.
|
||||
- **`worm_audit_verification`** [Verification]: `id`, `kind` (CHAIN, SEALED_EXPORT), `trigger`
|
||||
(SCHEDULE, USER), `started_at`, `finished_at`, `from_link_id`, `to_link_id`, `links_checked`,
|
||||
`outcome` (PASS, FAIL), `break_count`, `first_break_link_id`, `first_break_audit_id`,
|
||||
`first_break_reason`, `export_id`, `run_by`.
|
||||
- **`worm_audit_purge`** [Purge]: `id`, `requested_at`, `requested_by`, `cutoff`,
|
||||
`retention_interval`, `outcome` (PURGED, REFUSED), `reason`, `audits_deleted`,
|
||||
`details_deleted`, `links_deleted`, `anchor_link_id`, `anchor_hash`.
|
||||
|
||||
Nav (the app "WORM Audit"): Home, Verifications, Sealed Exports, Checkpoints, Audit Chain, Purges.
|
||||
Checkpoints list their links as a child list. Exports and verifications link their checkpoints
|
||||
and links.
|
||||
|
||||
## 8. Permissions and the assistant wall
|
||||
- Table permissions use the full name (`worm_audit_chain.read`, ...). There are no insert, edit or
|
||||
delete permissions to grant: the capabilities are absent.
|
||||
- Process permissions: `worm_audit.verify`, `worm_audit.export`. The home is visible with read on
|
||||
any `worm_audit_*` table.
|
||||
- The assistant channel is denied `worm_audit_*`. The qbit registers
|
||||
`ClientChannelsMetaData.ofOrWithNew(qInstance).withDeniedPattern("assistant", "worm_audit_*")`
|
||||
(core, QQQ 4.1.0-RC.6, #1141). On RC.5 it does so reflectively when the class is present. A
|
||||
one-line core PR also adds `worm_audit_*` to `ASSISTANT_DEFAULT_DENIED_PATTERNS`, so the wall
|
||||
holds even if the qbit is configured without its producer. Core's `audit*` pattern already
|
||||
covers the core audit tables.
|
||||
|
||||
## 9. App home
|
||||
A real home that owns the data (no duplicate dashboard), built from standard widgets, served
|
||||
through `/qqq/v1/widget` like every QQQ widget:
|
||||
- **Chain status**: links chained, last link time, unchained backlog (with the settle horizon),
|
||||
last verification (time, outcome, links), open breaks with the first break's exact row and a
|
||||
link to the verification record.
|
||||
- **Export status**: last sealed export (range, time, retain-until), lag in links and hours,
|
||||
gaps, and bucket lock state (COMPLIANCE, retention).
|
||||
- **Retention**: configured years, the purge anchor, the oldest live audit, the next eligible
|
||||
purge date.
|
||||
- **Recent purges**: the last 10 purge rows, refusals included.
|
||||
- **Setup warnings**: triggers missing on any table (checked via `pg_trigger`), runtime role owns
|
||||
the audit tables, signing key or trusted keys missing, bucket without Object Lock.
|
||||
|
||||
The home must be readable at every width (390 to 1920 px) in light and dark mode, with no inner
|
||||
horizontal scroll. Screenshots go in `docs/screenshots/`. If a custom endpoint is ever needed, it
|
||||
registers as a `MiddlewareSpecExtension` under `/qqq/v1/qbit/worm-audit/...`
|
||||
(docs/UI-API-EXTENSIONS.md). The design needs none.
|
||||
|
||||
## 10. Configuration (`WormAuditQBitConfig`)
|
||||
`retentionYears` (6), `signingKey` (supplier of an Ed25519 private key), `trustedPublicKeys`
|
||||
(key id to public key), `chainIntervalSeconds` (30), `settleHorizonSeconds` (60),
|
||||
`checkpointEveryLinks` (10,000), `checkpointEveryMinutes` (60), `exportBucketAlias` (`worm_audit`),
|
||||
`exportPrefix` (`worm-audit`), `exportCron` (02:00 daily), `verifyCron` (03:00 daily),
|
||||
`coreAuditTableNames` (physical names, defaulting to QQQ's), `tableMetaDataCustomizer`.
|
||||
|
||||
## 11. Tests (acceptance)
|
||||
1. **Postgres Testcontainers** (`postgres:16`): the rendered DDL applies twice (idempotent);
|
||||
UPDATE, DELETE and TRUNCATE on `audit`, `audit_detail` and each `worm_audit_*` table fail with
|
||||
the `worm_audit:` message; lookup renames fail and label changes succeed; the runtime role has
|
||||
no UPDATE/DELETE grant; `worm_audit_purge` refuses a recent cutoff (and records it), purges an
|
||||
expired prefix as the purger role, records the anchor, and verification passes from the
|
||||
anchor; setting `worm_audit.purging` from the app session does not allow DELETE.
|
||||
2. **Chain**: chain, verify (PASS); a forced tamper (triggers disabled as superuser, one
|
||||
`audit_detail.new_value` changed) is detected, naming that exact link, audit and detail; a
|
||||
deleted row, an inserted row, a reordered link and a forged checkpoint are each detected.
|
||||
3. **Concurrency**: 8 threads inserting audits (with late commits and details in separate
|
||||
statements) while 3 chainers run. The chain is contiguous, every audit and detail is covered
|
||||
exactly once, and verification passes.
|
||||
4. **Exports (MinIO Testcontainers, bucket with Object Lock)**: export, then verify the round
|
||||
trip; every object has COMPLIANCE retention; deleting or overwriting an object version is
|
||||
refused by MinIO; a tampered live row is reported by "Verify sealed export"; a bucket without
|
||||
Object Lock makes the export refuse and alert.
|
||||
5. **PHI**: a field classified PHI (`AiFieldMetaData.withClassification(DataClassification.PHI)`,
|
||||
read through `AiDataClassifications.ofField`) is written with the value `CANARY-PHI-<random>`.
|
||||
With secured-data's audit token hook (a stand-in producing the same `hmac:v1:` form in the
|
||||
qbit's tests until the core hook ships), the audit header, the details (old, new, message),
|
||||
the canonical content and the export JSONL contain neither the canary nor any substring of it
|
||||
longer than 4 characters.
|
||||
6. **Alerts**: a break raises `wormaudit.chain.break` through `QAlerts` (captured by a test
|
||||
sink); a later pass resolves it.
|
||||
7. Coverage gates: 70% instructions, 90% classes.
|
||||
|
||||
## 12. Decisions (all accepted as recommended, 2026-10-12)
|
||||
| # | Question | Decision |
|
||||
|---|---|---|
|
||||
| 1 | Chain construction | A single asynchronous chainer with a settle horizon, contiguous link ids and late-arrival links (5.2), not synchronous or per-partition chains. |
|
||||
| 2 | Chain storage | A sidecar table (`worm_audit_chain`), not columns on core tables, so core's schema is untouched and no UPDATE of audit rows is ever needed. |
|
||||
| 3 | Checkpoints | Hourly or every 10,000 links; Ed25519; Merkle root per range. |
|
||||
| 4 | Lookup tables | DELETE and id/name changes rejected; names resolved into the content. |
|
||||
| 5 | Purge | SQL function plus a purger role only, chain-prefix only, no UI action in 0.1. |
|
||||
| 6 | Export | Daily JSONL plus manifest, COMPLIANCE, six years, contiguous ranges, the exports form their own chain. |
|
||||
| 7 | Databases | Postgres enforced; MySQL and SQL Server documented; evidence on any backend. |
|
||||
|
||||
## 13. Phases
|
||||
- **(a)** DDL and renderer, capabilities guard, chain and chainer, checkpoints, verification
|
||||
action and daily job with alerts, the assistant deny pattern; tests 1–3, 5 and 6.
|
||||
- **(b)** Sealed exports and the export verifier through qbit-object-storage; test 4.
|
||||
- **(c)** App home and setup warnings, screenshots; kof22 adoption (draft Website-Backend PR, a
|
||||
feature changeset) and the infra proposal for an Object Lock bucket (section 14).
|
||||
|
||||
## 14. Adoption infrastructure (kof22, proposal only)
|
||||
- **Bucket:** `kof22-worm-audit` on the kof22 MinIO, created by `minio-bucket-init` with
|
||||
`mc mb --with-lock --ignore-existing` and `mc retention set --default COMPLIANCE 6y`. Object
|
||||
Lock can only be enabled at bucket creation.
|
||||
- **Policy:** `kof22-app` gets `s3:PutObject`, `s3:GetObject`, `s3:GetObjectVersion`, `s3:ListBucket`,
|
||||
`s3:GetObjectRetention`, `s3:PutObjectRetention` (needed to set the lock on each put),
|
||||
`s3:GetBucketObjectLockConfiguration` on that bucket. No delete, and no
|
||||
`s3:BypassGovernanceRetention` (irrelevant under COMPLIANCE anyway).
|
||||
- **Off-site:** COMPLIANCE on a single homelab MinIO stops every API deletion, but not someone
|
||||
destroying the volume. Recommend extending `media-offsite-copy` to replicate the bucket to a B2
|
||||
bucket with Object Lock (COMPLIANCE, 6 years).
|
||||
- **Secrets:** `WORM_AUDIT_SIGNING_KEY` (Ed25519 PKCS#8, base64) generated in Praetor
|
||||
`terraform/infisical-config` and delivered by ESO. The public key goes in the backend config.
|
||||
- **Database:** split the kof22 owner and runtime roles when convenient (4.3). Until then the
|
||||
home shows the owner warning.
|
||||
|
||||
These are owner-run (Praetor and cluster-gitops are applied by the owner). The qbit only
|
||||
documents them.
|
||||
|
||||
## Owner decisions (2026-10-14)
|
||||
|
||||
The 2026-10-12 decisions above remain the historical design. These decisions supersede the
|
||||
COMPLIANCE/six-year export default in §6.1 and the corresponding default assumptions in
|
||||
§§2, 4.5, 9, 11, 12 and the §14 infrastructure proposal.
|
||||
|
||||
- Settings live in the application as the standard QQQ `worm_audit_settings` table, record
|
||||
ID 1, labelled WORM Audit Settings and navigated as Settings after Purges. It allows edits,
|
||||
has no append-only triggers, and uses FIELD-level core audit so every change is chained.
|
||||
Reading needs `worm_audit_settings.read`; editing additionally needs
|
||||
`worm_audit_settings.edit` and `worm_audit.settings`. New IDs and deletion are refused.
|
||||
- Defaults are seeded insert-only in new changeset `worm-audit:10`; existing changesets are
|
||||
unchanged. An administrator's edits survive subsequent seeding. Config supplies defaults
|
||||
when a row/field/table is absent. Runtime reads cache at most 30 seconds, scoped to the
|
||||
QInstance. Chain interval and UTC Quartz export/verify schedules take effect after restart.
|
||||
- kof22 starts with **GOVERNANCE, 30 days** for sealed exports. An administrator with
|
||||
`s3:BypassGovernanceRetention` can remove those copies. This is deliberately reversible;
|
||||
it does not provide COMPLIANCE's irreversible retention. Runtime credentials still receive
|
||||
no delete or bypass permission. Bucket defaults should match the selected settings.
|
||||
- Changing Settings to COMPLIANCE affects new exports. Existing GOVERNANCE versions retain
|
||||
their original mode unless an administrator extends/upgrades them. Verification checks
|
||||
the catalog's recorded mode and date, so external mode changes are reported as mismatches.
|
||||
COMPLIANCE cannot be undone. The settings change itself is audited and chained.
|
||||
- Audit retention defaults to six years independently of export retention. The home warns
|
||||
about a shorter export horizon and a mismatch with the purge function's compiled interval.
|
||||
Updating the setting does not modify the purge function; that still requires a migration.
|
||||
|
||||
The README Settings table specifies all fields, defaults, bounds and timing. Signing keys,
|
||||
public-key pins, bucket alias, prefix and backend remain host configuration, never settings.
|
||||
@@ -0,0 +1,353 @@
|
||||
-- liquibase formatted sql
|
||||
|
||||
-- changeset worm-audit:1
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_chain (
|
||||
id bigint PRIMARY KEY,
|
||||
kind text NOT NULL,
|
||||
audit_id bigint NOT NULL,
|
||||
detail_ids text NOT NULL,
|
||||
prev_hash text NOT NULL,
|
||||
link_hash text NOT NULL,
|
||||
chained_at timestamptz NOT NULL,
|
||||
CHECK (id > 0),
|
||||
CHECK (kind IN ('AUDIT', 'LATE_DETAIL')),
|
||||
CHECK (prev_hash ~ '^[0-9a-f]{64}$' AND link_hash ~ '^[0-9a-f]{64}$'),
|
||||
CHECK (detail_ids = '' OR (detail_ids ~ '^(0|[1-9][0-9]*)(,(0|[1-9][0-9]*))*$' AND cardinality(string_to_array(detail_ids, ',')) <= 250))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_checkpoint (
|
||||
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
from_link_id bigint,
|
||||
to_link_id bigint,
|
||||
link_count bigint,
|
||||
head_hash text,
|
||||
merkle_root text,
|
||||
algorithm text,
|
||||
key_id text,
|
||||
public_key text,
|
||||
signature text,
|
||||
created_at timestamptz
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_export (
|
||||
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
from_link_id bigint,
|
||||
to_link_id bigint,
|
||||
from_checkpoint_id bigint,
|
||||
to_checkpoint_id bigint,
|
||||
bucket_alias text,
|
||||
object_prefix text,
|
||||
object_versions text,
|
||||
manifest_key text,
|
||||
manifest_sha256 text,
|
||||
prev_manifest_sha256 text,
|
||||
object_count bigint,
|
||||
byte_count bigint,
|
||||
lock_mode text,
|
||||
retain_until timestamptz,
|
||||
sealed_at timestamptz
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_verification (
|
||||
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
kind text,
|
||||
"trigger" text,
|
||||
started_at timestamptz,
|
||||
finished_at timestamptz,
|
||||
from_link_id bigint,
|
||||
to_link_id bigint,
|
||||
links_checked bigint,
|
||||
outcome text,
|
||||
break_count bigint,
|
||||
first_break_link_id bigint,
|
||||
first_break_audit_id bigint,
|
||||
first_break_reason text,
|
||||
export_id bigint,
|
||||
run_by text
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_purge (
|
||||
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
requested_at timestamptz,
|
||||
requested_by text,
|
||||
cutoff timestamptz,
|
||||
retention_interval text,
|
||||
outcome text,
|
||||
reason text,
|
||||
audits_deleted bigint,
|
||||
details_deleted bigint,
|
||||
links_deleted bigint,
|
||||
anchor_link_id bigint,
|
||||
anchor_hash text
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS worm_audit_chain_audit ON public.worm_audit_chain(audit_id) WHERE kind = 'AUDIT';
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_chain_audit_all ON public.worm_audit_chain(audit_id);
|
||||
|
||||
-- changeset worm-audit:2 splitStatements:false
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_reject() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp SET timezone = 'UTC' AS $$
|
||||
DECLARE
|
||||
old_time timestamptz;
|
||||
function_owner name;
|
||||
stack text;
|
||||
row_identity text := 'all';
|
||||
BEGIN
|
||||
IF TG_OP = 'DELETE' AND current_setting('worm_audit.purging', true) = 'on' THEN
|
||||
SELECT pg_get_userbyid(proowner) INTO function_owner FROM pg_proc
|
||||
WHERE oid = 'public.worm_audit_purge(timestamptz)'::regprocedure;
|
||||
GET DIAGNOSTICS stack = PG_CONTEXT;
|
||||
IF current_user = function_owner AND position('function worm_audit_purge(timestamp with time zone)' in stack) > 0 THEN
|
||||
IF TG_RELID = '"audit"'::regclass THEN
|
||||
old_time := OLD.timestamp;
|
||||
ELSIF TG_RELID = '"audit_detail"'::regclass THEN
|
||||
SELECT timestamp INTO old_time FROM "audit" WHERE id = OLD.audit_id;
|
||||
ELSIF TG_RELID = 'public.worm_audit_chain'::regclass THEN
|
||||
SELECT timestamp INTO old_time FROM "audit" WHERE id = OLD.audit_id;
|
||||
END IF;
|
||||
IF old_time < now() - interval '6 years' THEN
|
||||
RETURN OLD;
|
||||
END IF;
|
||||
END IF;
|
||||
END IF;
|
||||
IF TG_LEVEL = 'ROW' THEN
|
||||
row_identity := OLD.id::text;
|
||||
END IF;
|
||||
RAISE EXCEPTION USING ERRCODE = 'P0001',
|
||||
MESSAGE = format('worm_audit: audit rows are write-once; %s of "%s" is not allowed (id=%s). Expired rows are removed only by worm_audit_purge().',
|
||||
TG_OP, TG_TABLE_NAME, row_identity),
|
||||
HINT = 'see qbit-worm-audit docs/DESIGN.md section 4';
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:3 splitStatements:false
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_lookup_reject() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $$
|
||||
BEGIN
|
||||
IF TG_OP = 'UPDATE' AND NEW.id IS NOT DISTINCT FROM OLD.id AND NEW.name IS NOT DISTINCT FROM OLD.name THEN
|
||||
RETURN NEW;
|
||||
END IF;
|
||||
RAISE EXCEPTION USING ERRCODE = 'P0001',
|
||||
MESSAGE = format('worm_audit: audit rows are write-once; %s of "%s" is not allowed (id=%s). Expired rows are removed only by worm_audit_purge().',
|
||||
TG_OP, TG_TABLE_NAME, OLD.id),
|
||||
HINT = 'see qbit-worm-audit docs/DESIGN.md section 4';
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:4 splitStatements:false
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'worm_audit_purger') THEN
|
||||
CREATE ROLE worm_audit_purger NOLOGIN;
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:5 splitStatements:false
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_purge(p_cutoff timestamptz) RETURNS void
|
||||
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, public, pg_temp SET timezone = 'UTC' AS $$
|
||||
DECLARE
|
||||
boundary bigint;
|
||||
smaller bigint;
|
||||
anchor text;
|
||||
audits_count bigint;
|
||||
details_count bigint;
|
||||
links_count bigint;
|
||||
BEGIN
|
||||
IF p_cutoff IS NULL OR p_cutoff > now() - interval '6 years' THEN
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome, reason)
|
||||
VALUES(now(), session_user, p_cutoff, '6 years', 'REFUSED', 'Cutoff is within retention or null');
|
||||
RETURN;
|
||||
END IF;
|
||||
-- Same lock as the Java chainer; table locks also exclude concurrent audit inserts.
|
||||
PERFORM pg_advisory_xact_lock(86753090421);
|
||||
LOCK TABLE "audit", "audit_detail", public.worm_audit_chain IN SHARE ROW EXCLUSIVE MODE;
|
||||
SELECT coalesce(min(c.id) - 1, (SELECT max(id) FROM public.worm_audit_chain)) INTO boundary
|
||||
FROM public.worm_audit_chain c LEFT JOIN "audit" a ON a.id = c.audit_id
|
||||
WHERE a.id IS NULL OR a.timestamp IS NULL OR a.timestamp >= p_cutoff;
|
||||
-- Shrink to a prefix closed over every audit's links AND all its stored details.
|
||||
LOOP
|
||||
SELECT min(c.id) - 1 INTO smaller FROM public.worm_audit_chain c
|
||||
WHERE c.id <= boundary AND (
|
||||
EXISTS (SELECT FROM public.worm_audit_chain later WHERE later.audit_id = c.audit_id AND later.id > boundary)
|
||||
OR EXISTS (SELECT FROM "audit_detail" d WHERE d.audit_id = c.audit_id AND NOT EXISTS (
|
||||
SELECT FROM public.worm_audit_chain covered WHERE covered.audit_id = c.audit_id
|
||||
AND d.id::text = ANY(string_to_array(covered.detail_ids, ',')))));
|
||||
EXIT WHEN smaller IS NULL;
|
||||
boundary := smaller;
|
||||
END LOOP;
|
||||
SELECT link_hash INTO anchor FROM public.worm_audit_chain WHERE id = boundary;
|
||||
IF anchor IS NULL THEN
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome, reason)
|
||||
VALUES(now(), session_user, p_cutoff, '6 years', 'REFUSED', 'No complete expired chain prefix');
|
||||
RETURN;
|
||||
END IF;
|
||||
PERFORM set_config('worm_audit.purging', 'on', true);
|
||||
DELETE FROM "audit_detail" d WHERE EXISTS
|
||||
(SELECT FROM public.worm_audit_chain c WHERE c.audit_id = d.audit_id AND c.id <= boundary);
|
||||
GET DIAGNOSTICS details_count = ROW_COUNT;
|
||||
-- Keep ids locally because links must be deleted before their audit headers.
|
||||
WITH removed AS (DELETE FROM public.worm_audit_chain WHERE id <= boundary RETURNING audit_id),
|
||||
removed_audits AS (DELETE FROM "audit" WHERE id IN (SELECT audit_id FROM removed) RETURNING id)
|
||||
SELECT (SELECT count(*) FROM removed), (SELECT count(*) FROM removed_audits) INTO links_count, audits_count;
|
||||
PERFORM set_config('worm_audit.purging', 'off', true);
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome,
|
||||
audits_deleted, details_deleted, links_deleted, anchor_link_id, anchor_hash)
|
||||
VALUES(now(), session_user, p_cutoff, '6 years', 'PURGED', audits_count, details_count, links_count, boundary, anchor);
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:6
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON "audit";
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON "audit"
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON "audit";
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON "audit"
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON "audit" FROM "app";
|
||||
GRANT SELECT, INSERT ON "audit" TO "app";
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON "audit_detail";
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON "audit_detail"
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON "audit_detail";
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON "audit_detail"
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON "audit_detail" FROM "app";
|
||||
GRANT SELECT, INSERT ON "audit_detail" TO "app";
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_chain;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_chain
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_chain;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_chain
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_chain FROM "app";
|
||||
GRANT SELECT, INSERT ON public.worm_audit_chain TO "app";
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_checkpoint;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_checkpoint
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_checkpoint;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_checkpoint
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_checkpoint FROM "app";
|
||||
GRANT SELECT, INSERT ON public.worm_audit_checkpoint TO "app";
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_export;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_export
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_export;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_export
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_export FROM "app";
|
||||
GRANT SELECT, INSERT ON public.worm_audit_export TO "app";
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_verification;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_verification
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_verification;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_verification
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_verification FROM "app";
|
||||
GRANT SELECT, INSERT ON public.worm_audit_verification TO "app";
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_purge;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_purge
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_purge;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_purge
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_purge FROM "app";
|
||||
GRANT SELECT, INSERT ON public.worm_audit_purge TO "app";
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON "audit_table";
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON "audit_table"
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_lookup_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON "audit_table";
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON "audit_table"
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON "audit_table" FROM "app";
|
||||
GRANT SELECT, INSERT ON "audit_table" TO "app";
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON "audit_user";
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON "audit_user"
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_lookup_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON "audit_user";
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON "audit_user"
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON "audit_user" FROM "app";
|
||||
GRANT SELECT, INSERT ON "audit_user" TO "app";
|
||||
|
||||
-- Lookups may refresh display metadata, but never identities or names.
|
||||
GRANT UPDATE (label, create_date, modify_date) ON "audit_table" TO "app";
|
||||
GRANT UPDATE (create_date, modify_date) ON "audit_user" TO "app";
|
||||
|
||||
REVOKE ALL ON FUNCTION public.worm_audit_purge(timestamptz) FROM PUBLIC;
|
||||
REVOKE ALL ON FUNCTION public.worm_audit_purge(timestamptz) FROM "app";
|
||||
GRANT EXECUTE ON FUNCTION public.worm_audit_purge(timestamptz) TO worm_audit_purger;
|
||||
|
||||
-- changeset worm-audit:7 splitStatements:false
|
||||
DO $$
|
||||
DECLARE
|
||||
table_name text;
|
||||
sequence_name text;
|
||||
BEGIN
|
||||
FOREACH table_name IN ARRAY ARRAY['"audit"', '"audit_detail"', '"audit_table"', '"audit_user"',
|
||||
'public.worm_audit_checkpoint', 'public.worm_audit_export', 'public.worm_audit_verification', 'public.worm_audit_purge'] LOOP
|
||||
sequence_name := pg_get_serial_sequence(table_name, 'id');
|
||||
IF sequence_name IS NOT NULL THEN
|
||||
EXECUTE format('GRANT USAGE ON SEQUENCE %s TO "app"', sequence_name);
|
||||
END IF;
|
||||
END LOOP;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:8 splitStatements:false
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_detail_coverage() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $$
|
||||
BEGIN
|
||||
-- Serialize even direct SQL appends; the GIN index makes overlap checks indexed.
|
||||
PERFORM pg_advisory_xact_lock(86753090421);
|
||||
IF NEW.detail_ids <> '' AND (
|
||||
EXISTS (SELECT FROM unnest(string_to_array(NEW.detail_ids, ',')) AS d(id) GROUP BY d.id HAVING count(*) > 1)
|
||||
OR EXISTS (SELECT FROM public.worm_audit_chain c
|
||||
WHERE string_to_array(c.detail_ids, ',') && string_to_array(NEW.detail_ids, ','))) THEN
|
||||
RAISE EXCEPTION USING ERRCODE = '23505', MESSAGE = 'worm_audit: audit detail is already covered by the chain';
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:9
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_chain_detail_coverage ON public.worm_audit_chain USING gin (string_to_array(detail_ids, ','));
|
||||
DROP TRIGGER IF EXISTS worm_audit_detail_coverage_guard ON public.worm_audit_chain;
|
||||
CREATE TRIGGER worm_audit_detail_coverage_guard BEFORE INSERT ON public.worm_audit_chain
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_detail_coverage();
|
||||
|
||||
-- changeset worm-audit:10
|
||||
-- Editable singleton: deliberately excluded from append-only triggers.
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_settings (
|
||||
id integer PRIMARY KEY CHECK (id = 1),
|
||||
audit_retention_years integer NOT NULL DEFAULT 6 CHECK (audit_retention_years BETWEEN 1 AND 100),
|
||||
export_lock_mode text NOT NULL DEFAULT 'GOVERNANCE' CHECK (export_lock_mode IN ('GOVERNANCE', 'COMPLIANCE')),
|
||||
export_retention_days integer NOT NULL DEFAULT 30 CHECK (export_retention_days BETWEEN 1 AND 36500),
|
||||
checkpoint_every_links integer NOT NULL DEFAULT 10000 CHECK (checkpoint_every_links > 0),
|
||||
checkpoint_every_minutes integer NOT NULL DEFAULT 60 CHECK (checkpoint_every_minutes > 0),
|
||||
chain_interval_seconds integer NOT NULL DEFAULT 30 CHECK (chain_interval_seconds > 0),
|
||||
settle_horizon_seconds integer NOT NULL DEFAULT 60 CHECK (settle_horizon_seconds >= 0),
|
||||
late_detail_lookback_hours integer NOT NULL DEFAULT 24 CHECK (late_detail_lookback_hours > 0),
|
||||
export_cron text NOT NULL DEFAULT '0 0 2 * * ?',
|
||||
verify_cron text NOT NULL DEFAULT '0 0 3 * * ?',
|
||||
notes text,
|
||||
modify_date timestamptz NOT NULL DEFAULT now(),
|
||||
modified_by text NOT NULL DEFAULT current_user
|
||||
);
|
||||
REVOKE DELETE, TRUNCATE ON public.worm_audit_settings FROM "app";
|
||||
GRANT SELECT, INSERT, UPDATE ON public.worm_audit_settings TO "app";
|
||||
INSERT INTO public.worm_audit_settings
|
||||
(id, audit_retention_years, export_lock_mode, export_retention_days, checkpoint_every_links,
|
||||
checkpoint_every_minutes, chain_interval_seconds, settle_horizon_seconds, late_detail_lookback_hours,
|
||||
export_cron, verify_cron)
|
||||
SELECT 1, 6, 'GOVERNANCE', 30, 10000, 60, 30, 60, 24, '0 0 2 * * ?', '0 0 3 * * ?'
|
||||
WHERE NOT EXISTS (SELECT 1 FROM public.worm_audit_settings WHERE id = 1);
|
||||
@@ -0,0 +1,80 @@
|
||||
# WORM Audit screenshots
|
||||
|
||||
## RC.2 settings captures (2026-10-14)
|
||||
Home (GOVERNANCE, 30-day exports) and the WORM Audit Settings record at 390 x 844 and 1440 x 1000,
|
||||
light and dark, captured with Playwright against `WormAuditDemoServer` on port 8099:
|
||||
`worm-audit-home-{390,1440}-{light,dark}.jpg` (replaced) and
|
||||
`worm-audit-settings-{390,1440}-{light,dark}.jpg` (new). Overflow check on all eight: no page
|
||||
horizontal overflow, no inner horizontal scrollers, no double scrollbars.
|
||||
|
||||
## Earlier captures
|
||||
|
||||
Refreshed after record-screen polish **2026-10-13 (America/Chicago)** from the rebuilt test-scope demo using
|
||||
packaged Next UI 1.0.0-RC.14 and connected Edge through the browser screenshot
|
||||
and viewport tools. All files are JPEG viewport captures, below 300 KB each.
|
||||
|
||||
```powershell
|
||||
./mvnw.cmd -o test-compile exec:java "-Dexec.classpathScope=test" "-Dexec.mainClass=com.kingsrook.qbits.wormaudit.demo.WormAuditDemoServer" "-Dexec.args=8105"
|
||||
```
|
||||
|
||||
Open <http://localhost:8105/app/worm_audit/>. Port 8105 isolates this rebuilt instance from earlier demos. The seed creates genuine verification
|
||||
results and exports over synthetic data. Records captured: FAIL verification #8,
|
||||
PASS verification #15, sealed export #15, and chain link #329.
|
||||
|
||||
Required matrix: home at **390, 1440, 1920 px**, and all four records at **390,
|
||||
1440 px**, each in **light and dark**. Heights are 844, 1000 and 1080 px respectively.
|
||||
The existing supplementary 390 px light purge-history capture was refreshed too.
|
||||
Theme changes used Preferences and were restored to light; the temporary viewport override was reset afterward.
|
||||
|
||||
Every image was visually inspected. At each capture, the live DOM was checked after responsive layout settled for
|
||||
page-level horizontal overflow (`document.documentElement.scrollWidth > innerWidth`)
|
||||
and all rendered inner horizontal scrollers (`scrollWidth > clientWidth + 1` with
|
||||
computed `overflow-x` of `auto` or `scroll`). The scan included content below the
|
||||
viewport, including loaded child lists. Mobile verification accordions were also explicitly expanded and checked after their child rows loaded. There were **zero** page overflow failures
|
||||
and **zero** inner horizontal scrollers. Each home/verification/export view had
|
||||
one vertical scroll container (main); chain records had none. No double scrollbars
|
||||
were found. Long hashes, object versions and failure diagnostics wrap on mobile.
|
||||
No CSS or DOM overrides were injected for the captures.
|
||||
|
||||
All five tables have explicit field labels and meaningful record-label formats.
|
||||
Verification uses full-width Result, Scope, Run, and Chain links sections. Every
|
||||
child-list widget and section declares 12 grid columns. Kind and outcome use
|
||||
readable possible values; child queries translate those values too. Hashes stay
|
||||
complete because RC.5 has no separate list-only field display format. Numeric
|
||||
chain-reference labels keep the export title concise.
|
||||
|
||||
Core RC.5 shares the table label with its sidebar tree node. Existing plural
|
||||
navigation labels are retained, with singular record titles and record-section
|
||||
labels. Separate singular table labels and plural sidebar labels require core
|
||||
support; no application-wide metadata customizer was replaced to work around it.
|
||||
|
||||
`./mvnw.cmd -o verify` passed: 79 tests, no failures or errors, and 9
|
||||
Docker-dependent skips. Checkstyle and JaCoCo coverage gates passed. Metadata
|
||||
tests cover every field label, record-label formats and fields, verification
|
||||
section coverage, and 12-column sections and child widgets.
|
||||
|
||||
| Screenshot | Viewport | Size | Page horizontal overflow | Inner horizontal scrollers | Double scrollbars |
|
||||
|---|---|---|---|---|---|
|
||||
| [worm-audit-chain-link-1440-dark](worm-audit-chain-link-1440-dark.jpg) | 1440 x 1000 | 41.1 KB | None | None | None |
|
||||
| [worm-audit-chain-link-1440-light](worm-audit-chain-link-1440-light.jpg) | 1440 x 1000 | 41.4 KB | None | None | None |
|
||||
| [worm-audit-chain-link-390-dark](worm-audit-chain-link-390-dark.jpg) | 390 x 844 | 21.8 KB | None | None | None |
|
||||
| [worm-audit-chain-link-390-light](worm-audit-chain-link-390-light.jpg) | 390 x 844 | 21.8 KB | None | None | None |
|
||||
| [worm-audit-home-1440-dark](worm-audit-home-1440-dark.jpg) | 1440 x 1000 | 73.6 KB | None | None | None |
|
||||
| [worm-audit-home-1440-light](worm-audit-home-1440-light.jpg) | 1440 x 1000 | 72.4 KB | None | None | None |
|
||||
| [worm-audit-home-1920-dark](worm-audit-home-1920-dark.jpg) | 1920 x 1080 | 84.7 KB | None | None | None |
|
||||
| [worm-audit-home-1920-light](worm-audit-home-1920-light.jpg) | 1920 x 1080 | 83.6 KB | None | None | None |
|
||||
| [worm-audit-home-390-dark](worm-audit-home-390-dark.jpg) | 390 x 844 | 30.0 KB | None | None | None |
|
||||
| [worm-audit-home-390-light](worm-audit-home-390-light.jpg) | 390 x 844 | 28.8 KB | None | None | None |
|
||||
| [worm-audit-home-390-purges-light](worm-audit-home-390-purges-light.jpg) | 390 x 844 | 29.5 KB | None | None | None |
|
||||
| [worm-audit-sealed-export-1440-dark](worm-audit-sealed-export-1440-dark.jpg) | 1440 x 1000 | 117.6 KB | None | None | None |
|
||||
| [worm-audit-sealed-export-1440-light](worm-audit-sealed-export-1440-light.jpg) | 1440 x 1000 | 118.4 KB | None | None | None |
|
||||
| [worm-audit-sealed-export-390-dark](worm-audit-sealed-export-390-dark.jpg) | 390 x 844 | 38.4 KB | None | None | None |
|
||||
| [worm-audit-sealed-export-390-light](worm-audit-sealed-export-390-light.jpg) | 390 x 844 | 37.3 KB | None | None | None |
|
||||
| [worm-audit-verification-fail-1440-dark](worm-audit-verification-fail-1440-dark.jpg) | 1440 x 1000 | 53.6 KB | None | None | None |
|
||||
| [worm-audit-verification-fail-1440-light](worm-audit-verification-fail-1440-light.jpg) | 1440 x 1000 | 54.1 KB | None | None | None |
|
||||
| [worm-audit-verification-fail-390-dark](worm-audit-verification-fail-390-dark.jpg) | 390 x 844 | 28.1 KB | None | None | None |
|
||||
| [worm-audit-verification-fail-390-light](worm-audit-verification-fail-390-light.jpg) | 390 x 844 | 27.8 KB | None | None | None |
|
||||
| [worm-audit-verification-pass-1440-dark](worm-audit-verification-pass-1440-dark.jpg) | 1440 x 1000 | 49.0 KB | None | None | None |
|
||||
| [worm-audit-verification-pass-1440-light](worm-audit-verification-pass-1440-light.jpg) | 1440 x 1000 | 49.7 KB | None | None | None |
|
||||
| [worm-audit-verification-pass-390-dark](worm-audit-verification-pass-390-dark.jpg) | 390 x 844 | 23.5 KB | None | None | None |
|
||||
| [worm-audit-verification-pass-390-light](worm-audit-verification-pass-390-light.jpg) | 390 x 844 | 23.4 KB | None | None | None |
|
||||
|
After Width: | Height: | Size: 41 KiB |
|
After Width: | Height: | Size: 41 KiB |
|
After Width: | Height: | Size: 22 KiB |
|
After Width: | Height: | Size: 22 KiB |
|
After Width: | Height: | Size: 84 KiB |
|
After Width: | Height: | Size: 82 KiB |
|
After Width: | Height: | Size: 85 KiB |
|
After Width: | Height: | Size: 84 KiB |
|
After Width: | Height: | Size: 32 KiB |
|
After Width: | Height: | Size: 31 KiB |
|
After Width: | Height: | Size: 30 KiB |
|
After Width: | Height: | Size: 118 KiB |
|
After Width: | Height: | Size: 118 KiB |
|
After Width: | Height: | Size: 38 KiB |
|
After Width: | Height: | Size: 37 KiB |
|
After Width: | Height: | Size: 74 KiB |
|
After Width: | Height: | Size: 75 KiB |
|
After Width: | Height: | Size: 33 KiB |
|
After Width: | Height: | Size: 33 KiB |
|
After Width: | Height: | Size: 54 KiB |
|
After Width: | Height: | Size: 54 KiB |
|
After Width: | Height: | Size: 28 KiB |
|
After Width: | Height: | Size: 28 KiB |
|
After Width: | Height: | Size: 49 KiB |
|
After Width: | Height: | Size: 50 KiB |
|
After Width: | Height: | Size: 24 KiB |
|
After Width: | Height: | Size: 23 KiB |
@@ -0,0 +1,295 @@
|
||||
#!/bin/sh
|
||||
# ----------------------------------------------------------------------------
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Apache Maven Wrapper startup batch script, version 3.3.4
|
||||
#
|
||||
# Optional ENV vars
|
||||
# -----------------
|
||||
# JAVA_HOME - location of a JDK home dir, required when download maven via java source
|
||||
# MVNW_REPOURL - repo url base for downloading maven distribution
|
||||
# MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven
|
||||
# MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
set -euf
|
||||
[ "${MVNW_VERBOSE-}" != debug ] || set -x
|
||||
|
||||
# OS specific support.
|
||||
native_path() { printf %s\\n "$1"; }
|
||||
case "$(uname)" in
|
||||
CYGWIN* | MINGW*)
|
||||
[ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")"
|
||||
native_path() { cygpath --path --windows "$1"; }
|
||||
;;
|
||||
esac
|
||||
|
||||
# set JAVACMD and JAVACCMD
|
||||
set_java_home() {
|
||||
# For Cygwin and MinGW, ensure paths are in Unix format before anything is touched
|
||||
if [ -n "${JAVA_HOME-}" ]; then
|
||||
if [ -x "$JAVA_HOME/jre/sh/java" ]; then
|
||||
# IBM's JDK on AIX uses strange locations for the executables
|
||||
JAVACMD="$JAVA_HOME/jre/sh/java"
|
||||
JAVACCMD="$JAVA_HOME/jre/sh/javac"
|
||||
else
|
||||
JAVACMD="$JAVA_HOME/bin/java"
|
||||
JAVACCMD="$JAVA_HOME/bin/javac"
|
||||
|
||||
if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then
|
||||
echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2
|
||||
echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
else
|
||||
JAVACMD="$(
|
||||
'set' +e
|
||||
'unset' -f command 2>/dev/null
|
||||
'command' -v java
|
||||
)" || :
|
||||
JAVACCMD="$(
|
||||
'set' +e
|
||||
'unset' -f command 2>/dev/null
|
||||
'command' -v javac
|
||||
)" || :
|
||||
|
||||
if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then
|
||||
echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# hash string like Java String::hashCode
|
||||
hash_string() {
|
||||
str="${1:-}" h=0
|
||||
while [ -n "$str" ]; do
|
||||
char="${str%"${str#?}"}"
|
||||
h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296))
|
||||
str="${str#?}"
|
||||
done
|
||||
printf %x\\n $h
|
||||
}
|
||||
|
||||
verbose() { :; }
|
||||
[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; }
|
||||
|
||||
die() {
|
||||
printf %s\\n "$1" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
trim() {
|
||||
# MWRAPPER-139:
|
||||
# Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds.
|
||||
# Needed for removing poorly interpreted newline sequences when running in more
|
||||
# exotic environments such as mingw bash on Windows.
|
||||
printf "%s" "${1}" | tr -d '[:space:]'
|
||||
}
|
||||
|
||||
scriptDir="$(dirname "$0")"
|
||||
scriptName="$(basename "$0")"
|
||||
|
||||
# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties
|
||||
while IFS="=" read -r key value; do
|
||||
case "${key-}" in
|
||||
distributionUrl) distributionUrl=$(trim "${value-}") ;;
|
||||
distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;;
|
||||
esac
|
||||
done <"$scriptDir/.mvn/wrapper/maven-wrapper.properties"
|
||||
[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties"
|
||||
|
||||
case "${distributionUrl##*/}" in
|
||||
maven-mvnd-*bin.*)
|
||||
MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/
|
||||
case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in
|
||||
*AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;;
|
||||
:Darwin*x86_64) distributionPlatform=darwin-amd64 ;;
|
||||
:Darwin*arm64) distributionPlatform=darwin-aarch64 ;;
|
||||
:Linux*x86_64*) distributionPlatform=linux-amd64 ;;
|
||||
*)
|
||||
echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2
|
||||
distributionPlatform=linux-amd64
|
||||
;;
|
||||
esac
|
||||
distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip"
|
||||
;;
|
||||
maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;;
|
||||
*) MVN_CMD="mvn${scriptName#mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;;
|
||||
esac
|
||||
|
||||
# apply MVNW_REPOURL and calculate MAVEN_HOME
|
||||
# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-<version>,maven-mvnd-<version>-<platform>}/<hash>
|
||||
[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}"
|
||||
distributionUrlName="${distributionUrl##*/}"
|
||||
distributionUrlNameMain="${distributionUrlName%.*}"
|
||||
distributionUrlNameMain="${distributionUrlNameMain%-bin}"
|
||||
MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}"
|
||||
MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")"
|
||||
|
||||
exec_maven() {
|
||||
unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || :
|
||||
exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD"
|
||||
}
|
||||
|
||||
if [ -d "$MAVEN_HOME" ]; then
|
||||
verbose "found existing MAVEN_HOME at $MAVEN_HOME"
|
||||
exec_maven "$@"
|
||||
fi
|
||||
|
||||
case "${distributionUrl-}" in
|
||||
*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;;
|
||||
*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;;
|
||||
esac
|
||||
|
||||
# prepare tmp dir
|
||||
if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then
|
||||
clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; }
|
||||
trap clean HUP INT TERM EXIT
|
||||
else
|
||||
die "cannot create temp dir"
|
||||
fi
|
||||
|
||||
mkdir -p -- "${MAVEN_HOME%/*}"
|
||||
|
||||
# Download and Install Apache Maven
|
||||
verbose "Couldn't find MAVEN_HOME, downloading and installing it ..."
|
||||
verbose "Downloading from: $distributionUrl"
|
||||
verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName"
|
||||
|
||||
# select .zip or .tar.gz
|
||||
if ! command -v unzip >/dev/null; then
|
||||
distributionUrl="${distributionUrl%.zip}.tar.gz"
|
||||
distributionUrlName="${distributionUrl##*/}"
|
||||
fi
|
||||
|
||||
# verbose opt
|
||||
__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR=''
|
||||
[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v
|
||||
|
||||
# normalize http auth
|
||||
case "${MVNW_PASSWORD:+has-password}" in
|
||||
'') MVNW_USERNAME='' MVNW_PASSWORD='' ;;
|
||||
has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;;
|
||||
esac
|
||||
|
||||
if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then
|
||||
verbose "Found wget ... using wget"
|
||||
wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl"
|
||||
elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then
|
||||
verbose "Found curl ... using curl"
|
||||
curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl"
|
||||
elif set_java_home; then
|
||||
verbose "Falling back to use Java to download"
|
||||
javaSource="$TMP_DOWNLOAD_DIR/Downloader.java"
|
||||
targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName"
|
||||
cat >"$javaSource" <<-END
|
||||
public class Downloader extends java.net.Authenticator
|
||||
{
|
||||
protected java.net.PasswordAuthentication getPasswordAuthentication()
|
||||
{
|
||||
return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() );
|
||||
}
|
||||
public static void main( String[] args ) throws Exception
|
||||
{
|
||||
setDefault( new Downloader() );
|
||||
java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() );
|
||||
}
|
||||
}
|
||||
END
|
||||
# For Cygwin/MinGW, switch paths to Windows format before running javac and java
|
||||
verbose " - Compiling Downloader.java ..."
|
||||
"$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java"
|
||||
verbose " - Running Downloader.java ..."
|
||||
"$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")"
|
||||
fi
|
||||
|
||||
# If specified, validate the SHA-256 sum of the Maven distribution zip file
|
||||
if [ -n "${distributionSha256Sum-}" ]; then
|
||||
distributionSha256Result=false
|
||||
if [ "$MVN_CMD" = mvnd.sh ]; then
|
||||
echo "Checksum validation is not supported for maven-mvnd." >&2
|
||||
echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2
|
||||
exit 1
|
||||
elif command -v sha256sum >/dev/null; then
|
||||
if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c - >/dev/null 2>&1; then
|
||||
distributionSha256Result=true
|
||||
fi
|
||||
elif command -v shasum >/dev/null; then
|
||||
if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then
|
||||
distributionSha256Result=true
|
||||
fi
|
||||
else
|
||||
echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2
|
||||
echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ $distributionSha256Result = false ]; then
|
||||
echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2
|
||||
echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# unzip and move
|
||||
if command -v unzip >/dev/null; then
|
||||
unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip"
|
||||
else
|
||||
tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar"
|
||||
fi
|
||||
|
||||
# Find the actual extracted directory name (handles snapshots where filename != directory name)
|
||||
actualDistributionDir=""
|
||||
|
||||
# First try the expected directory name (for regular distributions)
|
||||
if [ -d "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" ]; then
|
||||
if [ -f "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/bin/$MVN_CMD" ]; then
|
||||
actualDistributionDir="$distributionUrlNameMain"
|
||||
fi
|
||||
fi
|
||||
|
||||
# If not found, search for any directory with the Maven executable (for snapshots)
|
||||
if [ -z "$actualDistributionDir" ]; then
|
||||
# enable globbing to iterate over items
|
||||
set +f
|
||||
for dir in "$TMP_DOWNLOAD_DIR"/*; do
|
||||
if [ -d "$dir" ]; then
|
||||
if [ -f "$dir/bin/$MVN_CMD" ]; then
|
||||
actualDistributionDir="$(basename "$dir")"
|
||||
break
|
||||
fi
|
||||
fi
|
||||
done
|
||||
set -f
|
||||
fi
|
||||
|
||||
if [ -z "$actualDistributionDir" ]; then
|
||||
verbose "Contents of $TMP_DOWNLOAD_DIR:"
|
||||
verbose "$(ls -la "$TMP_DOWNLOAD_DIR")"
|
||||
die "Could not find Maven distribution directory in extracted archive"
|
||||
fi
|
||||
|
||||
verbose "Found extracted Maven distribution directory: $actualDistributionDir"
|
||||
printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$actualDistributionDir/mvnw.url"
|
||||
mv -- "$TMP_DOWNLOAD_DIR/$actualDistributionDir" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME"
|
||||
|
||||
clean || :
|
||||
exec_maven "$@"
|
||||
@@ -0,0 +1,189 @@
|
||||
<# : batch portion
|
||||
@REM ----------------------------------------------------------------------------
|
||||
@REM Licensed to the Apache Software Foundation (ASF) under one
|
||||
@REM or more contributor license agreements. See the NOTICE file
|
||||
@REM distributed with this work for additional information
|
||||
@REM regarding copyright ownership. The ASF licenses this file
|
||||
@REM to you under the Apache License, Version 2.0 (the
|
||||
@REM "License"); you may not use this file except in compliance
|
||||
@REM with the License. You may obtain a copy of the License at
|
||||
@REM
|
||||
@REM http://www.apache.org/licenses/LICENSE-2.0
|
||||
@REM
|
||||
@REM Unless required by applicable law or agreed to in writing,
|
||||
@REM software distributed under the License is distributed on an
|
||||
@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
@REM KIND, either express or implied. See the License for the
|
||||
@REM specific language governing permissions and limitations
|
||||
@REM under the License.
|
||||
@REM ----------------------------------------------------------------------------
|
||||
|
||||
@REM ----------------------------------------------------------------------------
|
||||
@REM Apache Maven Wrapper startup batch script, version 3.3.4
|
||||
@REM
|
||||
@REM Optional ENV vars
|
||||
@REM MVNW_REPOURL - repo url base for downloading maven distribution
|
||||
@REM MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven
|
||||
@REM MVNW_VERBOSE - true: enable verbose log; others: silence the output
|
||||
@REM ----------------------------------------------------------------------------
|
||||
|
||||
@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0)
|
||||
@SET __MVNW_CMD__=
|
||||
@SET __MVNW_ERROR__=
|
||||
@SET __MVNW_PSMODULEP_SAVE=%PSModulePath%
|
||||
@SET PSModulePath=
|
||||
@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @(
|
||||
IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B)
|
||||
)
|
||||
@SET PSModulePath=%__MVNW_PSMODULEP_SAVE%
|
||||
@SET __MVNW_PSMODULEP_SAVE=
|
||||
@SET __MVNW_ARG0_NAME__=
|
||||
@SET MVNW_USERNAME=
|
||||
@SET MVNW_PASSWORD=
|
||||
@IF NOT "%__MVNW_CMD__%"=="" ("%__MVNW_CMD__%" %*)
|
||||
@echo Cannot start maven from wrapper >&2 && exit /b 1
|
||||
@GOTO :EOF
|
||||
: end batch / begin powershell #>
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
if ($env:MVNW_VERBOSE -eq "true") {
|
||||
$VerbosePreference = "Continue"
|
||||
}
|
||||
|
||||
# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties
|
||||
$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl
|
||||
if (!$distributionUrl) {
|
||||
Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties"
|
||||
}
|
||||
|
||||
switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) {
|
||||
"maven-mvnd-*" {
|
||||
$USE_MVND = $true
|
||||
$distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip"
|
||||
$MVN_CMD = "mvnd.cmd"
|
||||
break
|
||||
}
|
||||
default {
|
||||
$USE_MVND = $false
|
||||
$MVN_CMD = $script -replace '^mvnw','mvn'
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
# apply MVNW_REPOURL and calculate MAVEN_HOME
|
||||
# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-<version>,maven-mvnd-<version>-<platform>}/<hash>
|
||||
if ($env:MVNW_REPOURL) {
|
||||
$MVNW_REPO_PATTERN = if ($USE_MVND -eq $False) { "/org/apache/maven/" } else { "/maven/mvnd/" }
|
||||
$distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace "^.*$MVNW_REPO_PATTERN",'')"
|
||||
}
|
||||
$distributionUrlName = $distributionUrl -replace '^.*/',''
|
||||
$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$',''
|
||||
|
||||
$MAVEN_M2_PATH = "$HOME/.m2"
|
||||
if ($env:MAVEN_USER_HOME) {
|
||||
$MAVEN_M2_PATH = "$env:MAVEN_USER_HOME"
|
||||
}
|
||||
|
||||
if (-not (Test-Path -Path $MAVEN_M2_PATH)) {
|
||||
New-Item -Path $MAVEN_M2_PATH -ItemType Directory | Out-Null
|
||||
}
|
||||
|
||||
$MAVEN_WRAPPER_DISTS = $null
|
||||
if ((Get-Item $MAVEN_M2_PATH).Target[0] -eq $null) {
|
||||
$MAVEN_WRAPPER_DISTS = "$MAVEN_M2_PATH/wrapper/dists"
|
||||
} else {
|
||||
$MAVEN_WRAPPER_DISTS = (Get-Item $MAVEN_M2_PATH).Target[0] + "/wrapper/dists"
|
||||
}
|
||||
|
||||
$MAVEN_HOME_PARENT = "$MAVEN_WRAPPER_DISTS/$distributionUrlNameMain"
|
||||
$MAVEN_HOME_NAME = ([System.Security.Cryptography.SHA256]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join ''
|
||||
$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME"
|
||||
|
||||
if (Test-Path -Path "$MAVEN_HOME" -PathType Container) {
|
||||
Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME"
|
||||
Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD"
|
||||
exit $?
|
||||
}
|
||||
|
||||
if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) {
|
||||
Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl"
|
||||
}
|
||||
|
||||
# prepare tmp dir
|
||||
$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile
|
||||
$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir"
|
||||
$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null
|
||||
trap {
|
||||
if ($TMP_DOWNLOAD_DIR.Exists) {
|
||||
try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null }
|
||||
catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" }
|
||||
}
|
||||
}
|
||||
|
||||
New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null
|
||||
|
||||
# Download and Install Apache Maven
|
||||
Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..."
|
||||
Write-Verbose "Downloading from: $distributionUrl"
|
||||
Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName"
|
||||
|
||||
$webclient = New-Object System.Net.WebClient
|
||||
if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) {
|
||||
$webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD)
|
||||
}
|
||||
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
|
||||
$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null
|
||||
|
||||
# If specified, validate the SHA-256 sum of the Maven distribution zip file
|
||||
$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum
|
||||
if ($distributionSha256Sum) {
|
||||
if ($USE_MVND) {
|
||||
Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties."
|
||||
}
|
||||
Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash
|
||||
if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) {
|
||||
Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property."
|
||||
}
|
||||
}
|
||||
|
||||
# unzip and move
|
||||
Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null
|
||||
|
||||
# Find the actual extracted directory name (handles snapshots where filename != directory name)
|
||||
$actualDistributionDir = ""
|
||||
|
||||
# First try the expected directory name (for regular distributions)
|
||||
$expectedPath = Join-Path "$TMP_DOWNLOAD_DIR" "$distributionUrlNameMain"
|
||||
$expectedMvnPath = Join-Path "$expectedPath" "bin/$MVN_CMD"
|
||||
if ((Test-Path -Path $expectedPath -PathType Container) -and (Test-Path -Path $expectedMvnPath -PathType Leaf)) {
|
||||
$actualDistributionDir = $distributionUrlNameMain
|
||||
}
|
||||
|
||||
# If not found, search for any directory with the Maven executable (for snapshots)
|
||||
if (!$actualDistributionDir) {
|
||||
Get-ChildItem -Path "$TMP_DOWNLOAD_DIR" -Directory | ForEach-Object {
|
||||
$testPath = Join-Path $_.FullName "bin/$MVN_CMD"
|
||||
if (Test-Path -Path $testPath -PathType Leaf) {
|
||||
$actualDistributionDir = $_.Name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!$actualDistributionDir) {
|
||||
Write-Error "Could not find Maven distribution directory in extracted archive"
|
||||
}
|
||||
|
||||
Write-Verbose "Found extracted Maven distribution directory: $actualDistributionDir"
|
||||
Rename-Item -Path "$TMP_DOWNLOAD_DIR/$actualDistributionDir" -NewName $MAVEN_HOME_NAME | Out-Null
|
||||
try {
|
||||
Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null
|
||||
} catch {
|
||||
if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) {
|
||||
Write-Error "fail to move MAVEN_HOME"
|
||||
}
|
||||
} finally {
|
||||
try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null }
|
||||
catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" }
|
||||
}
|
||||
|
||||
Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD"
|
||||
@@ -1,42 +1,142 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
|
||||
http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<!--
|
||||
~ QQQ - Low-code Application Framework for Engineers.
|
||||
~ Copyright (C) 2021-2026. Kingsrook, LLC
|
||||
~ 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
~ contact@kingsrook.com
|
||||
~ https://github.com/Kingsrook/
|
||||
~
|
||||
~ Licensed under the Apache License, Version 2.0 (the "License");
|
||||
~ you may not use this file except in compliance with the License.
|
||||
~ You may obtain a copy of the License at
|
||||
~
|
||||
~ https://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
|
||||
<groupId>com.kingsrook.qbits</groupId>
|
||||
<artifactId>qbit-example-extension</artifactId>
|
||||
<version>0.1.0-SNAPSHOT</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>QBit Example Extension</name>
|
||||
<description>Example Extension QBit - rename this for your QBit</description>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
|
||||
<properties>
|
||||
<revision>0.1.0-SNAPSHOT</revision>
|
||||
<maven.compiler.source>21</maven.compiler.source>
|
||||
<maven.compiler.target>21</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<qqq.version>0.35.0</qqq.version>
|
||||
<jacoco.version>0.8.15</jacoco.version>
|
||||
<h2.version>2.2.224</h2.version>
|
||||
<testcontainers.version>2.0.5</testcontainers.version>
|
||||
</properties>
|
||||
|
||||
<parent>
|
||||
<groupId>com.kingsrook</groupId>
|
||||
<artifactId>qbit-build-parent</artifactId>
|
||||
<version>2.0.0</version>
|
||||
</parent>
|
||||
|
||||
<groupId>com.kingsrook.qbits</groupId>
|
||||
<artifactId>qbit-worm-audit</artifactId>
|
||||
<version>${revision}</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>QBit WORM Audit</name>
|
||||
<description>QBit for write-once (WORM) audit in QQQ applications: database-enforced append-only audit tables, hash-chained tamper evidence with signed checkpoints, verification, and sealed exports to S3 Object Lock</description>
|
||||
<url>https://github.com/QRun-IO/qbit-worm-audit</url>
|
||||
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<licenses>
|
||||
<license>
|
||||
<name>Apache License, Version 2.0</name>
|
||||
<url>https://www.apache.org/licenses/LICENSE-2.0</url>
|
||||
<distribution>repo</distribution>
|
||||
</license>
|
||||
</licenses>
|
||||
|
||||
<developers>
|
||||
<developer>
|
||||
<id>kingsrook</id>
|
||||
<name>Kingsrook Team</name>
|
||||
<email>contact@kingsrook.com</email>
|
||||
<organization>Kingsrook, LLC</organization>
|
||||
<organizationUrl>https://kingsrook.com</organizationUrl>
|
||||
</developer>
|
||||
</developers>
|
||||
|
||||
<scm>
|
||||
<connection>scm:git:git://github.com/QRun-IO/qbit-worm-audit.git</connection>
|
||||
<developerConnection>scm:git:ssh://github.com:QRun-IO/qbit-worm-audit.git</developerConnection>
|
||||
<url>https://github.com/QRun-IO/qbit-worm-audit</url>
|
||||
<tag>HEAD</tag>
|
||||
</scm>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
<artifactId>qqq-backend-core</artifactId>
|
||||
<version>${qqq.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- The memory backend is included in qqq-backend-core. -->
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
<artifactId>qqq-backend-module-rdbms</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qbits</groupId>
|
||||
<artifactId>qbit-object-storage</artifactId>
|
||||
<version>0.1.0-RC.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
<artifactId>qqq-middleware-javalin</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
<artifactId>qqq-frontend-next</artifactId>
|
||||
<version>1.0.0-RC.14</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.h2database</groupId>
|
||||
<artifactId>h2</artifactId>
|
||||
<version>${h2.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.postgresql</groupId>
|
||||
<artifactId>postgresql</artifactId>
|
||||
<version>42.7.14</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.testcontainers</groupId>
|
||||
<artifactId>testcontainers-postgresql</artifactId>
|
||||
<version>${testcontainers.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.testcontainers</groupId>
|
||||
<artifactId>testcontainers-minio</artifactId>
|
||||
<version>${testcontainers.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.testcontainers</groupId>
|
||||
<artifactId>testcontainers-junit-jupiter</artifactId>
|
||||
<version>${testcontainers.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter</artifactId>
|
||||
<version>5.10.0</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.assertj</groupId>
|
||||
<artifactId>assertj-core</artifactId>
|
||||
<version>3.24.2</version>
|
||||
<version>3.27.7</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
@@ -45,14 +145,105 @@
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.11.0</version>
|
||||
<artifactId>maven-checkstyle-plugin</artifactId>
|
||||
<version>3.3.1</version>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.puppycrawl.tools</groupId>
|
||||
<artifactId>checkstyle</artifactId>
|
||||
<version>10.12.6</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<configuration>
|
||||
<configLocation>checkstyle/config.xml</configLocation>
|
||||
<headerLocation>checkstyle/license.txt</headerLocation>
|
||||
<encoding>UTF-8</encoding>
|
||||
<consoleOutput>true</consoleOutput>
|
||||
<failsOnError>false</failsOnError>
|
||||
<failOnViolation>true</failOnViolation>
|
||||
<violationSeverity>warning</violationSeverity>
|
||||
<includeTestSourceDirectory>false</includeTestSourceDirectory>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>validate</id>
|
||||
<phase>validate</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<version>3.1.2</version>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>${jacoco.version}</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>check</id>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.70</minimum>
|
||||
</limit>
|
||||
<limit>
|
||||
<counter>CLASS</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.90</minimum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
|
||||
<profiles>
|
||||
<!-- QQQ 4.1 APIs until qbit-build-parent imports the 4.1 BOM (as in qbit-quick-search and qbit-cdn, ADR-0007). -->
|
||||
<profile>
|
||||
<id>qqq-snapshot</id>
|
||||
<activation>
|
||||
<activeByDefault>true</activeByDefault>
|
||||
</activation>
|
||||
<properties>
|
||||
<!-- Latest QQQ RC on Maven Central (2026-10-13). -->
|
||||
<qqq.snapshot.version>4.1.0-RC.5</qqq.snapshot.version>
|
||||
</properties>
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
<artifactId>qqq-bom-pom</artifactId>
|
||||
<version>${qqq.snapshot.version}</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
</profile>
|
||||
</profiles>
|
||||
</project>
|
||||
@@ -1,75 +0,0 @@
|
||||
/*******************************************************************************
|
||||
** Configuration for the Example Extension QBit.
|
||||
*******************************************************************************/
|
||||
package com.kingsrook.qbits.example;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configuration class for the Example Extension QBit.
|
||||
*******************************************************************************/
|
||||
public class ExampleExtensionQBitConfig implements QBitConfig
|
||||
{
|
||||
private String targetTableName;
|
||||
private Boolean enableFeatureX = true;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Validate configuration before QBit is produced.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void validate(QInstance qInstance, List<String> errors)
|
||||
{
|
||||
if(targetTableName != null && qInstance.getTable(targetTableName) == null)
|
||||
{
|
||||
errors.add("Target table not found: " + targetTableName);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for targetTableName
|
||||
*******************************************************************************/
|
||||
public String getTargetTableName()
|
||||
{
|
||||
return targetTableName;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for targetTableName
|
||||
*******************************************************************************/
|
||||
public ExampleExtensionQBitConfig withTargetTableName(String targetTableName)
|
||||
{
|
||||
this.targetTableName = targetTableName;
|
||||
return this;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for enableFeatureX
|
||||
*******************************************************************************/
|
||||
public Boolean getEnableFeatureX()
|
||||
{
|
||||
return enableFeatureX;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for enableFeatureX
|
||||
*******************************************************************************/
|
||||
public ExampleExtensionQBitConfig withEnableFeatureX(Boolean enableFeatureX)
|
||||
{
|
||||
this.enableFeatureX = enableFeatureX;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
/*******************************************************************************
|
||||
** Producer for the Example Extension QBit.
|
||||
*******************************************************************************/
|
||||
package com.kingsrook.qbits.example;
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitProducer;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Producer class for the Example Extension QBit.
|
||||
*******************************************************************************/
|
||||
public class ExampleExtensionQBitProducer implements QBitProducer
|
||||
{
|
||||
public static final String GROUP_ID = "com.kingsrook.qbits";
|
||||
public static final String ARTIFACT_ID = "qbit-example-extension";
|
||||
public static final String VERSION = "0.1.0";
|
||||
|
||||
private ExampleExtensionQBitConfig config;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Produce this QBit into the given QInstance.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void produce(QInstance qInstance, String namespace) throws QException
|
||||
{
|
||||
QBitMetaData qBitMetaData = new QBitMetaData()
|
||||
.withGroupId(GROUP_ID)
|
||||
.withArtifactId(ARTIFACT_ID)
|
||||
.withVersion(VERSION)
|
||||
.withNamespace(namespace)
|
||||
.withConfig(config);
|
||||
|
||||
qInstance.addQBit(qBitMetaData);
|
||||
|
||||
if(config != null && config.getTargetTableName() != null)
|
||||
{
|
||||
applyTableCustomizations(qInstance, config.getTargetTableName());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Apply customizations to the target table.
|
||||
*******************************************************************************/
|
||||
private void applyTableCustomizations(QInstance qInstance, String tableName)
|
||||
{
|
||||
// Implement your extension logic here
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for config
|
||||
*******************************************************************************/
|
||||
public ExampleExtensionQBitProducer withConfig(ExampleExtensionQBitConfig config)
|
||||
{
|
||||
this.config = config;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
/*******************************************************************************
|
||||
** Example table customizer.
|
||||
*******************************************************************************/
|
||||
package com.kingsrook.qbits.example.customizers;
|
||||
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Example table customizer that adds behavior to existing tables.
|
||||
*******************************************************************************/
|
||||
public class ExampleTableCustomizer
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
** Apply customizations to the given table.
|
||||
*******************************************************************************/
|
||||
public void customize(QTableMetaData table)
|
||||
{
|
||||
// Example: Add audit tracking fields
|
||||
// table.withField(new QFieldMetaData("createdAt", QFieldType.DATE_TIME));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,597 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import java.security.PrivateKey;
|
||||
import java.security.PublicKey;
|
||||
import java.util.Map;
|
||||
import java.util.function.Supplier;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Host configuration. Private signing material never enters metadata records.
|
||||
*******************************************************************************/
|
||||
public class WormAuditQBitConfig implements QBitConfig
|
||||
{
|
||||
public static final int DEFAULT_RETENTION_YEARS = 6;
|
||||
|
||||
/*******************************************************************************
|
||||
** Signing identity; the matching public key must be pinned in trustedPublicKeys.
|
||||
*******************************************************************************/
|
||||
public record SigningKey(String keyId, PrivateKey privateKey)
|
||||
{
|
||||
}
|
||||
|
||||
private String exportLockMode = "GOVERNANCE";
|
||||
private int exportRetentionDays = 30;
|
||||
private final WormAuditSettings settings = new WormAuditSettings(this);
|
||||
private String appName = "worm_audit";
|
||||
private String appLabel = "WORM Audit";
|
||||
private String parentAppName;
|
||||
private String tablePathPrefix = "/app";
|
||||
private Supplier<com.kingsrook.qbits.wormaudit.export.SealedStore> sealedStoreSupplier;
|
||||
private String backendName = null;
|
||||
private int retentionYears = 6;
|
||||
private Supplier<SigningKey> signingKey = null;
|
||||
private Map<String, PublicKey> trustedPublicKeys = Map.of();
|
||||
private int chainIntervalSeconds = 30;
|
||||
private int settleHorizonSeconds = 60;
|
||||
private int batchSize = 1000;
|
||||
private int lateDetailLookbackHours = 24;
|
||||
private int checkpointEveryLinks = 10000;
|
||||
private int checkpointEveryMinutes = 60;
|
||||
private String verifyCron = "0 0 3 * * ?";
|
||||
private String exportBucketAlias = "worm_audit";
|
||||
private String exportPrefix = "worm-audit";
|
||||
private String exportCron = "0 0 2 * * ?";
|
||||
private String schedulerName = "worm_audit";
|
||||
private Map<String, String> coreAuditTableNames = Map.of();
|
||||
private String appRole = "app";
|
||||
private MetaDataCustomizerInterface<QTableMetaData> tableMetaDataCustomizer = null;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read exportLockMode.
|
||||
*******************************************************************************/
|
||||
public String getExportLockMode()
|
||||
{
|
||||
return (exportLockMode);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure exportLockMode.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withExportLockMode(String exportLockMode)
|
||||
{
|
||||
this.exportLockMode = exportLockMode;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read exportRetentionDays.
|
||||
*******************************************************************************/
|
||||
public int getExportRetentionDays()
|
||||
{
|
||||
return (exportRetentionDays);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure exportRetentionDays.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withExportRetentionDays(int exportRetentionDays)
|
||||
{
|
||||
this.exportRetentionDays = exportRetentionDays;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read effective settings through the instance-scoped 30-second cache.
|
||||
*******************************************************************************/
|
||||
@JsonIgnore
|
||||
public WormAuditSettings.Values effectiveSettings()
|
||||
{
|
||||
return (settings.read());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Discard this instance's cached row after a successful local settings write.
|
||||
*******************************************************************************/
|
||||
public void invalidateSettings()
|
||||
{
|
||||
settings.invalidate();
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the app name.
|
||||
*******************************************************************************/
|
||||
public String getAppName()
|
||||
{
|
||||
return (appName);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure the app name.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withAppName(String appName)
|
||||
{
|
||||
this.appName = appName;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the app label.
|
||||
*******************************************************************************/
|
||||
public String getAppLabel()
|
||||
{
|
||||
return (appLabel);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure the app label.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withAppLabel(String appLabel)
|
||||
{
|
||||
this.appLabel = appLabel;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the optional parent app.
|
||||
*******************************************************************************/
|
||||
public String getParentAppName()
|
||||
{
|
||||
return (parentAppName);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure the optional parent app.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withParentAppName(String parentAppName)
|
||||
{
|
||||
this.parentAppName = parentAppName;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the table route prefix.
|
||||
*******************************************************************************/
|
||||
public String getTablePathPrefix()
|
||||
{
|
||||
return (tablePathPrefix);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure the table route prefix.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withTablePathPrefix(String tablePathPrefix)
|
||||
{
|
||||
this.tablePathPrefix = tablePathPrefix;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the optional storage boundary (defaults to qbit-object-storage).
|
||||
*******************************************************************************/
|
||||
@JsonIgnore
|
||||
public Supplier<com.kingsrook.qbits.wormaudit.export.SealedStore> getSealedStoreSupplier()
|
||||
{
|
||||
return (sealedStoreSupplier);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure the optional storage boundary (defaults to qbit-object-storage).
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withSealedStoreSupplier(Supplier<com.kingsrook.qbits.wormaudit.export.SealedStore> sealedStoreSupplier)
|
||||
{
|
||||
this.sealedStoreSupplier = sealedStoreSupplier;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read backendName.
|
||||
*******************************************************************************/
|
||||
public String getBackendName()
|
||||
{
|
||||
return (backendName);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure backendName.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withBackendName(String backendName)
|
||||
{
|
||||
this.backendName = backendName;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read retentionYears.
|
||||
*******************************************************************************/
|
||||
public int getRetentionYears()
|
||||
{
|
||||
return (retentionYears);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure retentionYears.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withRetentionYears(int retentionYears)
|
||||
{
|
||||
this.retentionYears = retentionYears;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read signingKey.
|
||||
*******************************************************************************/
|
||||
@JsonIgnore
|
||||
public Supplier<SigningKey> getSigningKey()
|
||||
{
|
||||
return (signingKey);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure signingKey.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withSigningKey(Supplier<SigningKey> signingKey)
|
||||
{
|
||||
this.signingKey = signingKey;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read trustedPublicKeys.
|
||||
*******************************************************************************/
|
||||
@JsonIgnore
|
||||
public Map<String, PublicKey> getTrustedPublicKeys()
|
||||
{
|
||||
return (trustedPublicKeys);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure trustedPublicKeys.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withTrustedPublicKeys(Map<String, PublicKey> trustedPublicKeys)
|
||||
{
|
||||
this.trustedPublicKeys = trustedPublicKeys;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read chainIntervalSeconds.
|
||||
*******************************************************************************/
|
||||
public int getChainIntervalSeconds()
|
||||
{
|
||||
return (chainIntervalSeconds);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure chainIntervalSeconds.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withChainIntervalSeconds(int chainIntervalSeconds)
|
||||
{
|
||||
this.chainIntervalSeconds = chainIntervalSeconds;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read settleHorizonSeconds.
|
||||
*******************************************************************************/
|
||||
public int getSettleHorizonSeconds()
|
||||
{
|
||||
return (settleHorizonSeconds);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure settleHorizonSeconds.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withSettleHorizonSeconds(int settleHorizonSeconds)
|
||||
{
|
||||
this.settleHorizonSeconds = settleHorizonSeconds;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the bounded late-detail polling window in hours.
|
||||
*******************************************************************************/
|
||||
public int getLateDetailLookbackHours()
|
||||
{
|
||||
return (lateDetailLookbackHours);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure late-detail polling; older omissions are reported by verification.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withLateDetailLookbackHours(int lateDetailLookbackHours)
|
||||
{
|
||||
this.lateDetailLookbackHours = lateDetailLookbackHours;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read batchSize.
|
||||
*******************************************************************************/
|
||||
public int getBatchSize()
|
||||
{
|
||||
return (batchSize);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure batchSize.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withBatchSize(int batchSize)
|
||||
{
|
||||
this.batchSize = batchSize;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read checkpointEveryLinks.
|
||||
*******************************************************************************/
|
||||
public int getCheckpointEveryLinks()
|
||||
{
|
||||
return (checkpointEveryLinks);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure checkpointEveryLinks.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withCheckpointEveryLinks(int checkpointEveryLinks)
|
||||
{
|
||||
this.checkpointEveryLinks = checkpointEveryLinks;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read checkpointEveryMinutes.
|
||||
*******************************************************************************/
|
||||
public int getCheckpointEveryMinutes()
|
||||
{
|
||||
return (checkpointEveryMinutes);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure checkpointEveryMinutes.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withCheckpointEveryMinutes(int checkpointEveryMinutes)
|
||||
{
|
||||
this.checkpointEveryMinutes = checkpointEveryMinutes;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read verifyCron.
|
||||
*******************************************************************************/
|
||||
public String getVerifyCron()
|
||||
{
|
||||
return (verifyCron);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the Object Lock bucket alias.
|
||||
*******************************************************************************/
|
||||
public String getExportBucketAlias()
|
||||
{
|
||||
return (exportBucketAlias);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure the Object Lock bucket alias.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withExportBucketAlias(String exportBucketAlias)
|
||||
{
|
||||
this.exportBucketAlias = exportBucketAlias;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the export key prefix.
|
||||
*******************************************************************************/
|
||||
public String getExportPrefix()
|
||||
{
|
||||
return (exportPrefix);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure the export key prefix.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withExportPrefix(String exportPrefix)
|
||||
{
|
||||
this.exportPrefix = exportPrefix;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the UTC export schedule.
|
||||
*******************************************************************************/
|
||||
public String getExportCron()
|
||||
{
|
||||
return (exportCron);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure the UTC export schedule.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withExportCron(String exportCron)
|
||||
{
|
||||
this.exportCron = exportCron;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure verifyCron.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withVerifyCron(String verifyCron)
|
||||
{
|
||||
this.verifyCron = verifyCron;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read schedulerName.
|
||||
*******************************************************************************/
|
||||
public String getSchedulerName()
|
||||
{
|
||||
return (schedulerName);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure schedulerName.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withSchedulerName(String schedulerName)
|
||||
{
|
||||
this.schedulerName = schedulerName;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read coreAuditTableNames.
|
||||
*******************************************************************************/
|
||||
public Map<String, String> getCoreAuditTableNames()
|
||||
{
|
||||
return (coreAuditTableNames);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure coreAuditTableNames.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withCoreAuditTableNames(Map<String, String> coreAuditTableNames)
|
||||
{
|
||||
this.coreAuditTableNames = coreAuditTableNames;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read appRole.
|
||||
*******************************************************************************/
|
||||
public String getAppRole()
|
||||
{
|
||||
return (appRole);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure appRole.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withAppRole(String appRole)
|
||||
{
|
||||
this.appRole = appRole;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read tableMetaDataCustomizer.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public MetaDataCustomizerInterface<QTableMetaData> getTableMetaDataCustomizer()
|
||||
{
|
||||
return (tableMetaDataCustomizer);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure tableMetaDataCustomizer.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withTableMetaDataCustomizer(MetaDataCustomizerInterface<QTableMetaData> tableMetaDataCustomizer)
|
||||
{
|
||||
this.tableMetaDataCustomizer = tableMetaDataCustomizer;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reject invalid runtime limits before producing metadata.
|
||||
*******************************************************************************/
|
||||
public void validate()
|
||||
{
|
||||
if(appName == null || !appName.matches("[A-Za-z0-9_-]+") || appLabel == null || appLabel.isBlank()
|
||||
|| (parentAppName != null && !parentAppName.matches("[A-Za-z0-9_-]+"))
|
||||
|| tablePathPrefix == null || !tablePathPrefix.matches("/[A-Za-z0-9_/-]*") || tablePathPrefix.endsWith("/"))
|
||||
{
|
||||
throw (new IllegalArgumentException("App names and a local table path prefix are required"));
|
||||
}
|
||||
if(exportBucketAlias == null || exportBucketAlias.isBlank() || exportCron == null || exportCron.isBlank()
|
||||
|| exportPrefix == null || !exportPrefix.matches("[A-Za-z0-9_-]+(/[A-Za-z0-9_-]+)*"))
|
||||
{
|
||||
throw (new IllegalArgumentException("Export bucket, prefix and schedule must be configured"));
|
||||
}
|
||||
WormAuditSettings.validate(WormAuditSettings.defaults(this));
|
||||
if(backendName == null || backendName.isBlank() || retentionYears < 1 || chainIntervalSeconds < 1
|
||||
|| lateDetailLookbackHours < 1 || settleHorizonSeconds < 0 || batchSize < 1 || checkpointEveryLinks < 1 || checkpointEveryMinutes < 1)
|
||||
{
|
||||
throw (new IllegalArgumentException("backendName is required and WORM audit limits must be positive (settle horizon may be zero)"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,276 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.wormaudit.app.WormAuditApp;
|
||||
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
|
||||
import com.kingsrook.qbits.wormaudit.guard.AppendOnlyGuard;
|
||||
import com.kingsrook.qbits.wormaudit.model.LinkRangeWidget;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.ExportProcesses;
|
||||
import com.kingsrook.qbits.wormaudit.processes.WormAuditProcesses;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertDefaultMonitor;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertSeverity;
|
||||
import com.kingsrook.qqq.backend.core.alerts.QAlertTypeMetaData;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaDataProducer;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.quartz.QuartzSchedulerMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Produces append-only audit metadata, verification actions and schedules.
|
||||
*******************************************************************************/
|
||||
public class WormAuditQBitProducer implements QBitMetaDataProducer<WormAuditQBitConfig>
|
||||
{
|
||||
public static final String GROUP_ID = "com.kingsrook.qbits";
|
||||
public static final String ARTIFACT_ID = "qbit-worm-audit";
|
||||
public static final String VERSION = "0.1.0";
|
||||
|
||||
private WormAuditQBitConfig qBitConfig;
|
||||
private String namespace;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for the config.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitProducer withQBitConfig(WormAuditQBitConfig config)
|
||||
{
|
||||
this.qBitConfig = config;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for the config.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public WormAuditQBitConfig getQBitConfig()
|
||||
{
|
||||
return (qBitConfig);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for the namespace.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitProducer withNamespace(String namespace)
|
||||
{
|
||||
this.namespace = namespace;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for the namespace.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getNamespace()
|
||||
{
|
||||
return (namespace);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Identity and configuration of this qbit.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QBitMetaData getQBitMetaData()
|
||||
{
|
||||
return (new QBitMetaData().withGroupId(GROUP_ID).withArtifactId(ARTIFACT_ID).withVersion(VERSION).withNamespace(namespace).withConfig(qBitConfig));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Package-scanned producers have no config: skip them.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean isEnabled()
|
||||
{
|
||||
return (qBitConfig != null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Register audit tables, verification/export actions and schedules.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public MetaDataProducerMultiOutput produce(QInstance qInstance) throws QException
|
||||
{
|
||||
qBitConfig.validate();
|
||||
if(qInstance.getBackend(qBitConfig.getBackendName()) == null)
|
||||
{
|
||||
throw (new IllegalArgumentException("Configured audit backend is missing"));
|
||||
}
|
||||
WormAuditDdl.resolveNames(qBitConfig, qInstance);
|
||||
for(String core : AppendOnlyGuard.CORE_TABLES)
|
||||
{
|
||||
if(!qBitConfig.getBackendName().equals(qInstance.getTable(core).getBackendName()))
|
||||
{
|
||||
throw (new IllegalArgumentException("Core and WORM audit tables must share one backend"));
|
||||
}
|
||||
}
|
||||
QBitMetaData qBit = getQBitMetaData();
|
||||
MetaDataProducerMultiOutput output = new MetaDataProducerMultiOutput().with(qBit);
|
||||
AppendOnlyGuard guard = new AppendOnlyGuard();
|
||||
for(QTableMetaData table : WormAuditTables.tables(qBitConfig))
|
||||
{
|
||||
if(qBitConfig.getTableMetaDataCustomizer() != null)
|
||||
{
|
||||
table = qBitConfig.getTableMetaDataCustomizer().customizeMetaData(qInstance, table);
|
||||
}
|
||||
for(var field : table.getFields().values())
|
||||
{
|
||||
if(field.getInlinePossibleValueSource() != null)
|
||||
{
|
||||
output.add(field.getInlinePossibleValueSource());
|
||||
field.withPossibleValueSourceName(field.getInlinePossibleValueSource().getName()).withInlinePossibleValueSource(null);
|
||||
}
|
||||
}
|
||||
guard.customizeMetaData(qInstance, table);
|
||||
table.setSourceQBitName(qBit.getName());
|
||||
output.add(table);
|
||||
}
|
||||
for(String core : AppendOnlyGuard.CORE_TABLES)
|
||||
{
|
||||
guard.customizeMetaData(qInstance, qInstance.getTable(core));
|
||||
}
|
||||
for(String name : WormAuditTables.ALL)
|
||||
{
|
||||
QPossibleValueSource source = QPossibleValueSource.newForTable(name);
|
||||
if(WormAuditTables.CHAIN.equals(name))
|
||||
{
|
||||
source.withValueFormat("%s").withValueFields(java.util.List.of("id"));
|
||||
}
|
||||
output.add(source);
|
||||
}
|
||||
for(String name : java.util.List.of(WormAuditTables.CHECKPOINT, WormAuditTables.EXPORT, WormAuditTables.VERIFICATION))
|
||||
{
|
||||
output.add(new QWidgetMetaData().withName(name + "_links").withLabel("Chain links").withType("childRecordList").withGridColumns(12)
|
||||
.withCodeReference(new QCodeReference(LinkRangeWidget.class)).withDefaultValue("parentTable", name));
|
||||
}
|
||||
List<QTableMetaData> producedTables = output.getEach(QTableMetaData.class);
|
||||
output.addSelfToInstance(qInstance);
|
||||
output = new MetaDataProducerMultiOutput();
|
||||
var previousInstance = QContext.getQInstance();
|
||||
var previousSession = QContext.getQSession();
|
||||
var previousTransaction = QContext.getQBackendTransaction();
|
||||
try
|
||||
{
|
||||
// Production is still assembling the host metadata. Do not validate it mid-build.
|
||||
QContext.setQInstance(qInstance);
|
||||
QContext.setQSession(new com.kingsrook.qqq.backend.core.model.session.QSession());
|
||||
QContext.setTransaction(null);
|
||||
addProcesses(output, qBit);
|
||||
}
|
||||
finally
|
||||
{
|
||||
QContext.setQInstance(previousInstance);
|
||||
QContext.setQSession(previousSession);
|
||||
QContext.setTransaction(previousTransaction);
|
||||
}
|
||||
if(qInstance.getSchedulers() == null || !qInstance.getSchedulers().containsKey(qBitConfig.getSchedulerName()))
|
||||
{
|
||||
qInstance.addScheduler(new QuartzSchedulerMetaData().withName(qBitConfig.getSchedulerName()));
|
||||
}
|
||||
output.add(WormAuditApp.app(qBitConfig, producedTables));
|
||||
for(QWidgetMetaData widget : WormAuditApp.widgets())
|
||||
{
|
||||
output.add(widget);
|
||||
}
|
||||
output.add(new QAlertTypeMetaData().withName("wormaudit.chain.break").withLabel("Audit chain break")
|
||||
.withSourceQBitName(qBit.getName()).withDefaultSeverity(AlertSeverity.CRITICAL)
|
||||
.withDescription("Audit chain verification found a break in content, continuity, coverage or checkpoint trust.")
|
||||
.withSubjectTableName(WormAuditTables.VERIFICATION).withVisibilityPermission("worm_audit_verification.read"));
|
||||
output.add(new QAlertTypeMetaData().withName("wormaudit.export.failed").withLabel("Sealed audit export failed")
|
||||
.withSourceQBitName(qBit.getName()).withDefaultSeverity(AlertSeverity.ERROR)
|
||||
.withDescription("The audit export could not be sealed with verified retention and object versions.")
|
||||
.withSubjectTableName(WormAuditTables.EXPORT).withVisibilityPermission("worm_audit_export.read"));
|
||||
for(String job : List.of("verify", "export"))
|
||||
{
|
||||
String name = "wormaudit." + job + ".daily";
|
||||
output.add(new QAlertTypeMetaData().withName(name).withLabel("Daily audit " + job + " overdue")
|
||||
.withSourceQBitName(qBit.getName()).withDefaultSeverity(AlertSeverity.ERROR)
|
||||
.withDescription("The daily audit " + job + " job has not sent a heartbeat within 26 hours.")
|
||||
.withVisibilityPermission(job.equals("verify") ? "worm_audit_verification.read" : "worm_audit_export.read")
|
||||
.withDefaultMonitor(AlertDefaultMonitor.heartbeat(name, name, 24 * 3600, 2 * 3600)));
|
||||
}
|
||||
output.addSelfToInstance(qInstance);
|
||||
denyAssistant(qInstance);
|
||||
return (new MetaDataProducerMultiOutput());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build schedules from settings available at producer startup.
|
||||
*******************************************************************************/
|
||||
private void addProcesses(MetaDataProducerMultiOutput output, QBitMetaData qBit)
|
||||
{
|
||||
for(var process : WormAuditProcesses.processes(qBitConfig))
|
||||
{
|
||||
process.setSourceQBitName(qBit.getName());
|
||||
output.add(process);
|
||||
}
|
||||
for(var process : ExportProcesses.processes(qBitConfig))
|
||||
{
|
||||
process.setSourceQBitName(qBit.getName());
|
||||
output.add(process);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** RC.5 compatibility: replace with a direct call when the qbit builds on RC.6 (qqq#1141, #1163).
|
||||
*******************************************************************************/
|
||||
private void denyAssistant(QInstance instance) throws QException
|
||||
{
|
||||
try
|
||||
{
|
||||
Class<?> type = Class.forName("com.kingsrook.qqq.backend.core.model.metadata.permissions.ClientChannelsMetaData");
|
||||
Object metadata = type.getMethod("ofOrWithNew", QInstance.class).invoke(null, instance);
|
||||
type.getMethod("withDeniedPattern", String.class, String.class).invoke(metadata, "assistant", "worm_audit_*");
|
||||
}
|
||||
catch(ClassNotFoundException expected)
|
||||
{
|
||||
// The assistant wall API is absent in RC.5.
|
||||
}
|
||||
catch(ReflectiveOperationException e)
|
||||
{
|
||||
throw (new QException("Cannot register the WORM audit assistant deny pattern", e));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,246 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import java.sql.SQLException;
|
||||
import java.time.Clock;
|
||||
import java.time.Instant;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import org.quartz.CronExpression;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Singleton settings merged over host defaults. Only absent rows/tables fall
|
||||
** back; database outages and invalid persisted settings fail closed.
|
||||
*******************************************************************************/
|
||||
public class WormAuditSettings
|
||||
{
|
||||
private final WormAuditQBitConfig config;
|
||||
private final Clock clock;
|
||||
private QInstance instance;
|
||||
private QRecord cached;
|
||||
private Instant expires = Instant.MIN;
|
||||
|
||||
/*******************************************************************************
|
||||
** An immutable snapshot for one run.
|
||||
*******************************************************************************/
|
||||
public record Values(int audit_retention_years,
|
||||
String export_lock_mode,
|
||||
int export_retention_days,
|
||||
int checkpoint_every_links,
|
||||
int checkpoint_every_minutes,
|
||||
int chain_interval_seconds,
|
||||
int settle_horizon_seconds,
|
||||
int late_detail_lookback_hours,
|
||||
String export_cron,
|
||||
String verify_cron)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Use the production clock.
|
||||
*******************************************************************************/
|
||||
public WormAuditSettings(WormAuditQBitConfig config)
|
||||
{
|
||||
this(config, Clock.systemUTC());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Inject a clock for cache boundary tests.
|
||||
*******************************************************************************/
|
||||
public WormAuditSettings(WormAuditQBitConfig config, Clock clock)
|
||||
{
|
||||
this.config = config;
|
||||
this.clock = clock;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Invalidate after a local edit; other application instances refresh within 30 seconds.
|
||||
*******************************************************************************/
|
||||
public synchronized void invalidate()
|
||||
{
|
||||
expires = Instant.MIN;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Seed the singleton only when absent; never overwrite an administrator edit.
|
||||
*******************************************************************************/
|
||||
public static void seed(WormAuditQBitConfig config) throws com.kingsrook.qqq.backend.core.exceptions.QException
|
||||
{
|
||||
var store = new AuditStore(null);
|
||||
if(store.get(WormAuditTables.SETTINGS, 1) == null)
|
||||
{
|
||||
store.insert(WormAuditTables.SETTINGS, java.util.List.of(defaults(config)));
|
||||
}
|
||||
config.invalidateSettings();
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read at most once per 30 seconds, isolated by QInstance identity.
|
||||
*******************************************************************************/
|
||||
public synchronized Values read()
|
||||
{
|
||||
QInstance current = QContext.getQInstance();
|
||||
if(current != instance || !clock.instant().isBefore(expires))
|
||||
{
|
||||
QRecord row = null;
|
||||
if(current != null && current.getTable(WormAuditTables.SETTINGS) != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
row = new AuditStore(null).get(WormAuditTables.SETTINGS, 1);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
if(!missingTable(e))
|
||||
{
|
||||
throw (new IllegalStateException("Cannot read WORM Audit Settings", e));
|
||||
}
|
||||
}
|
||||
}
|
||||
cached = row;
|
||||
instance = current;
|
||||
expires = row == null ? Instant.MIN : clock.instant().plusSeconds(30);
|
||||
}
|
||||
QRecord row = defaults(config);
|
||||
if(cached != null)
|
||||
{
|
||||
for(String field : row.getValues().keySet())
|
||||
{
|
||||
if(cached.getValue(field) != null)
|
||||
{
|
||||
row.setValue(field, cached.getValue(field));
|
||||
}
|
||||
}
|
||||
}
|
||||
validate(row);
|
||||
return (new Values(row.getValueInteger("audit_retention_years"),
|
||||
row.getValueString("export_lock_mode"),
|
||||
row.getValueInteger("export_retention_days"),
|
||||
row.getValueInteger("checkpoint_every_links"),
|
||||
row.getValueInteger("checkpoint_every_minutes"),
|
||||
row.getValueInteger("chain_interval_seconds"),
|
||||
row.getValueInteger("settle_horizon_seconds"),
|
||||
row.getValueInteger("late_detail_lookback_hours"),
|
||||
row.getValueString("export_cron"),
|
||||
row.getValueString("verify_cron")));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build insert-only seed values from the host defaults.
|
||||
*******************************************************************************/
|
||||
public static QRecord defaults(WormAuditQBitConfig config)
|
||||
{
|
||||
return (new QRecord().withValue("id", 1)
|
||||
.withValue("audit_retention_years", config.getRetentionYears())
|
||||
.withValue("export_lock_mode", config.getExportLockMode())
|
||||
.withValue("export_retention_days", config.getExportRetentionDays())
|
||||
.withValue("checkpoint_every_links", config.getCheckpointEveryLinks())
|
||||
.withValue("checkpoint_every_minutes", config.getCheckpointEveryMinutes())
|
||||
.withValue("chain_interval_seconds", config.getChainIntervalSeconds())
|
||||
.withValue("settle_horizon_seconds", config.getSettleHorizonSeconds())
|
||||
.withValue("late_detail_lookback_hours", config.getLateDetailLookbackHours())
|
||||
.withValue("export_cron", config.getExportCron())
|
||||
.withValue("verify_cron", config.getVerifyCron()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Validate supplied fields; partial updates leave omitted fields untouched.
|
||||
*******************************************************************************/
|
||||
public static void validate(QRecord row)
|
||||
{
|
||||
if(row.getValue("id") != null && !Integer.valueOf(1).equals(row.getValueInteger("id")))
|
||||
{
|
||||
throw (new IllegalArgumentException("WORM Audit Settings has one row only (ID 1)."));
|
||||
}
|
||||
range(row, "audit_retention_years", "Audit retention years", 1, 100);
|
||||
range(row, "export_retention_days", "Export retention days", 1, 36500);
|
||||
range(row, "checkpoint_every_links", "Checkpoint every links", 1, 2147483647);
|
||||
range(row, "checkpoint_every_minutes", "Checkpoint every minutes", 1, 2147483647);
|
||||
range(row, "chain_interval_seconds", "Chain interval seconds", 1, 2147483647);
|
||||
range(row, "settle_horizon_seconds", "Settle horizon seconds", 0, 2147483647);
|
||||
range(row, "late_detail_lookback_hours", "Late detail lookback hours", 1, 2147483647);
|
||||
if(row.getValues().containsKey("export_lock_mode") && !java.util.Arrays.asList("GOVERNANCE", "COMPLIANCE").contains(row.getValueString("export_lock_mode")))
|
||||
{
|
||||
throw (new IllegalArgumentException("Export lock mode must be GOVERNANCE or COMPLIANCE."));
|
||||
}
|
||||
for(String field : java.util.List.of("export_cron", "verify_cron"))
|
||||
{
|
||||
if(row.getValues().containsKey(field) && (row.getValueString(field) == null || !CronExpression.isValidExpression(row.getValueString(field))))
|
||||
{
|
||||
throw (new IllegalArgumentException((field.equals("export_cron") ? "Export" : "Verify") + " schedule must be a valid Quartz cron expression (UTC)."));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reject null, fractional and out-of-range numbers with a plain field label.
|
||||
*******************************************************************************/
|
||||
private static void range(QRecord row, String field, String label, int minimum, int maximum)
|
||||
{
|
||||
if(!row.getValues().containsKey(field))
|
||||
{
|
||||
return;
|
||||
}
|
||||
try
|
||||
{
|
||||
long value = Long.parseLong(String.valueOf(row.getValue(field)));
|
||||
if(value >= minimum && value <= maximum)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
catch(NumberFormatException ignored)
|
||||
{
|
||||
// Use the same clear message for invalid numbers and invalid ranges.
|
||||
}
|
||||
throw (new IllegalArgumentException(label + " must be a whole number from " + minimum + " to " + maximum + "."));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Recognize PostgreSQL/H2 missing-table errors without hiding other failures.
|
||||
*******************************************************************************/
|
||||
private static boolean missingTable(Throwable error)
|
||||
{
|
||||
for(Throwable cause = error; cause != null; cause = cause.getCause())
|
||||
{
|
||||
if(cause instanceof SQLException sql && ("42P01".equals(sql.getSQLState()) || "42S02".equals(sql.getSQLState()) || "42S04".equals(sql.getSQLState())))
|
||||
{
|
||||
return (true);
|
||||
}
|
||||
}
|
||||
return (false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.app;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSTransaction;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read-only PostgreSQL catalog probes; errors become value-free diagnostics.
|
||||
*******************************************************************************/
|
||||
public final class PostgresSetupWarnings
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private PostgresSetupWarnings()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Probe the runtime connection, never the configured migration role.
|
||||
*******************************************************************************/
|
||||
public static List<String> load(QBackendTransaction transaction, WormAuditQBitConfig config)
|
||||
{
|
||||
List<String> warnings = new ArrayList<>();
|
||||
int years = config.effectiveSettings().audit_retention_years();
|
||||
if(years != config.getRetentionYears())
|
||||
{
|
||||
warnings.add("Audit retention setting differs from the configured database purge interval (" + config.getRetentionYears() + " years). A reviewed migration is required.");
|
||||
}
|
||||
if(!(transaction instanceof RDBMSTransaction rdbms))
|
||||
{
|
||||
return (warnings);
|
||||
}
|
||||
try
|
||||
{
|
||||
var connection = rdbms.getConnection();
|
||||
if(!"PostgreSQL".equals(connection.getMetaData().getDatabaseProductName()))
|
||||
{
|
||||
return (warnings);
|
||||
}
|
||||
// The live function is authoritative on PostgreSQL, rather than the renderer defaults.
|
||||
warnings.clear();
|
||||
List<String> tables = new ArrayList<>(config.getCoreAuditTableNames().values());
|
||||
tables.addAll(WormAuditTables.ALL.stream().filter(t -> !WormAuditTables.SETTINGS.equals(t)).toList());
|
||||
for(String table : tables)
|
||||
{
|
||||
for(String trigger : List.of("worm_audit_row_guard", "worm_audit_truncate_guard"))
|
||||
{
|
||||
try(var statement = connection.prepareStatement("SELECT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid=to_regclass(?) AND tgname=? AND NOT tgisinternal AND tgenabled IN ('O','A'))"))
|
||||
{
|
||||
statement.setString(1, WormAuditDdl.identifier(table));
|
||||
statement.setString(2, trigger);
|
||||
try(var result = statement.executeQuery())
|
||||
{
|
||||
result.next();
|
||||
if(!result.getBoolean(1))
|
||||
{
|
||||
warnings.add("Missing or disabled " + trigger + " on " + table);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
try(var statement = connection.prepareStatement("SELECT EXISTS (SELECT 1 FROM pg_tables WHERE tableowner=current_user AND to_regclass(quote_ident(schemaname)||'.'||quote_ident(tablename))=to_regclass(?))"))
|
||||
{
|
||||
statement.setString(1, WormAuditDdl.identifier(table));
|
||||
try(var result = statement.executeQuery())
|
||||
{
|
||||
result.next();
|
||||
if(result.getBoolean(1))
|
||||
{
|
||||
warnings.add("Runtime role owns audit table " + table);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
try(var statement = connection.createStatement(); var result = statement.executeQuery("SELECT pg_get_functiondef(to_regprocedure('public.worm_audit_purge(timestamp with time zone)'))"))
|
||||
{
|
||||
if(result.next() && result.getString(1) != null)
|
||||
{
|
||||
var interval = java.util.regex.Pattern.compile("(?i)interval\\s+'([0-9]+) years'").matcher(result.getString(1));
|
||||
if(interval.find() && Integer.parseInt(interval.group(1)) != years)
|
||||
{
|
||||
warnings.add("Audit retention setting differs from the purge function's compiled interval (" + interval.group(1) + " years). A reviewed migration is required.");
|
||||
}
|
||||
}
|
||||
}
|
||||
try(var statement = connection.createStatement(); var result = statement.executeQuery("SELECT to_regprocedure('public.worm_audit_purge(timestamp with time zone)') IS NOT NULL"))
|
||||
{
|
||||
result.next();
|
||||
if(!result.getBoolean(1))
|
||||
{
|
||||
warnings.add("Retention purge function is missing");
|
||||
}
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
warnings.add("PostgreSQL setup checks unavailable");
|
||||
}
|
||||
return (warnings);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.app;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.VerificationRuns;
|
||||
import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.CustomPermissionChecker;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML;
|
||||
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** WORM Audit navigation and standard HTML widgets, scoped to this QInstance.
|
||||
*******************************************************************************/
|
||||
public final class WormAuditApp
|
||||
{
|
||||
public static final String HOME_WIDGET = "wormAuditHome";
|
||||
public static final String HEALTH_CARD_WIDGET = "wormAuditHealthCard";
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private WormAuditApp()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The home and optional host-dashboard summary share the app's permissions.
|
||||
*******************************************************************************/
|
||||
public static List<QWidgetMetaData> widgets()
|
||||
{
|
||||
return (List.of(
|
||||
new QWidgetMetaData().withName(HOME_WIDGET).withLabel("Overview").withType(WidgetType.HTML.getType())
|
||||
.withCodeReference(new QCodeReference(HomeRenderer.class)).withGridColumns(12).withIsCard(false).withPermissionRules(permissions()),
|
||||
new QWidgetMetaData().withName(HEALTH_CARD_WIDGET).withLabel("WORM Audit").withType(WidgetType.HTML.getType())
|
||||
.withCodeReference(new QCodeReference(HealthCardRenderer.class)).withGridColumns(12).withIsCard(true).withPermissionRules(permissions())));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Home, Verifications, Sealed Exports, Checkpoints, Audit Chain, Purges.
|
||||
*******************************************************************************/
|
||||
public static QAppMetaData app(WormAuditQBitConfig config, List<QTableMetaData> tables)
|
||||
{
|
||||
QAppMetaData app = new QAppMetaData().withName(config.getAppName()).withLabel(config.getAppLabel())
|
||||
.withIcon(new QIcon("verified_user")).withPermissionRules(permissions()).withWidgets(List.of(HOME_WIDGET));
|
||||
app.setParentAppName(config.getParentAppName());
|
||||
for(String name : WormAuditTables.NAV)
|
||||
{
|
||||
QTableMetaData table = tables.stream().filter(t -> t.getName().equals(name)).findFirst().orElseThrow();
|
||||
app.withSectionOfChildren(new QAppSection().withName(name).withLabel(WormAuditTables.navigationLabel(name)),
|
||||
WormAuditTables.SETTINGS.equals(name) ? table.clone().withLabel("Settings") : table);
|
||||
}
|
||||
return (app);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read on any WORM table grants the value-free aggregate overview.
|
||||
*******************************************************************************/
|
||||
public static QPermissionRules permissions()
|
||||
{
|
||||
return (new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION)
|
||||
.withCustomPermissionChecker(new QCodeReference(HomePermissionChecker.class)));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Honor the host's actual table permission rules, including custom checkers.
|
||||
*******************************************************************************/
|
||||
public static class HomePermissionChecker implements CustomPermissionChecker
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Reject direct widget requests unless at least one table is readable.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void checkPermissionsThrowing(AbstractActionInput input, MetaDataWithPermissionRules metadata) throws QPermissionDeniedException
|
||||
{
|
||||
for(String table : WormAuditTables.ALL)
|
||||
{
|
||||
try
|
||||
{
|
||||
PermissionsHelper.checkTablePermissionThrowing(new QueryInput(table), TablePermissionSubType.READ);
|
||||
return;
|
||||
}
|
||||
catch(QPermissionDeniedException expected)
|
||||
{
|
||||
// Another WORM table may grant access to the aggregate overview.
|
||||
}
|
||||
}
|
||||
throw (new QPermissionDeniedException("Read access to a WORM audit table is required"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Full-width home renderer.
|
||||
*******************************************************************************/
|
||||
public static class HomeRenderer extends AbstractWidgetRenderer
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Render only metadata, counts and value-free reasons.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public RenderWidgetOutput render(RenderWidgetInput input) throws QException
|
||||
{
|
||||
new HomePermissionChecker().checkPermissionsThrowing(input, null);
|
||||
return (new RenderWidgetOutput(new RawHTML(null, WormAuditHomeView.render(WormAuditHomeData.load(), VerificationRuns.config().getTablePathPrefix()))));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Optional compact host dashboard renderer.
|
||||
*******************************************************************************/
|
||||
public static class HealthCardRenderer extends AbstractWidgetRenderer
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Link the status summary to the owning app home.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public RenderWidgetOutput render(RenderWidgetInput input) throws QException
|
||||
{
|
||||
new HomePermissionChecker().checkPermissionsThrowing(input, null);
|
||||
WormAuditQBitConfig config = VerificationRuns.config();
|
||||
String path = config.getTablePathPrefix() + "/" + (config.getParentAppName() == null ? "" : config.getParentAppName() + "/") + config.getAppName();
|
||||
return (new RenderWidgetOutput(new RawHTML(config.getAppLabel(), WormAuditHomeView.renderHealthCard(WormAuditHomeData.load(), path))));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.app;
|
||||
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.export.ObjectStorageSealedStore;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedStore;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.VerificationRuns;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSTransaction;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read-only home snapshot. No audit messages, detail values or canonical content
|
||||
** enter this model. Unexported suffixes are lag; holes before the sealed head
|
||||
** are gaps. Only a later full scheduled chain PASS clears an open chain break.
|
||||
*******************************************************************************/
|
||||
public record WormAuditHomeData(long links, Instant lastLinkAt, long backlog, long settling, int settleSeconds,
|
||||
QRecord lastVerification, QRecord openBreak, QRecord lastExport, long exportLag, Long exportAgeHours,
|
||||
List<String> exportGaps, String bucketAlias, String lockCapability, String lockReason,
|
||||
String exportLockMode, int exportRetentionDays, int retentionYears, QRecord anchor, Instant oldestAudit, Instant nextPurge,
|
||||
List<QRecord> recentPurges, List<String> warnings)
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Load using the current instance's configuration and production clock.
|
||||
*******************************************************************************/
|
||||
public static WormAuditHomeData load() throws QException
|
||||
{
|
||||
return (load(VerificationRuns.config(), null, Instant.now()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The storage and time seams also serve the memory demo and deterministic tests.
|
||||
*******************************************************************************/
|
||||
public static WormAuditHomeData load(WormAuditQBitConfig config, SealedStore storage, Instant now) throws QException
|
||||
{
|
||||
var settings = config.effectiveSettings();
|
||||
List<String> warnings = new ArrayList<>();
|
||||
if("GOVERNANCE".equals(settings.export_lock_mode()))
|
||||
{
|
||||
warnings.add("Sealed exports use GOVERNANCE mode: an administrator with s3:BypassGovernanceRetention can delete them. Switch to COMPLIANCE for write-once copies.");
|
||||
}
|
||||
if(now.plus(Duration.ofDays(settings.export_retention_days())).isBefore(now.atZone(ZoneOffset.UTC).plusYears(settings.audit_retention_years()).toInstant()))
|
||||
{
|
||||
warnings.add("Export retention is shorter than audit retention.");
|
||||
}
|
||||
String capability = "UNKNOWN";
|
||||
String reason = "Object Lock capability could not be confirmed";
|
||||
try
|
||||
{
|
||||
SealedStore resolved = storage != null ? storage : config.getSealedStoreSupplier() == null
|
||||
? new ObjectStorageSealedStore(config.getExportBucketAlias(), "home") : config.getSealedStoreSupplier().get();
|
||||
CapabilityStatus status = resolved.lockCapability();
|
||||
if(status == CapabilityStatus.SUPPORTED || status == CapabilityStatus.UNSUPPORTED)
|
||||
{
|
||||
capability = status.name();
|
||||
reason = status == CapabilityStatus.SUPPORTED ? null : "Export bucket does not support Object Lock";
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
// Provider exception messages can contain credentials or object content.
|
||||
reason = "Object Lock capability check failed";
|
||||
}
|
||||
if(reason != null)
|
||||
{
|
||||
warnings.add(reason);
|
||||
}
|
||||
if(config.getSigningKey() == null)
|
||||
{
|
||||
warnings.add("No signing key configured");
|
||||
}
|
||||
if(config.getTrustedPublicKeys() == null || config.getTrustedPublicKeys().isEmpty())
|
||||
{
|
||||
warnings.add("No trusted public keys configured");
|
||||
}
|
||||
try(QBackendTransaction transaction = QBackendTransaction.openFor(new QueryInput(WormAuditTables.CHAIN)))
|
||||
{
|
||||
try
|
||||
{
|
||||
if(transaction instanceof RDBMSTransaction rdbms)
|
||||
{
|
||||
rdbms.getConnection().setTransactionIsolation(Connection.TRANSACTION_REPEATABLE_READ);
|
||||
}
|
||||
AuditStore store = new AuditStore(transaction);
|
||||
QRecord lastLink = store.last(WormAuditTables.CHAIN);
|
||||
long links = store.count(WormAuditTables.CHAIN, new QQueryFilter());
|
||||
Instant cutoff = now.minusSeconds(settings.settle_horizon_seconds());
|
||||
long backlog = store.unchainedCount(cutoff, true);
|
||||
long settling = store.unchainedCount(cutoff, false);
|
||||
QRecord verification = latest(store, WormAuditTables.VERIFICATION, new QQueryFilter(), "finished_at");
|
||||
QQueryFilter dailyFilter = new QQueryFilter(new QFilterCriteria("kind", QCriteriaOperator.EQUALS, "CHAIN"),
|
||||
new QFilterCriteria("trigger", QCriteriaOperator.EQUALS, "SCHEDULE"));
|
||||
QRecord daily = latest(store, WormAuditTables.VERIFICATION, dailyFilter, "finished_at");
|
||||
if(daily == null || daily.getValueInstant("finished_at") == null)
|
||||
{
|
||||
warnings.add("Daily verification has never run");
|
||||
}
|
||||
else if(daily.getValueInstant("finished_at").isBefore(now.minus(Duration.ofHours(26))))
|
||||
{
|
||||
warnings.add("Daily verification last ran more than 26 hours ago");
|
||||
}
|
||||
QRecord failure = latest(store, WormAuditTables.VERIFICATION, new QQueryFilter(
|
||||
new QFilterCriteria("kind", QCriteriaOperator.EQUALS, "CHAIN"), new QFilterCriteria("outcome", QCriteriaOperator.EQUALS, "FAIL")), "id");
|
||||
QRecord fullPass = latest(store, WormAuditTables.VERIFICATION, new QQueryFilter(
|
||||
new QFilterCriteria("kind", QCriteriaOperator.EQUALS, "CHAIN"), new QFilterCriteria("trigger", QCriteriaOperator.EQUALS, "SCHEDULE"),
|
||||
new QFilterCriteria("outcome", QCriteriaOperator.EQUALS, "PASS")), "id");
|
||||
QRecord openBreak = failure != null && (fullPass == null || failure.getValueLong("id") > fullPass.getValueLong("id")) ? failure : null;
|
||||
QRecord export = store.last(WormAuditTables.EXPORT);
|
||||
QRecord anchor = store.anchor();
|
||||
long anchorId = anchor.getValueLong("anchor_link_id");
|
||||
long exported = export == null ? anchorId : export.getValueLong("to_link_id");
|
||||
long lag = store.count(WormAuditTables.CHAIN, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, exported)));
|
||||
List<String> gaps = gaps(store, anchorId);
|
||||
List<QRecord> oldest = store.summary("audit", new QQueryFilter().withOrderBy(new QFilterOrderBy("timestamp", true)).withLimit(1), Set.of("timestamp"));
|
||||
Instant oldestAt = oldest.isEmpty() ? null : oldest.getFirst().getValueInstant("timestamp");
|
||||
List<QRecord> purges = store.query(WormAuditTables.PURGE, new QQueryFilter()
|
||||
.withOrderBy(new QFilterOrderBy("requested_at", false)).withOrderBy(new QFilterOrderBy("id", false)).withLimit(10));
|
||||
warnings.addAll(PostgresSetupWarnings.load(transaction, config));
|
||||
transaction.commit();
|
||||
return (new WormAuditHomeData(links, lastLink == null ? null : lastLink.getValueInstant("chained_at"), backlog, settling,
|
||||
settings.settle_horizon_seconds(), verification, openBreak, export, lag,
|
||||
export == null || export.getValueInstant("sealed_at") == null ? null : Math.max(0, Duration.between(export.getValueInstant("sealed_at"), now).toHours()),
|
||||
List.copyOf(gaps), config.getExportBucketAlias(), capability, reason, settings.export_lock_mode(), settings.export_retention_days(), settings.audit_retention_years(), anchorId == 0 ? null : anchor,
|
||||
oldestAt, oldestAt == null ? null : oldestAt.atZone(ZoneOffset.UTC).plusYears(settings.audit_retention_years()).toInstant(), List.copyOf(purges), List.copyOf(warnings)));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
transaction.rollback();
|
||||
throw (new QException("Cannot load WORM audit home summary"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read one result, using id to break equal-time ties deterministically.
|
||||
*******************************************************************************/
|
||||
private static QRecord latest(AuditStore store, String table, QQueryFilter filter, String time) throws QException
|
||||
{
|
||||
List<QRecord> rows = store.query(table, filter.withOrderBy(new QFilterOrderBy(time, false))
|
||||
.withOrderBy(new QFilterOrderBy("id", false)).withLimit(1));
|
||||
return (rows.isEmpty() ? null : rows.getFirst());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Sweep export intervals in bounded pages; never enumerate links or objects.
|
||||
** The writer appends ranges in id order. Overlapping ranges extend coverage.
|
||||
*******************************************************************************/
|
||||
private static List<String> gaps(AuditStore store, long anchor) throws QException
|
||||
{
|
||||
List<String> gaps = new ArrayList<>();
|
||||
long next = anchor + 1;
|
||||
long cursor = 0;
|
||||
while(true)
|
||||
{
|
||||
List<QRecord> page = store.page(WormAuditTables.EXPORT, cursor, 1000);
|
||||
if(page.isEmpty())
|
||||
{
|
||||
return (gaps);
|
||||
}
|
||||
for(QRecord export : page)
|
||||
{
|
||||
long from = export.getValueLong("from_link_id");
|
||||
long to = export.getValueLong("to_link_id");
|
||||
if(from > next)
|
||||
{
|
||||
gaps.add(next + "–" + (from - 1));
|
||||
}
|
||||
next = Math.max(next, to + 1);
|
||||
}
|
||||
cursor = page.getLast().getValueLong("id");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.app;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stacked, wrapping definition lists. Colors follow BackupsHomeView exactly:
|
||||
** inherited text/link colors, translucent neutral borders, muted opacity. Only
|
||||
** allowlisted summary fields are rendered; no audit content is ever inspected.
|
||||
*******************************************************************************/
|
||||
public final class WormAuditHomeView
|
||||
{
|
||||
private static final DateTimeFormatter WHEN = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm 'UTC'").withZone(ZoneOffset.UTC);
|
||||
private static final String CARD = "border:1px solid rgba(127,127,127,.3);border-radius:8px;padding:1rem;min-width:0;overflow-wrap:anywhere;";
|
||||
private static final String LINK = "color:inherit;text-decoration:underline;text-underline-offset:2px;";
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private WormAuditHomeView()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Full home with explicit empty states and no fixed-width content.
|
||||
*******************************************************************************/
|
||||
public static String render(WormAuditHomeData data, String prefix)
|
||||
{
|
||||
StringBuilder html = new StringBuilder("<div class=\"worm-audit-home\" style=\"display:grid;gap:1rem;min-width:0;max-width:100%;\">");
|
||||
open(html, "Chain status");
|
||||
row(html, "Links chained", data.links());
|
||||
row(html, "Last link", when(data.lastLinkAt(), "No links chained yet"));
|
||||
row(html, "Unchained backlog", data.backlog() + " settled; " + data.settling() + " settling (" + data.settleSeconds() + " s horizon)");
|
||||
QRecord verification = data.lastVerification();
|
||||
if(verification == null)
|
||||
{
|
||||
row(html, "Last verification", "No verification has run yet");
|
||||
}
|
||||
else
|
||||
{
|
||||
linkedRow(html, "Last verification", recordLink(prefix, WormAuditTables.VERIFICATION, verification.getValueLong("id"), "Verification #" + verification.getValueLong("id")));
|
||||
row(html, "Verified at", when(verification.getValueInstant("finished_at"), "Not recorded"));
|
||||
row(html, "Result", verification.getValueString("outcome") + "; " + verification.getValueLong("links_checked") + " links checked");
|
||||
}
|
||||
QRecord broken = data.openBreak();
|
||||
if(broken == null)
|
||||
{
|
||||
row(html, "Open chain breaks", verification == null ? "Unknown until verification runs" : "None reported");
|
||||
}
|
||||
else
|
||||
{
|
||||
linkedRow(html, "Open chain breaks", recordLink(prefix, WormAuditTables.VERIFICATION, broken.getValueLong("id"), broken.getValueLong("break_count") + " reported"));
|
||||
linkedRow(html, "First break link", recordLink(prefix, WormAuditTables.CHAIN, broken.getValueLong("first_break_link_id"), "Link #" + broken.getValueLong("first_break_link_id")));
|
||||
row(html, "First break audit", broken.getValueLong("first_break_audit_id"));
|
||||
row(html, "Reason", broken.getValueString("first_break_reason"));
|
||||
}
|
||||
close(html);
|
||||
open(html, "Export status");
|
||||
row(html, "Export lock mode", data.exportLockMode());
|
||||
row(html, "Export retention", data.exportRetentionDays() + " days");
|
||||
QRecord export = data.lastExport();
|
||||
if(export == null)
|
||||
{
|
||||
row(html, "Last sealed export", "No sealed exports yet");
|
||||
}
|
||||
else
|
||||
{
|
||||
linkedRow(html, "Last sealed export", recordLink(prefix, WormAuditTables.EXPORT, export.getValueLong("id"), "Export #" + export.getValueLong("id")));
|
||||
row(html, "Link range", export.getValueLong("from_link_id") + "–" + export.getValueLong("to_link_id"));
|
||||
row(html, "Sealed at", when(export.getValueInstant("sealed_at"), "Not recorded"));
|
||||
row(html, "Retain until", when(export.getValueInstant("retain_until"), "Not recorded"));
|
||||
row(html, "Last export lock mode", export.getValueString("lock_mode"));
|
||||
}
|
||||
row(html, "Export lag", data.exportLag() + " links; " + (data.exportAgeHours() == null ? "no previous export time" : data.exportAgeHours() + " h since last export"));
|
||||
row(html, "Gaps in sealed ranges", data.exportGaps().isEmpty() ? (export == null ? "No sealed ranges to check" : "None") : String.join(", ", data.exportGaps()));
|
||||
row(html, "Bucket alias", data.bucketAlias());
|
||||
row(html, "Object Lock capability", data.lockCapability() + (data.lockReason() == null ? "" : ": " + data.lockReason()));
|
||||
close(html);
|
||||
open(html, "Retention");
|
||||
row(html, "Configured retention", data.retentionYears() + " years");
|
||||
QRecord anchor = data.anchor();
|
||||
row(html, "Purge anchor", anchor == null ? "None" : "Link #" + anchor.getValueLong("anchor_link_id") + "; hash " + shortHash(anchor.getValueString("anchor_hash")));
|
||||
row(html, "Oldest live audit", when(data.oldestAudit(), "No live audits"));
|
||||
row(html, "Next eligible purge", when(data.nextPurge(), "No live audits to purge"));
|
||||
close(html);
|
||||
open(html, "Recent purges");
|
||||
if(data.recentPurges().isEmpty())
|
||||
{
|
||||
row(html, "History", "No purge requests yet");
|
||||
}
|
||||
for(QRecord purge : data.recentPurges())
|
||||
{
|
||||
linkedRow(html, "Purge", recordLink(prefix, WormAuditTables.PURGE, purge.getValueLong("id"), "Purge #" + purge.getValueLong("id")));
|
||||
row(html, "Requested at", when(purge.getValueInstant("requested_at"), "Not recorded"));
|
||||
row(html, "Requested by", purge.getValueString("requested_by"));
|
||||
row(html, "Outcome", purge.getValueString("outcome"));
|
||||
row(html, "Cutoff", when(purge.getValueInstant("cutoff"), "Not recorded"));
|
||||
row(html, "Deleted", purge.getValueLong("audits_deleted") + " audits; " + purge.getValueLong("details_deleted") + " details; " + purge.getValueLong("links_deleted") + " links");
|
||||
row(html, "Reason", purge.getValueString("reason"));
|
||||
}
|
||||
close(html);
|
||||
if(!data.warnings().isEmpty())
|
||||
{
|
||||
open(html, "Setup warnings");
|
||||
for(String warning : data.warnings())
|
||||
{
|
||||
row(html, "Warning", warning);
|
||||
}
|
||||
close(html);
|
||||
}
|
||||
return (html.append("</div>").toString());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Honest compact status, owned by and linked to the full home.
|
||||
*******************************************************************************/
|
||||
public static String renderHealthCard(WormAuditHomeData data, String homePath)
|
||||
{
|
||||
String status = data.openBreak() != null ? "Chain breaks reported" : data.lastVerification() == null ? "Not verified yet"
|
||||
: data.warnings().isEmpty() ? "No open chain breaks reported" : "Setup needs attention";
|
||||
return ("<div style=\"" + CARD + "\"><strong>" + esc(status) + "</strong><p>" + data.links() + " links; " + data.backlog()
|
||||
+ " settled audits waiting; " + data.exportLag() + " links awaiting export.</p><a style=\"" + LINK + "\" href=\"" + esc(homePath) + "\">Open WORM Audit</a></div>");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Section and definition list, with fluid wrapping label/value rows.
|
||||
*******************************************************************************/
|
||||
private static void open(StringBuilder html, String title)
|
||||
{
|
||||
html.append("<section style=\"").append(CARD).append("\"><h3 style=\"margin:0 0 .75rem;font-size:1rem;font-weight:600;\">")
|
||||
.append(esc(title)).append("</h3><dl style=\"margin:0;font-size:.9rem;\">");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Close a section.
|
||||
*******************************************************************************/
|
||||
private static void close(StringBuilder html)
|
||||
{
|
||||
html.append("</dl></section>");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Escape every value, including reasons and runtime identities.
|
||||
*******************************************************************************/
|
||||
private static void row(StringBuilder html, String label, Object value)
|
||||
{
|
||||
linkedRow(html, label, esc(value == null ? "Not recorded" : value.toString()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Only internally built and escaped links may supply HTML here.
|
||||
*******************************************************************************/
|
||||
private static void linkedRow(StringBuilder html, String label, String value)
|
||||
{
|
||||
html.append("<div style=\"display:flex;flex-wrap:wrap;gap:.25rem .75rem;margin:.4rem 0;min-width:0;\"><dt style=\"opacity:.75;flex:0 1 12rem;\">")
|
||||
.append(esc(label)).append("</dt><dd style=\"margin:0;flex:1 1 14rem;min-width:0;overflow-wrap:anywhere;\">").append(value).append("</dd></div>");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Standard QQQ record route; identifiers are numeric, never content.
|
||||
*******************************************************************************/
|
||||
private static String recordLink(String prefix, String table, Long id, String text)
|
||||
{
|
||||
return (id == null ? "Not recorded" : "<a style=\"" + LINK + "\" href=\"" + esc(prefix + "/" + table + "/" + id) + "\">" + esc(text) + "</a>");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** UTC time with an explicit empty state.
|
||||
*******************************************************************************/
|
||||
private static String when(Instant time, String empty)
|
||||
{
|
||||
return (time == null ? empty : WHEN.format(time));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Keep hashes compact even on a narrow screen.
|
||||
*******************************************************************************/
|
||||
private static String shortHash(String hash)
|
||||
{
|
||||
return (hash == null ? "Not recorded" : hash.substring(0, Math.min(12, hash.length())) + "…");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** HTML text and quoted attribute escaping.
|
||||
*******************************************************************************/
|
||||
private static String esc(String text)
|
||||
{
|
||||
return (text == null ? "" : text.replace("&", "&").replace("<", "<").replace(">", ">").replace("\"", """).replace("'", "'"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.canonical;
|
||||
|
||||
|
||||
import java.time.ZoneOffset;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Comparator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** v1 audit content, exactly as stored. Subsets use the same document schema for
|
||||
** LATE_DETAIL links; no canonical content is persisted by this qbit.
|
||||
*******************************************************************************/
|
||||
public final class AuditCanonical
|
||||
{
|
||||
private static final DateTimeFormatter TIMESTAMP = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'").withZone(ZoneOffset.UTC);
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private AuditCanonical()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve names outside this function, without looking up application data.
|
||||
*******************************************************************************/
|
||||
public static byte[] bytes(QRecord audit, String table, String user, List<QRecord> details)
|
||||
{
|
||||
Map<String, Object> header = new LinkedHashMap<>();
|
||||
header.put("clientChannel", audit.getValueString("clientChannel"));
|
||||
header.put("id", audit.getValueLong("id"));
|
||||
header.put("message", audit.getValueString("message"));
|
||||
header.put("recordId", audit.getValueString("recordId"));
|
||||
header.put("table", table);
|
||||
header.put("timestamp", TIMESTAMP.format(audit.getValueInstant("timestamp")));
|
||||
header.put("user", user);
|
||||
List<Map<String, Object>> rows = details.stream().sorted(Comparator.comparing(r -> r.getValueLong("id"))).map(detail ->
|
||||
{
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
row.put("id", detail.getValueLong("id"));
|
||||
for(String field : List.of("fieldName", "message", "newValue", "oldValue"))
|
||||
{
|
||||
row.put(field, detail.getValueString(field));
|
||||
}
|
||||
return (row);
|
||||
}).toList();
|
||||
return (Jcs.bytes(Map.of("audit", header, "details", rows, "v", 1)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.canonical;
|
||||
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** RFC 8785 serialization of the audit schema: strings, integer longs, null,
|
||||
** objects and arrays. Java String ordering is UTF-16 code-unit ordering. Reject
|
||||
** malformed surrogate pairs instead of silently replacing them during UTF-8 encoding.
|
||||
*******************************************************************************/
|
||||
public final class Jcs
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private Jcs()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Encode a document with no insignificant whitespace.
|
||||
*******************************************************************************/
|
||||
public static byte[] bytes(Object value)
|
||||
{
|
||||
return (string(value).getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Serialize only the value types supported by the v1 schema.
|
||||
*******************************************************************************/
|
||||
public static String string(Object value)
|
||||
{
|
||||
if(value == null)
|
||||
{
|
||||
return ("null");
|
||||
}
|
||||
if(value instanceof String text)
|
||||
{
|
||||
return (quote(text));
|
||||
}
|
||||
if(value instanceof Long || value instanceof Integer)
|
||||
{
|
||||
return (value.toString());
|
||||
}
|
||||
if(value instanceof List<?> list)
|
||||
{
|
||||
return ("[" + String.join(",", list.stream().map(Jcs::string).toList()) + "]");
|
||||
}
|
||||
if(value instanceof Map<?, ?> map)
|
||||
{
|
||||
TreeMap<String, Object> sorted = new TreeMap<>();
|
||||
map.forEach((key, item) -> sorted.put((String) key, item));
|
||||
return ("{" + String.join(",", sorted.entrySet().stream().map(e -> quote(e.getKey()) + ":" + string(e.getValue())).toList()) + "}");
|
||||
}
|
||||
throw (new IllegalArgumentException("Unsupported canonical value type"));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** JSON escaping mandated by JCS, including lowercase control escapes.
|
||||
*******************************************************************************/
|
||||
private static String quote(String text)
|
||||
{
|
||||
StringBuilder out = new StringBuilder("\"");
|
||||
for(int i = 0; i < text.length(); i++)
|
||||
{
|
||||
char c = text.charAt(i);
|
||||
switch(c)
|
||||
{
|
||||
case '"' -> out.append("\\\"");
|
||||
case '\\' -> out.append("\\\\");
|
||||
case '\b' -> out.append("\\b");
|
||||
case '\f' -> out.append("\\f");
|
||||
case '\n' -> out.append("\\n");
|
||||
case '\r' -> out.append("\\r");
|
||||
case '\t' -> out.append("\\t");
|
||||
default ->
|
||||
{
|
||||
if(c < 32)
|
||||
{
|
||||
out.append(String.format(java.util.Locale.ROOT, "\\u%04x", (int) c));
|
||||
}
|
||||
else if(Character.isHighSurrogate(c))
|
||||
{
|
||||
if(i + 1 >= text.length() || !Character.isLowSurrogate(text.charAt(i + 1)))
|
||||
{
|
||||
throw (new IllegalArgumentException("Unpaired surrogate in canonical string"));
|
||||
}
|
||||
out.append(c).append(text.charAt(++i));
|
||||
}
|
||||
else if(Character.isLowSurrogate(c))
|
||||
{
|
||||
throw (new IllegalArgumentException("Unpaired surrogate in canonical string"));
|
||||
}
|
||||
else
|
||||
{
|
||||
out.append(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return (out.append('"').toString());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,540 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.chain;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.Timestamp;
|
||||
import java.sql.Types;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import com.kingsrook.qbits.wormaudit.canonical.AuditCanonical;
|
||||
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
|
||||
import com.kingsrook.qbits.wormaudit.guard.AppendOnlyGuard;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.VerificationResult;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSTransaction;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** QQQ storage shared by chainer, verifier and future export code. All reads use
|
||||
** the caller's transaction; only insert operations exist here. Bypass applies
|
||||
** to internal catalog work after process permission checks, never user requests.
|
||||
*******************************************************************************/
|
||||
public class AuditStore
|
||||
{
|
||||
private final QBackendTransaction transaction;
|
||||
private final Map<String, String> names = new LinkedHashMap<>(16, 0.75F, true);
|
||||
private Boolean timestampWithZone;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Bind operations to one snapshot/transaction, or the memory backend.
|
||||
*******************************************************************************/
|
||||
public AuditStore(QBackendTransaction transaction)
|
||||
{
|
||||
this.transaction = transaction;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read in deterministic id order, explicitly fetching TEXT fields.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> query(String table, QQueryFilter filter) throws QException
|
||||
{
|
||||
QueryInput input = new QueryInput(table).withFilter(filter).withTransaction(transaction).withShouldFetchHeavyFields(true);
|
||||
input.withPermissionBypass("WORM audit internal read");
|
||||
return (new QueryAction().execute(input).getRecords());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Project only summary fields; home reads never fetch audit content.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> summary(String table, QQueryFilter filter, Set<String> fields) throws QException
|
||||
{
|
||||
QueryInput input = new QueryInput(table).withFilter(filter).withTransaction(transaction)
|
||||
.withFieldNamesToInclude(fields).withShouldFetchHeavyFields(true);
|
||||
input.withPermissionBypass("WORM audit home summary after widget permission check");
|
||||
return (new QueryAction().execute(input).getRecords());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Aggregate counts avoid loading the live chain for the home.
|
||||
*******************************************************************************/
|
||||
public long count(String table, QQueryFilter filter) throws QException
|
||||
{
|
||||
CountInput input = new CountInput(table).withFilter(filter).withTransaction(transaction);
|
||||
input.withPermissionBypass("WORM audit home count after widget permission check");
|
||||
return (new CountAction().execute(input).getCount());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Count the same AUDIT anti-join as the writer, split at the settle horizon.
|
||||
*******************************************************************************/
|
||||
public long unchainedCount(Instant cutoff, boolean settled) throws QException
|
||||
{
|
||||
if(isRdbms())
|
||||
{
|
||||
String sql = "SELECT count(*) FROM " + physical("audit") + " a WHERE a." + column("audit", "timestamp")
|
||||
+ (settled ? " < ?" : " >= ?") + unchainedPredicate();
|
||||
return (sqlIds(sql, cutoff).getFirst());
|
||||
}
|
||||
Set<Long> covered = summary(WormAuditTables.CHAIN, new QQueryFilter(new QFilterCriteria("kind", QCriteriaOperator.EQUALS, "AUDIT")), Set.of("audit_id"))
|
||||
.stream().map(r -> r.getValueLong("audit_id")).collect(Collectors.toSet());
|
||||
return (summary("audit", new QQueryFilter(), Set.of("id", "timestamp")).stream()
|
||||
.filter(r -> r.getValueInstant("timestamp") != null && r.getValueInstant("timestamp").isBefore(cutoff) == settled
|
||||
&& !covered.contains(r.getValueLong("id"))).count());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Shared SQL coverage predicate for both writer selection and home counts.
|
||||
*******************************************************************************/
|
||||
private String unchainedPredicate()
|
||||
{
|
||||
return (" AND NOT EXISTS (SELECT 1 FROM worm_audit_chain c WHERE c.audit_id=a." + column("audit", "id") + " AND c.kind='AUDIT')");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** All records ordered by id.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> all(String table) throws QException
|
||||
{
|
||||
return (query(table, new QQueryFilter().withOrderBy(new QFilterOrderBy("id", true))));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** An equality query.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> where(String table, String field, Serializable value) throws QException
|
||||
{
|
||||
// Core lookup keys are INTEGER while audit and chain keys are LONG.
|
||||
// Memory equality is type-sensitive; normalize to the declared field type.
|
||||
if(value instanceof Number number && QContext.getQInstance().getTable(table).getField(field).getType()
|
||||
== com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType.INTEGER)
|
||||
{
|
||||
value = Math.toIntExact(number.longValue());
|
||||
}
|
||||
return (query(table, new QQueryFilter(new QFilterCriteria(field, QCriteriaOperator.EQUALS, value)).withOrderBy(new QFilterOrderBy("id", true))));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Return a row or null if a tamper removed it.
|
||||
*******************************************************************************/
|
||||
public QRecord get(String table, long id) throws QException
|
||||
{
|
||||
return (where(table, "id", id).stream().findFirst().orElse(null));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Insert a batch, propagating failures so a transaction can roll back.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> insert(String table, List<QRecord> rows) throws QException
|
||||
{
|
||||
if(rows.isEmpty())
|
||||
{
|
||||
return (rows);
|
||||
}
|
||||
try
|
||||
{
|
||||
if(WormAuditTables.ALL.contains(table) && QContext.getQInstance().getBackendForTable(table)
|
||||
instanceof com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData)
|
||||
{
|
||||
// RC.5 binds Instant as a UTC LocalDateTime, which shifts timestamptz
|
||||
// in non-UTC sessions. Its Timestamp path preserves the actual instant.
|
||||
rows.forEach(row -> row.getValues().replaceAll((field, value) -> value instanceof Instant instant ? Timestamp.from(instant) : value));
|
||||
}
|
||||
InsertInput input = new InsertInput(table).withRecords(rows).withTransaction(transaction).withOmitDmlAudit(true);
|
||||
input.withPermissionBypass("WORM audit internal append");
|
||||
List<QRecord> inserted = new InsertAction().execute(input).getRecords();
|
||||
for(QRecord row : inserted)
|
||||
{
|
||||
if(row.getErrors() != null && !row.getErrors().isEmpty())
|
||||
{
|
||||
// Never echo a rejected record value into logs or alerts.
|
||||
throw (new QException("WORM audit append failed for " + table));
|
||||
}
|
||||
}
|
||||
return (inserted);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw (AppendOnlyGuard.map(e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Parse explicit coverage; duplicates are retained so verification sees them.
|
||||
*******************************************************************************/
|
||||
public static List<Long> detailIds(QRecord link)
|
||||
{
|
||||
String ids = link.getValueString("detail_ids");
|
||||
return (ids == null || ids.isEmpty() ? List.of() : java.util.Arrays.stream(ids.split(",")).map(Long::valueOf).toList());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve a lookup name without touching application tables.
|
||||
*******************************************************************************/
|
||||
public String name(String table, Long id) throws QException
|
||||
{
|
||||
if(id == null)
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
String key = table + ":" + id;
|
||||
if(names.containsKey(key))
|
||||
{
|
||||
return (names.get(key));
|
||||
}
|
||||
QRecord row = get(table, id);
|
||||
if(row == null)
|
||||
{
|
||||
throw (new QException("Missing " + table + " " + id));
|
||||
}
|
||||
if(names.size() >= 2000)
|
||||
{
|
||||
names.remove(names.keySet().iterator().next());
|
||||
}
|
||||
names.put(key, row.getValueString("name"));
|
||||
return (row.getValueString("name"));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Canonical bytes for the exact detail ids recorded on a link.
|
||||
*******************************************************************************/
|
||||
public byte[] canonical(QRecord audit, List<Long> ids) throws QException
|
||||
{
|
||||
return (canonical(audit, ids, index(in("auditDetail", "id", ids))));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Settled unchained headers. RDBMS uses an anti-join under the writer lock;
|
||||
** memory uses QQQ queries and the same ordering and cutoff semantics.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> unchained(Instant cutoff, int limit) throws QException
|
||||
{
|
||||
if(isRdbms())
|
||||
{
|
||||
String sql = "SELECT a." + column("audit", "id") + " FROM " + physical("audit") + " a WHERE a." + column("audit", "timestamp")
|
||||
+ " < ?" + unchainedPredicate() + " ORDER BY a." + column("audit", "timestamp") + ", a." + column("audit", "id") + " LIMIT " + limit;
|
||||
Map<Long, QRecord> rows = index(in("audit", "id", sqlIds(sql, cutoff)));
|
||||
return (rows.values().stream().sorted(Comparator.comparing((QRecord r) -> r.getValueInstant("timestamp"))
|
||||
.thenComparing(r -> r.getValueLong("id"))).toList());
|
||||
}
|
||||
var covered = all(WormAuditTables.CHAIN).stream().filter(r -> "AUDIT".equals(r.getValueString("kind"))).map(r -> r.getValueLong("audit_id")).collect(java.util.stream.Collectors.toSet());
|
||||
return (all("audit").stream().filter(r -> r.getValueInstant("timestamp").isBefore(cutoff) && !covered.contains(r.getValueLong("id")))
|
||||
.sorted(Comparator.comparing((QRecord r) -> r.getValueInstant("timestamp")).thenComparing(r -> r.getValueLong("id"))).limit(limit).toList());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Latest successful retention anchor, or genesis.
|
||||
*******************************************************************************/
|
||||
public QRecord anchor() throws QException
|
||||
{
|
||||
List<QRecord> rows = query(WormAuditTables.PURGE, new QQueryFilter(new QFilterCriteria("outcome", QCriteriaOperator.EQUALS, "PURGED"))
|
||||
.withOrderBy(new QFilterOrderBy("id", false)).withLimit(1));
|
||||
return (rows.isEmpty() ? new QRecord().withValue("anchor_link_id", 0L).withValue("anchor_hash", Hashes.GENESIS) : rows.getFirst());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether SQL anti-joins are available in this snapshot.
|
||||
*******************************************************************************/
|
||||
public boolean isRdbms()
|
||||
{
|
||||
return (transaction instanceof RDBMSTransaction);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Only the in-memory backend may use an unbounded completeness index.
|
||||
*******************************************************************************/
|
||||
public boolean isMemory()
|
||||
{
|
||||
return ("memory".equals(QContext.getQInstance().getBackendForTable(WormAuditTables.CHAIN).getBackendType()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fetch one keyset page, never using an offset or a whole-chain read.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> page(String table, long after, int limit) throws QException
|
||||
{
|
||||
return (query(table, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, after))
|
||||
.withOrderBy(new QFilterOrderBy("id", true)).withLimit(limit)));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fetch just the tail row.
|
||||
*******************************************************************************/
|
||||
public QRecord last(String table) throws QException
|
||||
{
|
||||
List<QRecord> rows = query(table, new QQueryFilter().withOrderBy(new QFilterOrderBy("id", false)).withLimit(1));
|
||||
return (rows.isEmpty() ? null : rows.getFirst());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One batched IN query, with memory-backend integer key normalization.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> in(String table, String field, Collection<Long> ids) throws QException
|
||||
{
|
||||
if(ids.isEmpty())
|
||||
{
|
||||
return (List.of());
|
||||
}
|
||||
boolean integer = QContext.getQInstance().getTable(table).getField(field).getType()
|
||||
== com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType.INTEGER;
|
||||
List<Serializable> values = ids.stream().distinct().map(id -> integer ? (Serializable) Math.toIntExact(id) : id).toList();
|
||||
return (query(table, new QQueryFilter(new QFilterCriteria(field, QCriteriaOperator.IN, values)).withOrderBy(new QFilterOrderBy("id", true))));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Index one bounded batch by primary key.
|
||||
*******************************************************************************/
|
||||
public static Map<Long, QRecord> index(List<QRecord> rows)
|
||||
{
|
||||
return (rows.stream().collect(Collectors.toMap(r -> r.getValueLong("id"), r -> r)));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Canonicalize already fetched content; a missing detail never triggers a query.
|
||||
*******************************************************************************/
|
||||
public byte[] canonical(QRecord audit, List<Long> ids, Map<Long, QRecord> details) throws QException
|
||||
{
|
||||
List<QRecord> rows = new ArrayList<>();
|
||||
for(long id : ids)
|
||||
{
|
||||
QRecord detail = details.get(id);
|
||||
if(detail == null || !audit.getValueLong("id").equals(detail.getValueLong("auditId")))
|
||||
{
|
||||
throw (new QException("audit_detail " + id + " is missing or belongs to another audit"));
|
||||
}
|
||||
rows.add(detail);
|
||||
}
|
||||
return (AuditCanonical.bytes(audit, name("auditTable", audit.getValueLong("auditTableId")), name("auditUser", audit.getValueLong("auditUserId")), rows));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Poll uncovered details in a bounded time window; memory retains its simple fallback.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> lateDetails(Instant cutoff, int hours, int limit) throws QException
|
||||
{
|
||||
if(isRdbms())
|
||||
{
|
||||
String sql = "SELECT d." + column("auditDetail", "id") + " FROM " + physical("auditDetail") + " d JOIN " + physical("audit")
|
||||
+ " a ON a." + column("audit", "id") + "=d." + column("auditDetail", "auditId")
|
||||
+ " WHERE a." + column("audit", "timestamp") + " < ? AND a." + column("audit", "timestamp") + " >= ?"
|
||||
+ " AND EXISTS (SELECT 1 FROM worm_audit_chain h WHERE h.kind='AUDIT' AND h.audit_id=a." + column("audit", "id") + ")"
|
||||
+ " AND NOT EXISTS (SELECT 1 FROM worm_audit_chain c WHERE " + containsDetail() + ") ORDER BY d." + column("auditDetail", "id") + " LIMIT " + limit;
|
||||
return (in("auditDetail", "id", sqlIds(sql, cutoff, cutoff.minusSeconds(hours * 3600L))));
|
||||
}
|
||||
List<QRecord> links = all(WormAuditTables.CHAIN);
|
||||
Set<Long> headers = links.stream().filter(r -> "AUDIT".equals(r.getValueString("kind"))).map(r -> r.getValueLong("audit_id")).collect(Collectors.toSet());
|
||||
Set<Long> covered = new HashSet<>();
|
||||
links.forEach(link -> covered.addAll(detailIds(link)));
|
||||
Set<Long> settled = all("audit").stream().filter(a -> headers.contains(a.getValueLong("id")) && a.getValueInstant("timestamp").isBefore(cutoff))
|
||||
.map(a -> a.getValueLong("id")).collect(Collectors.toSet());
|
||||
return (all("auditDetail").stream().filter(d -> settled.contains(d.getValueLong("auditId")) && !covered.contains(d.getValueLong("id"))).toList());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** SQL completeness counts all offenders but returns only the first 100 of each type.
|
||||
*******************************************************************************/
|
||||
public void completeness(Instant cutoff, VerificationResult result) throws QException
|
||||
{
|
||||
String auditId = "a." + column("audit", "id");
|
||||
String time = "a." + column("audit", "timestamp");
|
||||
String headers = "SELECT " + auditId + " AS id, (SELECT count(*) FROM worm_audit_chain c WHERE c.kind='AUDIT' AND c.audit_id=" + auditId
|
||||
+ ") AS coverage FROM " + physical("audit") + " a WHERE " + time + " < ? OR " + time + " IS NULL";
|
||||
String details = "SELECT d." + column("auditDetail", "id") + " AS id, (SELECT count(*) FROM worm_audit_chain c WHERE " + containsDetail()
|
||||
+ ") AS coverage FROM " + physical("auditDetail") + " d LEFT JOIN " + physical("audit") + " a ON " + auditId + "=d."
|
||||
+ column("auditDetail", "auditId") + " WHERE " + time + " < ? OR " + time + " IS NULL";
|
||||
reportCoverage(headers, "audit", cutoff, result);
|
||||
reportCoverage(details, "audit_detail", cutoff, result);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Keep exact offender ids and total counts without materializing the result set.
|
||||
*******************************************************************************/
|
||||
private void reportCoverage(String source, String table, Instant cutoff, VerificationResult result) throws QException
|
||||
{
|
||||
String sql = "SELECT id, coverage, count(*) OVER() AS total FROM (" + source + ") offenders WHERE coverage <> 1 ORDER BY id LIMIT 100";
|
||||
try(PreparedStatement statement = ((RDBMSTransaction) transaction).getConnection().prepareStatement(sql))
|
||||
{
|
||||
bindCutoff(statement, 1, cutoff);
|
||||
long total = 0;
|
||||
int reported = 0;
|
||||
try(ResultSet rows = statement.executeQuery())
|
||||
{
|
||||
while(rows.next())
|
||||
{
|
||||
long id = rows.getLong(1);
|
||||
total = rows.getLong(3);
|
||||
result.add(null, table.equals("audit") ? id : null, table + " " + com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier.number(id)
|
||||
+ (rows.getLong(2) == 0 ? " is not covered by the chain" : " is covered twice or more by the chain"));
|
||||
reported++;
|
||||
}
|
||||
}
|
||||
result.omitted(total - reported);
|
||||
if(total > reported)
|
||||
{
|
||||
result.warn(table + ": " + total + " coverage offenders; first 100 ids reported");
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw (AppendOnlyGuard.map(e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** PostgreSQL's containment expression matches the GIN expression index.
|
||||
*******************************************************************************/
|
||||
private String containsDetail() throws QException
|
||||
{
|
||||
try
|
||||
{
|
||||
String id = "d." + column("auditDetail", "id");
|
||||
if("PostgreSQL".equals(((RDBMSTransaction) transaction).getConnection().getMetaData().getDatabaseProductName()))
|
||||
{
|
||||
return ("string_to_array(c.detail_ids, ',') @> ARRAY[" + id + "::text]");
|
||||
}
|
||||
return ("POSITION(',' || CAST(" + id + " AS varchar) || ',' IN ',' || c.detail_ids || ',') > 0");
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw (AppendOnlyGuard.map(e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read ids from an anti-join, binding all instants according to column type.
|
||||
*******************************************************************************/
|
||||
private List<Long> sqlIds(String sql, Instant... cutoffs) throws QException
|
||||
{
|
||||
try(PreparedStatement statement = ((RDBMSTransaction) transaction).getConnection().prepareStatement(sql))
|
||||
{
|
||||
for(int i = 0; i < cutoffs.length; i++)
|
||||
{
|
||||
bindCutoff(statement, i + 1, cutoffs[i]);
|
||||
}
|
||||
List<Long> ids = new ArrayList<>();
|
||||
try(ResultSet rows = statement.executeQuery())
|
||||
{
|
||||
while(rows.next())
|
||||
{
|
||||
ids.add(rows.getLong(1));
|
||||
}
|
||||
}
|
||||
return (ids);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw (AppendOnlyGuard.map(e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** UTC local timestamps for without-tz columns; offset timestamps for timestamptz.
|
||||
*******************************************************************************/
|
||||
private void bindCutoff(PreparedStatement statement, int index, Instant cutoff) throws Exception
|
||||
{
|
||||
if(timestampWithZone == null)
|
||||
{
|
||||
try(var probe = ((RDBMSTransaction) transaction).getConnection().createStatement();
|
||||
var rows = probe.executeQuery("SELECT " + column("audit", "timestamp") + " FROM " + physical("audit") + " WHERE 1=0"))
|
||||
{
|
||||
var metadata = rows.getMetaData();
|
||||
timestampWithZone = metadata.getColumnType(1) == Types.TIMESTAMP_WITH_TIMEZONE
|
||||
|| "timestamptz".equalsIgnoreCase(metadata.getColumnTypeName(1));
|
||||
}
|
||||
}
|
||||
statement.setObject(index, timestampWithZone ? cutoff.atOffset(ZoneOffset.UTC) : LocalDateTime.ofInstant(cutoff, ZoneOffset.UTC));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve and quote physical core table names.
|
||||
*******************************************************************************/
|
||||
private String physical(String table)
|
||||
{
|
||||
return (WormAuditDdl.identifier(((RDBMSTableBackendDetails) QContext.getQInstance().getTable(table).getBackendDetails()).getTableName()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve and quote physical field names for runtime queries.
|
||||
*******************************************************************************/
|
||||
private String column(String table, String field)
|
||||
{
|
||||
String name = QContext.getQInstance().getTable(table).getField(field).getBackendName();
|
||||
return (WormAuditDdl.identifier(name == null ? field : name));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.chain;
|
||||
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HexFormat;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** SHA-256 primitives. Link predecessors are the 64 lowercase ASCII hex bytes;
|
||||
** Merkle leaves are the decoded 32-byte link hashes, with odd nodes promoted.
|
||||
*******************************************************************************/
|
||||
public final class Hashes
|
||||
{
|
||||
public static final String GENESIS = "0".repeat(64);
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private Hashes()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** SHA-256 digest of concatenated byte arrays.
|
||||
*******************************************************************************/
|
||||
public static byte[] digest(byte[]... parts)
|
||||
{
|
||||
try
|
||||
{
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
for(byte[] part : parts)
|
||||
{
|
||||
digest.update(part);
|
||||
}
|
||||
return (digest.digest());
|
||||
}
|
||||
catch(NoSuchAlgorithmException e)
|
||||
{
|
||||
throw (new IllegalStateException("JDK lacks SHA-256", e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Link hash over predecessor, NUL delimiter and canonical bytes.
|
||||
*******************************************************************************/
|
||||
public static String link(String previous, byte[] canonical)
|
||||
{
|
||||
return (HexFormat.of().formatHex(digest(previous.getBytes(StandardCharsets.US_ASCII), new byte[] { 0 }, canonical)));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Binary Merkle root; an empty range cannot be checkpointed.
|
||||
*******************************************************************************/
|
||||
public static String merkle(List<String> hashes)
|
||||
{
|
||||
if(hashes.isEmpty())
|
||||
{
|
||||
throw (new IllegalArgumentException("A Merkle range must contain links"));
|
||||
}
|
||||
List<byte[]> level = hashes.stream().map(HexFormat.of()::parseHex).toList();
|
||||
while(level.size() > 1)
|
||||
{
|
||||
List<byte[]> next = new ArrayList<>();
|
||||
for(int i = 0; i < level.size(); i += 2)
|
||||
{
|
||||
next.add(i + 1 == level.size() ? level.get(i) : digest(level.get(i), level.get(i + 1)));
|
||||
}
|
||||
level = next;
|
||||
}
|
||||
return (HexFormat.of().formatHex(level.getFirst()));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.chain;
|
||||
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.Statement;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import java.util.stream.Collectors;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditSettings;
|
||||
import com.kingsrook.qbits.wormaudit.checkpoint.Checkpoints;
|
||||
import com.kingsrook.qbits.wormaudit.guard.AppendOnlyGuard;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSTransaction;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Single asynchronous writer. PostgreSQL uses a transaction-scoped advisory
|
||||
** lock shared with the retention function. Other backends use a JVM-wide lock:
|
||||
** those deployments must run their chainer in one JVM only.
|
||||
*******************************************************************************/
|
||||
public class WormAuditChainer
|
||||
{
|
||||
public static final long ADVISORY_LOCK = 86753090421L;
|
||||
public static final ReentrantLock JVM_LOCK = new ReentrantLock();
|
||||
private final WormAuditQBitConfig config;
|
||||
|
||||
/*******************************************************************************
|
||||
** Counts describe committed work; a contended writer tick is skipped.
|
||||
*******************************************************************************/
|
||||
public record Counts(int audits, int lateDetails, int links, int checkpoints, boolean skipped)
|
||||
{
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Bind host configuration.
|
||||
*******************************************************************************/
|
||||
public WormAuditChainer(WormAuditQBitConfig config)
|
||||
{
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Run a writer tick under one transaction; link ids cannot gap on rollback.
|
||||
*******************************************************************************/
|
||||
public Counts chainOnce() throws QException
|
||||
{
|
||||
return (chainOnce(Instant.now()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Clock-injected tick for deterministic settling and checkpoint tests.
|
||||
*******************************************************************************/
|
||||
public Counts chainOnce(Instant now) throws QException
|
||||
{
|
||||
config.validate();
|
||||
JVM_LOCK.lock();
|
||||
try(QBackendTransaction transaction = QBackendTransaction.openFor(new QueryInput(WormAuditTables.CHAIN)))
|
||||
{
|
||||
try
|
||||
{
|
||||
if(transaction instanceof RDBMSTransaction rdbms && "PostgreSQL".equals(rdbms.getConnection().getMetaData().getDatabaseProductName()))
|
||||
{
|
||||
try(Statement statement = rdbms.getConnection().createStatement(); ResultSet result = statement.executeQuery("SELECT pg_try_advisory_xact_lock(" + ADVISORY_LOCK + ")"))
|
||||
{
|
||||
result.next();
|
||||
if(!result.getBoolean(1))
|
||||
{
|
||||
return (new Counts(0, 0, 0, 0, true));
|
||||
}
|
||||
}
|
||||
}
|
||||
Counts counts = append(new AuditStore(transaction), now, config.effectiveSettings());
|
||||
transaction.commit();
|
||||
return (counts);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
transaction.rollback();
|
||||
throw (AppendOnlyGuard.map(e));
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
JVM_LOCK.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Append settled headers and uncovered detail chunks, then checkpoint.
|
||||
*******************************************************************************/
|
||||
private Counts append(AuditStore store, Instant now, WormAuditSettings.Values settings) throws Exception
|
||||
{
|
||||
QRecord tail = store.last(WormAuditTables.CHAIN);
|
||||
QRecord anchor = store.anchor();
|
||||
long nextId = tail == null ? anchor.getValueLong("anchor_link_id") + 1 : tail.getValueLong("id") + 1;
|
||||
String previous = tail == null ? anchor.getValueString("anchor_hash") : tail.getValueString("link_hash");
|
||||
List<QRecord> appended = new ArrayList<>();
|
||||
Instant cutoff = now.minusSeconds(settings.settle_horizon_seconds());
|
||||
List<QRecord> audits = store.unchained(cutoff, config.getBatchSize());
|
||||
Map<Long, QRecord> details = AuditStore.index(store.in("auditDetail", "auditId", audits.stream().map(r -> r.getValueLong("id")).toList()));
|
||||
Map<Long, List<QRecord>> grouped = details.values().stream().collect(Collectors.groupingBy(r -> r.getValueLong("auditId")));
|
||||
int lateDetails = 0;
|
||||
for(QRecord audit : audits)
|
||||
{
|
||||
List<Long> ids = grouped.getOrDefault(audit.getValueLong("id"), List.of()).stream().map(r -> r.getValueLong("id")).sorted().toList();
|
||||
for(int start = 0; start == 0 || start < ids.size(); start += 250)
|
||||
{
|
||||
List<Long> chunk = ids.subList(start, Math.min(start + 250, ids.size()));
|
||||
QRecord link = link(store, nextId++, start == 0 ? "AUDIT" : "LATE_DETAIL", audit, chunk, details, previous, now);
|
||||
appended.add(link);
|
||||
previous = link.getValueString("link_hash");
|
||||
if(start > 0)
|
||||
{
|
||||
lateDetails += chunk.size();
|
||||
}
|
||||
}
|
||||
}
|
||||
insertLinks(store, appended);
|
||||
int linkCount = appended.size();
|
||||
appended.clear();
|
||||
details = AuditStore.index(store.lateDetails(cutoff, settings.late_detail_lookback_hours(), config.getBatchSize()));
|
||||
grouped = details.values().stream().collect(Collectors.groupingBy(r -> r.getValueLong("auditId"), TreeMap::new, Collectors.toList()));
|
||||
Map<Long, QRecord> headers = AuditStore.index(store.in("audit", "id", grouped.keySet()));
|
||||
for(var entry : grouped.entrySet())
|
||||
{
|
||||
List<Long> ids = entry.getValue().stream().map(r -> r.getValueLong("id")).sorted().toList();
|
||||
for(int start = 0; start < ids.size(); start += 250)
|
||||
{
|
||||
List<Long> chunk = ids.subList(start, Math.min(start + 250, ids.size()));
|
||||
QRecord link = link(store, nextId++, "LATE_DETAIL", headers.get(entry.getKey()), chunk, details, previous, now);
|
||||
appended.add(link);
|
||||
previous = link.getValueString("link_hash");
|
||||
lateDetails += chunk.size();
|
||||
}
|
||||
}
|
||||
insertLinks(store, appended);
|
||||
return (new Counts(audits.size(), lateDetails, linkCount + appended.size(), checkpoint(store, now, settings), false));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Persist bounded insert batches inside the writer transaction.
|
||||
*******************************************************************************/
|
||||
private void insertLinks(AuditStore store, List<QRecord> links) throws QException
|
||||
{
|
||||
for(int start = 0; start < links.size(); start += config.getBatchSize())
|
||||
{
|
||||
store.insert(WormAuditTables.CHAIN, links.subList(start, Math.min(start + config.getBatchSize(), links.size())));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build one immutable link from live stored content.
|
||||
*******************************************************************************/
|
||||
private QRecord link(AuditStore store, long id, String kind, QRecord audit, List<Long> details, Map<Long, QRecord> rows, String previous, Instant now) throws QException
|
||||
{
|
||||
return (new QRecord().withValue("id", id).withValue("kind", kind).withValue("audit_id", audit.getValueLong("id"))
|
||||
.withValue("detail_ids", details.stream().map(Object::toString).collect(Collectors.joining(",")))
|
||||
.withValue("prev_hash", previous).withValue("link_hash", Hashes.link(previous, store.canonical(audit, details, rows))).withValue("chained_at", now));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Checkpoint the uncheckpointed suffix by count or elapsed time.
|
||||
*******************************************************************************/
|
||||
private int checkpoint(AuditStore store, Instant now, WormAuditSettings.Values settings) throws Exception
|
||||
{
|
||||
if(config.getSigningKey() == null)
|
||||
{
|
||||
return (0);
|
||||
}
|
||||
QRecord latest = store.last(WormAuditTables.CHECKPOINT);
|
||||
long lastId = latest == null ? store.anchor().getValueLong("anchor_link_id") : latest.getValueLong("to_link_id");
|
||||
int count = 0;
|
||||
while(true)
|
||||
{
|
||||
List<QRecord> pending = store.page(WormAuditTables.CHAIN, lastId, settings.checkpoint_every_links());
|
||||
if(pending.isEmpty())
|
||||
{
|
||||
return (count);
|
||||
}
|
||||
Instant since = latest == null ? pending.getFirst().getValueInstant("chained_at") : latest.getValueInstant("created_at");
|
||||
if(pending.size() < settings.checkpoint_every_links() && now.isBefore(since.plusSeconds(settings.checkpoint_every_minutes() * 60L)))
|
||||
{
|
||||
return (count);
|
||||
}
|
||||
latest = store.insert(WormAuditTables.CHECKPOINT, List.of(Checkpoints.sign(pending, config, now))).getFirst();
|
||||
lastId = latest.getValueLong("to_link_id");
|
||||
count++;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.checkpoint;
|
||||
|
||||
|
||||
import java.security.GeneralSecurityException;
|
||||
import java.security.PublicKey;
|
||||
import java.security.Signature;
|
||||
import java.time.Instant;
|
||||
import java.util.Base64;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.canonical.Jcs;
|
||||
import com.kingsrook.qbits.wormaudit.chain.Hashes;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Ed25519 checkpoint envelopes. Trust comes exclusively from host-pinned keys,
|
||||
** never the public_key supplied by an untrusted checkpoint row.
|
||||
*******************************************************************************/
|
||||
public final class Checkpoints
|
||||
{
|
||||
private static final List<String> SIGNED_FIELDS = List.of("from_link_id", "to_link_id", "link_count", "head_hash", "merkle_root", "algorithm", "key_id");
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private Checkpoints()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The exact signed payload; public_key and signature are envelope fields.
|
||||
*******************************************************************************/
|
||||
public static byte[] payload(QRecord checkpoint)
|
||||
{
|
||||
Map<String, Object> fields = new LinkedHashMap<>();
|
||||
for(String field : SIGNED_FIELDS)
|
||||
{
|
||||
fields.put(field, field.endsWith("_id") && !field.equals("key_id") || field.equals("link_count")
|
||||
? checkpoint.getValueLong(field) : checkpoint.getValueString(field));
|
||||
}
|
||||
return (Jcs.bytes(fields));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Create a checkpoint only when the configured private key matches its pin.
|
||||
*******************************************************************************/
|
||||
public static QRecord sign(List<QRecord> links, WormAuditQBitConfig config, Instant now) throws GeneralSecurityException
|
||||
{
|
||||
WormAuditQBitConfig.SigningKey key = config.getSigningKey().get();
|
||||
PublicKey publicKey = config.getTrustedPublicKeys().get(key.keyId());
|
||||
if(publicKey == null)
|
||||
{
|
||||
throw (new GeneralSecurityException("Signing key id is not pinned"));
|
||||
}
|
||||
QRecord record = new QRecord().withValue("from_link_id", links.getFirst().getValueLong("id"))
|
||||
.withValue("to_link_id", links.getLast().getValueLong("id")).withValue("link_count", (long) links.size())
|
||||
.withValue("head_hash", links.getLast().getValueString("link_hash"))
|
||||
.withValue("merkle_root", Hashes.merkle(links.stream().map(r -> r.getValueString("link_hash")).toList()))
|
||||
.withValue("algorithm", "Ed25519").withValue("key_id", key.keyId())
|
||||
.withValue("public_key", Base64.getEncoder().encodeToString(publicKey.getEncoded())).withValue("created_at", now);
|
||||
Signature signature = Signature.getInstance("Ed25519");
|
||||
signature.initSign(key.privateKey());
|
||||
signature.update(payload(record));
|
||||
record.setValue("signature", Base64.getEncoder().encodeToString(signature.sign()));
|
||||
if(!verify(record, publicKey))
|
||||
{
|
||||
throw (new GeneralSecurityException("Signing private key does not match its pin"));
|
||||
}
|
||||
return (record);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify with a pinned key, rejecting substituted algorithms and public keys.
|
||||
*******************************************************************************/
|
||||
public static boolean verify(QRecord record, PublicKey pin)
|
||||
{
|
||||
try
|
||||
{
|
||||
if(pin == null || !"Ed25519".equals(record.getValueString("algorithm"))
|
||||
|| !Base64.getEncoder().encodeToString(pin.getEncoded()).equals(record.getValueString("public_key")))
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
Signature signature = Signature.getInstance("Ed25519");
|
||||
signature.initVerify(pin);
|
||||
signature.update(payload(record));
|
||||
return (signature.verify(Base64.getDecoder().decode(record.getValueString("signature"))));
|
||||
}
|
||||
catch(GeneralSecurityException | IllegalArgumentException e)
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.ddl;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Render a reviewed migration for the host's physical audit table names.
|
||||
** Core physical columns must use snake_case: audit.id/timestamp,
|
||||
** audit_detail.id/audit_id, audit_table.id/name/label/create_date/modify_date,
|
||||
** and audit_user.id/name/create_date/modify_date. Only table names are rendered;
|
||||
** QInstance field backend-name overrides do not change this DDL contract.
|
||||
** Identifiers are restricted and quoted; configuration is never executable SQL.
|
||||
*******************************************************************************/
|
||||
public final class WormAuditDdl
|
||||
{
|
||||
public static final Map<String, String> DEFAULT_NAMES = Map.of("audit", "audit", "auditDetail", "audit_detail", "auditTable", "audit_table", "auditUser", "audit_user");
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private WormAuditDdl()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A safe SQL identifier, optionally schema qualified.
|
||||
*******************************************************************************/
|
||||
public static String identifier(String value)
|
||||
{
|
||||
if(value == null || !value.matches("[A-Za-z_][A-Za-z0-9_]*(\\.[A-Za-z_][A-Za-z0-9_]*)?"))
|
||||
{
|
||||
throw (new IllegalArgumentException("Invalid SQL identifier"));
|
||||
}
|
||||
return ("\"" + value.replace(".", "\".\"") + "\"");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve backend names while preserving explicit DDL overrides.
|
||||
*******************************************************************************/
|
||||
public static void resolveNames(WormAuditQBitConfig config, QInstance instance)
|
||||
{
|
||||
Map<String, String> names = new LinkedHashMap<>();
|
||||
for(String name : DEFAULT_NAMES.keySet())
|
||||
{
|
||||
QTableMetaData table = instance.getTable(name);
|
||||
if(table == null)
|
||||
{
|
||||
throw (new IllegalArgumentException("Core audit table is required: " + name));
|
||||
}
|
||||
names.put(name, table.getBackendDetails() instanceof RDBMSTableBackendDetails details ? details.getTableName() : DEFAULT_NAMES.get(name));
|
||||
}
|
||||
names.putAll(config.getCoreAuditTableNames());
|
||||
config.withCoreAuditTableNames(Map.copyOf(names));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Render the template. Retention is compiled into the function, never a GUC.
|
||||
*******************************************************************************/
|
||||
public static String postgres(WormAuditQBitConfig config)
|
||||
{
|
||||
if(config.getRetentionYears() < 1 || config.getAppRole().contains("."))
|
||||
{
|
||||
throw (new IllegalArgumentException("Invalid retention or app role"));
|
||||
}
|
||||
try(var stream = WormAuditDdl.class.getResourceAsStream("/com/kingsrook/qbits/wormaudit/ddl/postgres/worm-audit.sql"))
|
||||
{
|
||||
String sql = new String(java.util.Objects.requireNonNull(stream).readAllBytes(), StandardCharsets.UTF_8);
|
||||
Map<String, String> names = new LinkedHashMap<>(DEFAULT_NAMES);
|
||||
names.putAll(config.getCoreAuditTableNames());
|
||||
for(var entry : DEFAULT_NAMES.entrySet())
|
||||
{
|
||||
sql = sql.replace("${" + entry.getValue() + "}", identifier(names.get(entry.getKey())));
|
||||
}
|
||||
var defaults = com.kingsrook.qbits.wormaudit.WormAuditSettings.defaults(config);
|
||||
com.kingsrook.qbits.wormaudit.WormAuditSettings.validate(defaults);
|
||||
String values = java.util.List.of("id", "audit_retention_years", "export_lock_mode", "export_retention_days", "checkpoint_every_links",
|
||||
"checkpoint_every_minutes", "chain_interval_seconds", "settle_horizon_seconds", "late_detail_lookback_hours", "export_cron", "verify_cron")
|
||||
.stream().map(field -> defaults.getValue(field) instanceof Number ? defaults.getValue(field).toString()
|
||||
: "'" + defaults.getValueString(field).replace("'", "''") + "'").collect(java.util.stream.Collectors.joining(", "));
|
||||
sql = sql.replace("${settings_values}", values);
|
||||
return (sql.replace("${app_role}", identifier(config.getAppRole())).replace("${retention}", config.getRetentionYears() + " years"));
|
||||
}
|
||||
catch(IOException e)
|
||||
{
|
||||
throw (new IllegalStateException("Cannot read WORM audit DDL", e));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.export;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.Serializable;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.security.MessageDigest;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HexFormat;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.fasterxml.jackson.core.JsonParser;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.kingsrook.qbits.objectstorage.api.StoredObject;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Version-one export encoding and temporary files. JSON errors never echo input.
|
||||
*******************************************************************************/
|
||||
public class ExportFiles implements AutoCloseable
|
||||
{
|
||||
public static final List<String> NAMES = List.of("links.jsonl", "checkpoints.jsonl", "manifest.json");
|
||||
private static final ObjectMapper JSON = new ObjectMapper().enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION)
|
||||
.enable(DeserializationFeature.USE_LONG_FOR_INTS).enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
|
||||
private final Path directory;
|
||||
|
||||
/*******************************************************************************
|
||||
** Create a private temporary directory, removed on all normal failure paths.
|
||||
*******************************************************************************/
|
||||
public ExportFiles() throws IOException
|
||||
{
|
||||
directory = Files.createTempDirectory("worm-audit-");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve only one of the fixed filenames, never a manifest-supplied path.
|
||||
*******************************************************************************/
|
||||
public Path file(String name)
|
||||
{
|
||||
if(!NAMES.contains(name))
|
||||
{
|
||||
throw (new IllegalArgumentException("Unknown export file"));
|
||||
}
|
||||
return (directory.resolve(name));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Parse the JCS schema with strict duplicate and trailing-token rejection.
|
||||
*******************************************************************************/
|
||||
public static Map<String, Object> parse(byte[] bytes) throws IOException
|
||||
{
|
||||
try
|
||||
{
|
||||
return (JSON.readValue(bytes, new TypeReference<Map<String, Object>>() {}));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw (new IOException("Invalid sealed JSON"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Wire representation of a chain row, excluding database-only timestamps.
|
||||
*******************************************************************************/
|
||||
public static Map<String, Object> link(QRecord row)
|
||||
{
|
||||
return (Map.of("id", row.getValueLong("id"), "kind", row.getValueString("kind"),
|
||||
"auditId", row.getValueLong("audit_id"), "detailIds", AuditStore.detailIds(row),
|
||||
"prevHash", row.getValueString("prev_hash"), "linkHash", row.getValueString("link_hash")));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Preserve every checkpoint field, representing the timestamp as UTC text.
|
||||
*******************************************************************************/
|
||||
public static Map<String, Object> checkpoint(QRecord row)
|
||||
{
|
||||
Map<String, Object> fields = new LinkedHashMap<>();
|
||||
for(String key : List.of("id", "from_link_id", "to_link_id", "link_count"))
|
||||
{
|
||||
fields.put(key, row.getValueLong(key));
|
||||
}
|
||||
for(String key : List.of("head_hash", "merkle_root", "algorithm", "key_id", "public_key", "signature"))
|
||||
{
|
||||
fields.put(key, row.getValueString(key));
|
||||
}
|
||||
fields.put("created_at", row.getValueInstant("created_at").toString());
|
||||
return (fields);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Convert a checkpoint envelope back to the phase (a) signature input.
|
||||
*******************************************************************************/
|
||||
public static QRecord record(Map<String, Object> fields)
|
||||
{
|
||||
QRecord row = new QRecord();
|
||||
fields.forEach((key, value) -> row.setValue(key, (Serializable) value));
|
||||
return (row);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Compute a digest without materializing a file.
|
||||
*******************************************************************************/
|
||||
public static String sha256(Path path) throws Exception
|
||||
{
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
try(InputStream input = Files.newInputStream(path))
|
||||
{
|
||||
byte[] buffer = new byte[65536];
|
||||
int read;
|
||||
while((read = input.read(buffer)) != -1)
|
||||
{
|
||||
digest.update(buffer, 0, read);
|
||||
}
|
||||
}
|
||||
return (HexFormat.of().formatHex(digest.digest()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A lock echo is required even if a provider advertises the capability.
|
||||
*******************************************************************************/
|
||||
public static boolean locked(StoredObject object, String mode, java.time.Instant until)
|
||||
{
|
||||
return (object != null && java.util.Objects.equals(mode, object.getLockMode()) && object.getLockRetainUntil() != null
|
||||
&& !object.getLockRetainUntil().isBefore(until));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A null version belongs to unversioned storage and cannot be sealed.
|
||||
*******************************************************************************/
|
||||
public static boolean validVersion(String versionId)
|
||||
{
|
||||
return (versionId != null && !versionId.isBlank() && !"null".equals(versionId));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fetch covered detail rows in bounded IN batches.
|
||||
*******************************************************************************/
|
||||
public static Map<Long, QRecord> details(AuditStore store, List<QRecord> links) throws Exception
|
||||
{
|
||||
List<Long> ids = new ArrayList<>();
|
||||
for(QRecord link : links)
|
||||
{
|
||||
List<Long> covered = AuditStore.detailIds(link);
|
||||
if(covered.size() > 250)
|
||||
{
|
||||
throw (new IOException("Invalid detail coverage"));
|
||||
}
|
||||
ids.addAll(covered);
|
||||
}
|
||||
Map<Long, QRecord> rows = new LinkedHashMap<>();
|
||||
for(int start = 0; start < ids.size(); start += 1000)
|
||||
{
|
||||
rows.putAll(AuditStore.index(store.in("auditDetail", "id", ids.subList(start, Math.min(start + 1000, ids.size())))));
|
||||
}
|
||||
return (rows);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Remove only the three owned files and their directory.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void close() throws IOException
|
||||
{
|
||||
IOException failure = null;
|
||||
for(String name : NAMES)
|
||||
{
|
||||
try
|
||||
{
|
||||
Files.deleteIfExists(file(name));
|
||||
}
|
||||
catch(IOException e)
|
||||
{
|
||||
failure = new IOException("Cannot remove temporary export files");
|
||||
}
|
||||
}
|
||||
Files.deleteIfExists(directory);
|
||||
if(failure != null)
|
||||
{
|
||||
throw (failure);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.export;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Instant;
|
||||
import com.kingsrook.qbits.objectstorage.api.Capability;
|
||||
import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
|
||||
import com.kingsrook.qbits.objectstorage.api.ListRequest;
|
||||
import com.kingsrook.qbits.objectstorage.api.ObjectStorage;
|
||||
import com.kingsrook.qbits.objectstorage.api.ObjectSummary;
|
||||
import com.kingsrook.qbits.objectstorage.api.Page;
|
||||
import com.kingsrook.qbits.objectstorage.api.PutRequest;
|
||||
import com.kingsrook.qbits.objectstorage.api.StorageBucket;
|
||||
import com.kingsrook.qbits.objectstorage.api.StorageKeys;
|
||||
import com.kingsrook.qbits.objectstorage.api.StoredObject;
|
||||
import com.kingsrook.qbits.objectstorage.config.BucketRole;
|
||||
import com.kingsrook.qbits.objectstorage.internal.ObjectStorageRuntime;
|
||||
import com.kingsrook.qbits.objectstorage.internal.SecretRefs;
|
||||
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Thin system-caller adapter to qbit-object-storage.
|
||||
*******************************************************************************/
|
||||
public class ObjectStorageSealedStore implements SealedStore
|
||||
{
|
||||
private final StorageBucket bucket;
|
||||
|
||||
/*******************************************************************************
|
||||
** Bind the recorded alias, so old exports survive configuration changes.
|
||||
*******************************************************************************/
|
||||
public ObjectStorageSealedStore(String alias, String runId)
|
||||
{
|
||||
bucket = ObjectStorage.asSystem("worm_audit", runId).bucket(alias);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Refuse unknown, partial, or unsupported lock capabilities.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean supportsLock()
|
||||
{
|
||||
return (bucket.config().getRole() == BucketRole.BACKUP
|
||||
&& bucket.capability(Capability.OBJECT_LOCK) == CapabilityStatus.SUPPORTED);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Report the provider capability without collapsing UNKNOWN into UNSUPPORTED.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public CapabilityStatus lockCapability()
|
||||
{
|
||||
return (bucket.capability(Capability.OBJECT_LOCK));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stream a bounded upload and close the source even when the provider fails.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String put(String key, Path file, com.kingsrook.qbits.objectstorage.api.LockMode mode, Instant retainUntil) throws Exception
|
||||
{
|
||||
try(InputStream input = Files.newInputStream(file))
|
||||
{
|
||||
return (bucket.put(PutRequest.of(key, input).withContentLength(Files.size(file))
|
||||
.withObjectLock(mode, retainUntil)).getVersionId());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read retention metadata.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public StoredObject head(String key)
|
||||
{
|
||||
return (bucket.head(key).orElse(null));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** RC.2 has no version GET facade. Reuse its configured system connection,
|
||||
** key normalization and limiter; closing the stream closes the S3 response.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public InputStream getVersion(String key, String versionId) throws IOException
|
||||
{
|
||||
if(!ExportFiles.validVersion(versionId))
|
||||
{
|
||||
throw (new IOException("Missing sealed object version"));
|
||||
}
|
||||
var connection = ObjectStorageRuntime.connection(bucket.config().getConnection());
|
||||
try(var ignored = connection.getRateLimiter().acquire())
|
||||
{
|
||||
var input = connection.client().getObject(GetObjectRequest.builder()
|
||||
.bucket(SecretRefs.resolveSetting(bucket.config().getBucketName())).key(StorageKeys.normalize(key)).versionId(versionId).build());
|
||||
if(!versionId.equals(input.response().versionId()))
|
||||
{
|
||||
input.close();
|
||||
throw (new IOException("Storage returned a different object version"));
|
||||
}
|
||||
return (input);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Forward bounded listing.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public Page<ObjectSummary> list(ListRequest request)
|
||||
{
|
||||
return (bucket.list(request));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,424 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.export;
|
||||
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.sql.Connection;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.canonical.Jcs;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.chain.Hashes;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.checkpoint.Checkpoints;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.VerificationRuns;
|
||||
import com.kingsrook.qbits.wormaudit.verify.VerificationResult;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSTransaction;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify the sealed evidence before comparing it to a repeatable live snapshot.
|
||||
** No exported content or provider/parser exception text enters diagnostics.
|
||||
*******************************************************************************/
|
||||
public class SealedExportVerifier
|
||||
{
|
||||
private final WormAuditQBitConfig config;
|
||||
private final SealedStore suppliedStorage;
|
||||
|
||||
/*******************************************************************************
|
||||
** Findings plus the number verified without live rows after a retention purge.
|
||||
*******************************************************************************/
|
||||
public record Report(VerificationResult result, long purged)
|
||||
{
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Production entry point.
|
||||
*******************************************************************************/
|
||||
public SealedExportVerifier(WormAuditQBitConfig config)
|
||||
{
|
||||
this(config, null);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Testable storage boundary.
|
||||
*******************************************************************************/
|
||||
public SealedExportVerifier(WormAuditQBitConfig config, SealedStore storage)
|
||||
{
|
||||
this.config = config;
|
||||
this.suppliedStorage = storage;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Persist exactly one verification row for this selected export, even on failure.
|
||||
*******************************************************************************/
|
||||
public Report verify(QRecord export, String trigger) throws QException
|
||||
{
|
||||
config.effectiveSettings();
|
||||
VerificationResult failure = new VerificationResult();
|
||||
failure.scope(0, export.getValueLong("from_link_id"), export.getValueLong("to_link_id"));
|
||||
Report report;
|
||||
WormAuditChainer.JVM_LOCK.lock();
|
||||
try(QBackendTransaction transaction = QBackendTransaction.openFor(new QueryInput(WormAuditTables.CHAIN));
|
||||
ExportFiles files = new ExportFiles())
|
||||
{
|
||||
try
|
||||
{
|
||||
if(transaction instanceof RDBMSTransaction rdbms)
|
||||
{
|
||||
rdbms.getConnection().setTransactionIsolation(Connection.TRANSACTION_REPEATABLE_READ);
|
||||
}
|
||||
SealedStore storage = suppliedStorage == null
|
||||
? config.getSealedStoreSupplier() == null || !config.getExportBucketAlias().equals(export.getValueString("bucket_alias"))
|
||||
? new ObjectStorageSealedStore(export.getValueString("bucket_alias"), "verify-" + export.getValueLong("id"))
|
||||
: config.getSealedStoreSupplier().get() : suppliedStorage;
|
||||
Map<String, Object> versions = ExportFiles.parse(export.getValueString("object_versions").getBytes(StandardCharsets.UTF_8));
|
||||
for(String name : ExportFiles.NAMES)
|
||||
{
|
||||
String key = export.getValueString("object_prefix") + name;
|
||||
if(!(versions.get(name) instanceof String version) || !ExportFiles.validVersion(version))
|
||||
{
|
||||
failure.add(null, null, "object " + name + " has no recorded sealed version id");
|
||||
throw (new IOException("Missing sealed object version"));
|
||||
}
|
||||
var head = storage.head(key);
|
||||
if(head == null)
|
||||
{
|
||||
failure.add(null, null, "object " + key + " is missing" + ("GOVERNANCE".equals(export.getValueString("lock_mode"))
|
||||
? "; possible GOVERNANCE bypass removal by an administrator" : ""));
|
||||
}
|
||||
if(head != null && !version.equals(head.getVersionId()))
|
||||
{
|
||||
failure.add(null, null, "object " + name + " was overwritten after sealing (current version " + head.getVersionId()
|
||||
+ ", sealed version " + version + " is still retained under Object Lock)");
|
||||
}
|
||||
if(!ExportFiles.locked(head, export.getValueString("lock_mode"), export.getValueInstant("retain_until")))
|
||||
{
|
||||
failure.add(null, null, "object " + name + " lacks the recorded " + export.getValueString("lock_mode") + " retention");
|
||||
}
|
||||
}
|
||||
for(String name : ExportFiles.NAMES)
|
||||
{
|
||||
String key = export.getValueString("object_prefix") + name;
|
||||
try(InputStream input = storage.getVersion(key, (String) versions.get(name)))
|
||||
{
|
||||
Files.copy(input, files.file(name));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
failure.add(null, null, "object " + key + " sealed version cannot be read" + ("GOVERNANCE".equals(export.getValueString("lock_mode"))
|
||||
? "; if removed, possible GOVERNANCE bypass removal by an administrator" : ""));
|
||||
throw (new IOException("Cannot read sealed object version"));
|
||||
}
|
||||
}
|
||||
report = inspect(files, export, new AuditStore(transaction));
|
||||
for(var item : failure.getBreaks())
|
||||
{
|
||||
report.result().add(item.linkId(), item.auditId(), item.reason());
|
||||
}
|
||||
transaction.commit();
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
transaction.rollback();
|
||||
throw (e);
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
failure.add(null, null, "Sealed export cannot be read or has invalid structure");
|
||||
report = new Report(failure, 0);
|
||||
}
|
||||
finally
|
||||
{
|
||||
WormAuditChainer.JVM_LOCK.unlock();
|
||||
}
|
||||
VerificationRuns.save(report.result(), trigger, export.getValueLong("id"));
|
||||
return (report);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Inspect local files, also used by the exporter before committing any objects.
|
||||
*******************************************************************************/
|
||||
public Report inspect(ExportFiles files, QRecord export, AuditStore store) throws Exception
|
||||
{
|
||||
VerificationResult result = new VerificationResult();
|
||||
long from = export.getValueLong("from_link_id");
|
||||
long to = export.getValueLong("to_link_id");
|
||||
result.scope(0, from, to);
|
||||
if(!Objects.equals(ExportFiles.sha256(files.file("manifest.json")), export.getValueString("manifest_sha256")))
|
||||
{
|
||||
result.add(null, null, "object manifest.json hash does not match the export row");
|
||||
return (new Report(result, 0));
|
||||
}
|
||||
if(Files.size(files.file("manifest.json")) > 65536)
|
||||
{
|
||||
result.add(null, null, "Manifest exceeds the format size limit");
|
||||
return (new Report(result, 0));
|
||||
}
|
||||
Map<String, Object> manifest = ExportFiles.parse(Files.readAllBytes(files.file("manifest.json")));
|
||||
if(!"worm-audit-export/1".equals(manifest.get("format")) || from < 1 || to < from
|
||||
|| !Objects.equals(manifest.get("fromLinkId"), from) || !Objects.equals(manifest.get("toLinkId"), to)
|
||||
|| !Objects.equals(manifest.get("linkCount"), to - from + 1)
|
||||
|| !Objects.equals(manifest.get("createdAt"), export.getValueInstant("sealed_at").toString())
|
||||
|| !java.util.Set.of("GOVERNANCE", "COMPLIANCE").contains(export.getValueString("lock_mode"))
|
||||
|| !Objects.equals(export.getValueString("manifest_key"), export.getValueString("object_prefix") + "manifest.json"))
|
||||
{
|
||||
result.add(null, null, "Manifest metadata does not match the export row");
|
||||
}
|
||||
QQueryFilter previousFilter = new QQueryFilter().withOrderBy(new QFilterOrderBy("id", false)).withLimit(1);
|
||||
if(export.getValueLong("id") != null)
|
||||
{
|
||||
previousFilter.withCriteria(new QFilterCriteria("id", QCriteriaOperator.LESS_THAN, export.getValueLong("id")));
|
||||
}
|
||||
List<QRecord> previousRows = store.query(WormAuditTables.EXPORT, previousFilter);
|
||||
QRecord previous = previousRows.isEmpty() ? null : previousRows.getFirst();
|
||||
String previousHash = previous == null ? null : previous.getValueString("manifest_sha256");
|
||||
if(!Objects.equals(manifest.get("prevManifestSha256"), previousHash)
|
||||
|| !Objects.equals(export.getValueString("prev_manifest_sha256"), previousHash)
|
||||
|| (previous != null && previous.getValueLong("to_link_id") != from - 1))
|
||||
{
|
||||
result.add(null, null, "prevManifestSha256 does not match the previous export row");
|
||||
}
|
||||
Map<?, ?> hashes = (Map<?, ?>) manifest.get("files");
|
||||
for(String name : List.of("links.jsonl", "checkpoints.jsonl"))
|
||||
{
|
||||
if(!Objects.equals(ExportFiles.sha256(files.file(name)), hashes.get(name)))
|
||||
{
|
||||
result.add(null, null, "object " + name + " hash does not match the manifest");
|
||||
}
|
||||
}
|
||||
if(!result.passed())
|
||||
{
|
||||
return (new Report(result, 0));
|
||||
}
|
||||
return (checkLines(files, export, store, manifest, result));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Consume links alongside their checkpoint envelopes. Only one checkpoint's
|
||||
** hashes and one content page are retained, never the whole export.
|
||||
*******************************************************************************/
|
||||
private Report checkLines(ExportFiles files, QRecord export, AuditStore store, Map<String, Object> manifest, VerificationResult result) throws Exception
|
||||
{
|
||||
long from = export.getValueLong("from_link_id");
|
||||
long to = export.getValueLong("to_link_id");
|
||||
long cursor = from - 1;
|
||||
long checkpointCount = 0;
|
||||
long checked = 0;
|
||||
long purged = 0;
|
||||
long anchor = store.anchor().getValueLong("anchor_link_id");
|
||||
String previousHash = from == 1 ? Hashes.GENESIS : null;
|
||||
if(cursor == anchor)
|
||||
{
|
||||
previousHash = store.anchor().getValueString("anchor_hash");
|
||||
}
|
||||
long lastCheckpoint = 0;
|
||||
try(BufferedReader links = Files.newBufferedReader(files.file("links.jsonl"), StandardCharsets.UTF_8);
|
||||
BufferedReader checkpoints = Files.newBufferedReader(files.file("checkpoints.jsonl"), StandardCharsets.UTF_8))
|
||||
{
|
||||
String line;
|
||||
while((line = checkpoints.readLine()) != null)
|
||||
{
|
||||
QRecord checkpoint = ExportFiles.record(ExportFiles.parse(line.getBytes(StandardCharsets.UTF_8)));
|
||||
long checkpointId = checkpoint.getValueLong("id");
|
||||
String label = "checkpoint " + WormAuditVerifier.number(checkpointId) + ": ";
|
||||
long end = checkpoint.getValueLong("to_link_id");
|
||||
long start = checkpoint.getValueLong("from_link_id");
|
||||
long count = checkpoint.getValueLong("link_count");
|
||||
if(start != cursor + 1 || end < start || end > to || count != end - start + 1
|
||||
|| count > Integer.MAX_VALUE || checkpointId <= lastCheckpoint)
|
||||
{
|
||||
result.add(null, null, label + "invalid or incomplete checkpoint range");
|
||||
break;
|
||||
}
|
||||
if(!Checkpoints.verify(checkpoint, config.getTrustedPublicKeys().get(checkpoint.getValueString("key_id"))))
|
||||
{
|
||||
result.add(null, null, label + "signature does not verify");
|
||||
}
|
||||
if(checkpointCount == 0 && !Objects.equals(export.getValueLong("from_checkpoint_id"), checkpointId))
|
||||
{
|
||||
result.add(null, null, label + "first checkpoint does not match the export row");
|
||||
}
|
||||
List<String> hashes = new ArrayList<>();
|
||||
while(cursor < end)
|
||||
{
|
||||
List<Map<String, Object>> page = new ArrayList<>();
|
||||
for(int i = 0, size = (int) Math.min(1000, end - cursor); i < size; i++)
|
||||
{
|
||||
String linkLine = links.readLine();
|
||||
if(linkLine == null)
|
||||
{
|
||||
result.add(cursor + 1, null, "Sealed links are missing from the checkpoint range");
|
||||
result.scope(checked, from, to);
|
||||
return (new Report(result, purged));
|
||||
}
|
||||
page.add(ExportFiles.parse(linkLine.getBytes(StandardCharsets.UTF_8)));
|
||||
}
|
||||
compareLive(page, store, export, anchor, result);
|
||||
for(Map<String, Object> document : page)
|
||||
{
|
||||
Map<?, ?> link = (Map<?, ?>) document.get("link");
|
||||
long id = (Long) link.get("id");
|
||||
long auditId = (Long) link.get("auditId");
|
||||
String prefix = "link " + WormAuditVerifier.number(id) + " (audit " + WormAuditVerifier.number(auditId) + "): ";
|
||||
String prev = (String) link.get("prevHash");
|
||||
String hash = (String) link.get("linkHash");
|
||||
if(id != cursor + 1 || (previousHash != null && !Objects.equals(prev, previousHash)))
|
||||
{
|
||||
result.add(id, auditId, prefix + "sealed chain continuity differs");
|
||||
}
|
||||
Map<?, ?> content = (Map<?, ?>) document.get("content");
|
||||
Map<?, ?> audit = (Map<?, ?>) content.get("audit");
|
||||
List<?> detailIds = (List<?>) link.get("detailIds");
|
||||
List<?> details = (List<?>) content.get("details");
|
||||
List<?> contentIds = details.stream().map(d -> ((Map<?, ?>) d).get("id")).toList();
|
||||
if(!Objects.equals(audit.get("id"), auditId) || !Objects.equals(content.get("v"), 1L)
|
||||
|| !List.of("AUDIT", "LATE_DETAIL").contains(link.get("kind"))
|
||||
|| detailIds.size() > 250 || !detailIds.equals(contentIds)
|
||||
|| detailIds.stream().distinct().count() != detailIds.size()
|
||||
|| ("LATE_DETAIL".equals(link.get("kind")) && detailIds.isEmpty()))
|
||||
{
|
||||
result.add(id, auditId, prefix + "sealed content identifiers or coverage differ");
|
||||
}
|
||||
if(!Hashes.link(prev, Jcs.bytes(content)).equals(hash))
|
||||
{
|
||||
result.add(id, auditId, prefix + "sealed content differs from the chained hash");
|
||||
}
|
||||
hashes.add(hash);
|
||||
previousHash = hash;
|
||||
cursor++;
|
||||
checked++;
|
||||
if(id <= anchor)
|
||||
{
|
||||
purged++;
|
||||
}
|
||||
}
|
||||
}
|
||||
if(!Objects.equals(Hashes.merkle(hashes), checkpoint.getValueString("merkle_root"))
|
||||
|| !Objects.equals(previousHash, checkpoint.getValueString("head_hash")))
|
||||
{
|
||||
result.add(null, null, label + "Merkle root or head hash differs");
|
||||
}
|
||||
lastCheckpoint = checkpointId;
|
||||
checkpointCount++;
|
||||
}
|
||||
if(cursor != to || links.readLine() != null || !Objects.equals(manifest.get("checkpointCount"), checkpointCount)
|
||||
|| !Objects.equals(export.getValueLong("to_checkpoint_id"), lastCheckpoint)
|
||||
|| !Objects.equals(manifest.get("headHash"), previousHash))
|
||||
{
|
||||
result.add(null, null, "Sealed range, counts or head hash differ from the manifest");
|
||||
}
|
||||
}
|
||||
result.scope(checked, from, to);
|
||||
return (new Report(result, purged));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Compare a bounded page by exported ids, so missing live rows cannot vanish
|
||||
** from the verification scope. Purged ids are checked from signed copies only.
|
||||
*******************************************************************************/
|
||||
private void compareLive(List<Map<String, Object>> page, AuditStore store, QRecord export, long anchor, VerificationResult result) throws Exception
|
||||
{
|
||||
List<QRecord> sealed = page.stream().map(document ->
|
||||
{
|
||||
Map<?, ?> link = (Map<?, ?>) document.get("link");
|
||||
List<?> ids = (List<?>) link.get("detailIds");
|
||||
return (new QRecord().withValue("id", (Long) link.get("id")).withValue("audit_id", (Long) link.get("auditId"))
|
||||
.withValue("kind", (String) link.get("kind")).withValue("prev_hash", (String) link.get("prevHash"))
|
||||
.withValue("link_hash", (String) link.get("linkHash"))
|
||||
.withValue("detail_ids", ids.stream().map(Object::toString).collect(java.util.stream.Collectors.joining(","))));
|
||||
}).toList();
|
||||
List<QRecord> liveScope = sealed.stream().filter(r -> r.getValueLong("id") > anchor).toList();
|
||||
Map<Long, QRecord> live = AuditStore.index(store.in(WormAuditTables.CHAIN, "id", liveScope.stream().map(r -> r.getValueLong("id")).toList()));
|
||||
Map<Long, QRecord> audits = AuditStore.index(store.in("audit", "id", liveScope.stream().map(r -> r.getValueLong("audit_id")).toList()));
|
||||
Map<Long, QRecord> details = ExportFiles.details(store, liveScope);
|
||||
for(int i = 0; i < sealed.size(); i++)
|
||||
{
|
||||
QRecord link = sealed.get(i);
|
||||
long id = link.getValueLong("id");
|
||||
long auditId = link.getValueLong("audit_id");
|
||||
if(id <= anchor)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
String prefix = "link " + WormAuditVerifier.number(id) + " (audit " + WormAuditVerifier.number(auditId) + "): ";
|
||||
QRecord current = live.get(id);
|
||||
if(current == null)
|
||||
{
|
||||
result.add(id, auditId, "link " + WormAuditVerifier.number(id) + " is missing from the live chain");
|
||||
}
|
||||
else
|
||||
{
|
||||
for(String field : List.of("kind", "audit_id", "detail_ids", "prev_hash", "link_hash"))
|
||||
{
|
||||
if(!Objects.equals(link.getValue(field), current.getValue(field)))
|
||||
{
|
||||
result.add(id, auditId, prefix + "live chain " + field + " differs from the sealed copy");
|
||||
}
|
||||
}
|
||||
}
|
||||
try
|
||||
{
|
||||
QRecord audit = audits.get(auditId);
|
||||
if(audit == null)
|
||||
{
|
||||
result.add(id, auditId, prefix + "live audit header is missing");
|
||||
}
|
||||
else if(!Arrays.equals(store.canonical(audit, AuditStore.detailIds(link), details), Jcs.bytes(page.get(i).get("content"))))
|
||||
{
|
||||
result.add(id, auditId, prefix + "live content differs from the sealed copy (sealed "
|
||||
+ export.getValueInstant("sealed_at").atZone(ZoneOffset.UTC).toLocalDate() + ")");
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
result.add(id, auditId, prefix + "live content is missing or invalid");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,392 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.export;
|
||||
|
||||
|
||||
import java.io.BufferedWriter;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.Statement;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitProducer;
|
||||
import com.kingsrook.qbits.wormaudit.canonical.Jcs;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.chain.Hashes;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertSignal;
|
||||
import com.kingsrook.qqq.backend.core.alerts.QAlerts;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSTransaction;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Seal the checkpointed suffix under the chainer/purger lock. Only successful
|
||||
** uploads with verified lock echoes produce an export row.
|
||||
*******************************************************************************/
|
||||
public class SealedExporter
|
||||
{
|
||||
private final WormAuditQBitConfig config;
|
||||
private final SealedStore suppliedStorage;
|
||||
|
||||
/*******************************************************************************
|
||||
** A review scope; execution always recomputes it under the writer lock.
|
||||
*******************************************************************************/
|
||||
public record Scope(long from, long to, long checkpoints, QRecord previous)
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Number of links, including the empty checkpoint suffix.
|
||||
*******************************************************************************/
|
||||
public long count()
|
||||
{
|
||||
return (Math.max(0, to - from + 1));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Value-free Review text.
|
||||
*******************************************************************************/
|
||||
public String review()
|
||||
{
|
||||
return (count() == 0 ? "Nothing to seal: no new checkpoint" : WormAuditVerifier.number(count())
|
||||
+ " chain links will be sealed (links " + WormAuditVerifier.number(from) + "–" + WormAuditVerifier.number(to)
|
||||
+ ", " + WormAuditVerifier.number(checkpoints) + " checkpoints)");
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** A completed, empty, or failed run suitable for a Result line.
|
||||
*******************************************************************************/
|
||||
public record Result(QRecord row, String message, boolean failed)
|
||||
{
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Production storage is resolved lazily, so an empty run needs no bucket.
|
||||
*******************************************************************************/
|
||||
public SealedExporter(WormAuditQBitConfig config)
|
||||
{
|
||||
this(config, null);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Inject the tiny storage boundary for deterministic tests.
|
||||
*******************************************************************************/
|
||||
public SealedExporter(WormAuditQBitConfig config, SealedStore storage)
|
||||
{
|
||||
this.config = config;
|
||||
this.suppliedStorage = storage;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Select the contiguous suffix, never silently skipping an unexported purge.
|
||||
*******************************************************************************/
|
||||
public static Scope scope(AuditStore store) throws QException
|
||||
{
|
||||
QRecord previous = store.last(WormAuditTables.EXPORT);
|
||||
long anchor = store.anchor().getValueLong("anchor_link_id");
|
||||
long after = previous == null ? anchor : previous.getValueLong("to_link_id");
|
||||
if(after < anchor)
|
||||
{
|
||||
throw (new QException("Unexported links were purged"));
|
||||
}
|
||||
QRecord checkpoint = store.last(WormAuditTables.CHECKPOINT);
|
||||
long to = checkpoint == null ? after : Math.max(after, checkpoint.getValueLong("to_link_id"));
|
||||
long count = 0;
|
||||
long cursor = 0;
|
||||
while(true)
|
||||
{
|
||||
List<QRecord> page = checkpoints(store, after + 1, to, cursor);
|
||||
if(page.isEmpty())
|
||||
{
|
||||
break;
|
||||
}
|
||||
count += page.size();
|
||||
cursor = page.getLast().getValueLong("id");
|
||||
}
|
||||
return (new Scope(after + 1, to, count, previous));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Select checkpoint envelopes wholly contained in the export range.
|
||||
*******************************************************************************/
|
||||
private static List<QRecord> checkpoints(AuditStore store, long from, long to, long after) throws QException
|
||||
{
|
||||
return (store.query(WormAuditTables.CHECKPOINT, new QQueryFilter(
|
||||
new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, after),
|
||||
new QFilterCriteria("from_link_id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, from),
|
||||
new QFilterCriteria("to_link_id", QCriteriaOperator.LESS_THAN_OR_EQUALS, to))
|
||||
.withOrderBy(new QFilterOrderBy("id", true)).withLimit(1000)));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Execute with alerts on user and scheduled failures; schedules always beat.
|
||||
*******************************************************************************/
|
||||
public Result run(String trigger, Instant now)
|
||||
{
|
||||
try
|
||||
{
|
||||
config.validate();
|
||||
Result result = lockedRun(now);
|
||||
QAlerts.resolve("wormaudit.export.failed", "export", "Sealed audit export completed");
|
||||
return (result);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
// All exposed failures originate here or use a fixed, value-free code.
|
||||
String reason = e instanceof ExportFailure ? e.getMessage() : "Sealed export failed: storage or database operation failed";
|
||||
QAlerts.raise(new AlertSignal("wormaudit.export.failed").withDedupKey("export").withTitle(reason));
|
||||
return (new Result(null, reason, true));
|
||||
}
|
||||
finally
|
||||
{
|
||||
if("SCHEDULE".equals(trigger))
|
||||
{
|
||||
QAlerts.heartbeat("wormaudit.export.daily");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Serialize with writers in this JVM and PostgreSQL writers on other hosts.
|
||||
*******************************************************************************/
|
||||
private Result lockedRun(Instant now) throws Exception
|
||||
{
|
||||
WormAuditChainer.JVM_LOCK.lock();
|
||||
try(QBackendTransaction transaction = QBackendTransaction.openFor(new QueryInput(WormAuditTables.CHAIN)))
|
||||
{
|
||||
try
|
||||
{
|
||||
if(transaction instanceof RDBMSTransaction rdbms && "PostgreSQL".equals(rdbms.getConnection().getMetaData().getDatabaseProductName()))
|
||||
{
|
||||
try(Statement statement = rdbms.getConnection().createStatement();
|
||||
ResultSet result = statement.executeQuery("SELECT pg_advisory_xact_lock(" + WormAuditChainer.ADVISORY_LOCK + ")"))
|
||||
{
|
||||
result.next();
|
||||
}
|
||||
}
|
||||
AuditStore store = new AuditStore(transaction);
|
||||
Scope scope = scope(store);
|
||||
Result result;
|
||||
if(scope.count() == 0)
|
||||
{
|
||||
result = new Result(null, scope.review(), false);
|
||||
}
|
||||
else
|
||||
{
|
||||
SealedStore storage = suppliedStorage == null
|
||||
? config.getSealedStoreSupplier() == null ? new ObjectStorageSealedStore(config.getExportBucketAlias(), java.util.UUID.randomUUID().toString())
|
||||
: config.getSealedStoreSupplier().get() : suppliedStorage;
|
||||
result = seal(store, storage, scope, now.truncatedTo(ChronoUnit.MILLIS));
|
||||
}
|
||||
transaction.commit();
|
||||
return (result);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
transaction.rollback();
|
||||
throw (e);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
WormAuditChainer.JVM_LOCK.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build locally before uploading. Partial remote uploads have no database row.
|
||||
*******************************************************************************/
|
||||
private Result seal(AuditStore store, SealedStore storage, Scope scope, Instant now) throws Exception
|
||||
{
|
||||
if(!storage.supportsLock())
|
||||
{
|
||||
throw (new ExportFailure("Export refused: bucket does not support Object Lock"));
|
||||
}
|
||||
var settings = config.effectiveSettings();
|
||||
var mode = com.kingsrook.qbits.objectstorage.api.LockMode.valueOf(settings.export_lock_mode());
|
||||
Instant until = now.plus(settings.export_retention_days(), ChronoUnit.DAYS);
|
||||
String prefix = config.getExportPrefix() + "/" + DateTimeFormatter.ofPattern("yyyy/MM/dd").withZone(ZoneOffset.UTC).format(now)
|
||||
+ "/" + scope.from() + "-" + scope.to() + "/";
|
||||
QRecord row;
|
||||
try(ExportFiles files = new ExportFiles())
|
||||
{
|
||||
String head = writeLinks(files, store, scope);
|
||||
long[] checkpointIds = writeCheckpoints(files, store, scope);
|
||||
Map<String, Object> manifest = new LinkedHashMap<>();
|
||||
manifest.put("format", "worm-audit-export/1");
|
||||
manifest.put("fromLinkId", scope.from());
|
||||
manifest.put("toLinkId", scope.to());
|
||||
manifest.put("linkCount", scope.count());
|
||||
manifest.put("checkpointCount", scope.checkpoints());
|
||||
manifest.put("files", Map.of("links.jsonl", ExportFiles.sha256(files.file("links.jsonl")),
|
||||
"checkpoints.jsonl", ExportFiles.sha256(files.file("checkpoints.jsonl"))));
|
||||
manifest.put("headHash", head);
|
||||
String previous = scope.previous() == null ? null : scope.previous().getValueString("manifest_sha256");
|
||||
manifest.put("prevManifestSha256", previous);
|
||||
manifest.put("qbitVersion", WormAuditQBitProducer.VERSION);
|
||||
manifest.put("createdAt", now.toString());
|
||||
Files.write(files.file("manifest.json"), Jcs.bytes(manifest));
|
||||
row = new QRecord().withValue("from_link_id", scope.from()).withValue("to_link_id", scope.to())
|
||||
.withValue("from_checkpoint_id", checkpointIds[0]).withValue("to_checkpoint_id", checkpointIds[1])
|
||||
.withValue("bucket_alias", config.getExportBucketAlias()).withValue("object_prefix", prefix)
|
||||
.withValue("manifest_key", prefix + "manifest.json").withValue("manifest_sha256", ExportFiles.sha256(files.file("manifest.json")))
|
||||
.withValue("prev_manifest_sha256", previous).withValue("object_count", 3L)
|
||||
.withValue("lock_mode", mode.name()).withValue("retain_until", until).withValue("sealed_at", now);
|
||||
if(!new SealedExportVerifier(config, storage).inspect(files, row, store).result().passed())
|
||||
{
|
||||
throw (new ExportFailure("Export refused: source does not match the signed chain"));
|
||||
}
|
||||
long bytes = 0;
|
||||
Map<String, String> versions = new LinkedHashMap<>();
|
||||
for(String name : ExportFiles.NAMES)
|
||||
{
|
||||
String version = storage.put(prefix + name, files.file(name), mode, until);
|
||||
if(!ExportFiles.validVersion(version))
|
||||
{
|
||||
throw (new ExportFailure("Export refused: object " + name + " returned no version id"));
|
||||
}
|
||||
var headObject = storage.head(prefix + name);
|
||||
if(!ExportFiles.locked(headObject, mode.name(), until))
|
||||
{
|
||||
throw (new ExportFailure("Export refused: object " + name + " did not echo " + mode.name() + " retention"));
|
||||
}
|
||||
if(!version.equals(headObject.getVersionId()))
|
||||
{
|
||||
throw (new ExportFailure("Export refused: object " + name + " version changed after upload"));
|
||||
}
|
||||
versions.put(name, version);
|
||||
bytes += Files.size(files.file(name));
|
||||
}
|
||||
row.withValue("byte_count", bytes).withValue("object_versions", Jcs.string(versions));
|
||||
}
|
||||
row = store.insert(WormAuditTables.EXPORT, List.of(row)).getFirst();
|
||||
return (new Result(row, WormAuditVerifier.number(scope.count()) + " links sealed in export " + row.getValueLong("id")
|
||||
+ " (retained until " + until + ")", false));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stream canonical lines in pages of 1000 with batched content lookups.
|
||||
*******************************************************************************/
|
||||
private String writeLinks(ExportFiles files, AuditStore store, Scope scope) throws Exception
|
||||
{
|
||||
long cursor = scope.from() - 1;
|
||||
String head = null;
|
||||
try(BufferedWriter writer = Files.newBufferedWriter(files.file("links.jsonl"), StandardCharsets.UTF_8))
|
||||
{
|
||||
while(cursor < scope.to())
|
||||
{
|
||||
List<QRecord> links = store.page(WormAuditTables.CHAIN, cursor, (int) Math.min(1000, scope.to() - cursor));
|
||||
if(links.isEmpty())
|
||||
{
|
||||
throw (new ExportFailure("Export refused: chain range is incomplete"));
|
||||
}
|
||||
Map<Long, QRecord> audits = AuditStore.index(store.in("audit", "id", links.stream().map(r -> r.getValueLong("audit_id")).toList()));
|
||||
Map<Long, QRecord> details = ExportFiles.details(store, links);
|
||||
for(QRecord link : links)
|
||||
{
|
||||
if(link.getValueLong("id") != cursor + 1 || link.getValueLong("id") > scope.to())
|
||||
{
|
||||
throw (new ExportFailure("Export refused: chain range is incomplete"));
|
||||
}
|
||||
byte[] content = store.canonical(audits.get(link.getValueLong("audit_id")), AuditStore.detailIds(link), details);
|
||||
if(!Hashes.link(link.getValueString("prev_hash"), content).equals(link.getValueString("link_hash")))
|
||||
{
|
||||
throw (new ExportFailure("Export refused: content differs at link " + link.getValueLong("id")));
|
||||
}
|
||||
writer.write(Jcs.string(Map.of("link", ExportFiles.link(link), "content", ExportFiles.parse(content))));
|
||||
writer.write('\n');
|
||||
head = link.getValueString("link_hash");
|
||||
cursor = link.getValueLong("id");
|
||||
}
|
||||
}
|
||||
}
|
||||
return (head);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stream complete signed checkpoint envelopes in keyset pages.
|
||||
*******************************************************************************/
|
||||
private long[] writeCheckpoints(ExportFiles files, AuditStore store, Scope scope) throws Exception
|
||||
{
|
||||
long first = 0;
|
||||
long cursor = 0;
|
||||
try(BufferedWriter writer = Files.newBufferedWriter(files.file("checkpoints.jsonl"), StandardCharsets.UTF_8))
|
||||
{
|
||||
while(true)
|
||||
{
|
||||
List<QRecord> page = checkpoints(store, scope.from(), scope.to(), cursor);
|
||||
if(page.isEmpty())
|
||||
{
|
||||
break;
|
||||
}
|
||||
for(QRecord row : page)
|
||||
{
|
||||
first = first == 0 ? row.getValueLong("id") : first;
|
||||
cursor = row.getValueLong("id");
|
||||
writer.write(Jcs.string(ExportFiles.checkpoint(row)));
|
||||
writer.write('\n');
|
||||
}
|
||||
}
|
||||
}
|
||||
return (new long[] { first, cursor });
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Only fixed messages constructed here may reach an alert or Result screen.
|
||||
*******************************************************************************/
|
||||
private static class ExportFailure extends IOException
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Carry a value-free exporter reason without a provider exception cause.
|
||||
*******************************************************************************/
|
||||
ExportFailure(String reason)
|
||||
{
|
||||
super(reason);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.export;
|
||||
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Instant;
|
||||
import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
|
||||
import com.kingsrook.qbits.objectstorage.api.ListRequest;
|
||||
import com.kingsrook.qbits.objectstorage.api.ObjectSummary;
|
||||
import com.kingsrook.qbits.objectstorage.api.Page;
|
||||
import com.kingsrook.qbits.objectstorage.api.StoredObject;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Minimal storage boundary. The application never deletes sealed objects.
|
||||
*******************************************************************************/
|
||||
public interface SealedStore
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Only positively confirmed Object Lock support is acceptable.
|
||||
*******************************************************************************/
|
||||
boolean supportsLock();
|
||||
|
||||
/*******************************************************************************
|
||||
** Preserve unknown provider capabilities for read-only setup diagnostics.
|
||||
*******************************************************************************/
|
||||
default CapabilityStatus lockCapability()
|
||||
{
|
||||
return (supportsLock() ? CapabilityStatus.SUPPORTED : CapabilityStatus.UNSUPPORTED);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Upload a file with the selected Object Lock mode and retention.
|
||||
*******************************************************************************/
|
||||
String put(String key, Path file, com.kingsrook.qbits.objectstorage.api.LockMode mode, Instant retainUntil) throws Exception;
|
||||
|
||||
/*******************************************************************************
|
||||
** Read current object metadata, or null for an absent object.
|
||||
*******************************************************************************/
|
||||
StoredObject head(String key) throws Exception;
|
||||
|
||||
/*******************************************************************************
|
||||
** Open the exact sealed version; the caller owns and closes the stream.
|
||||
*******************************************************************************/
|
||||
InputStream getVersion(String key, String versionId) throws Exception;
|
||||
|
||||
/*******************************************************************************
|
||||
** List one bounded page.
|
||||
*******************************************************************************/
|
||||
Page<ObjectSummary> list(ListRequest request) throws Exception;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.guard;
|
||||
|
||||
|
||||
import java.sql.SQLException;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Metadata defense and a value-free mapping of the database write-once error.
|
||||
*******************************************************************************/
|
||||
public class AppendOnlyGuard implements MetaDataCustomizerInterface<QTableMetaData>
|
||||
{
|
||||
public static final List<String> CORE_TABLES = List.of("audit", "auditDetail", "auditTable", "auditUser");
|
||||
public static final String MESSAGE = "Audit records are write-once and cannot be changed or deleted.";
|
||||
|
||||
/*******************************************************************************
|
||||
** Protect core and sidecar audit metadata after host customization.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QTableMetaData customizeMetaData(QInstance instance, QTableMetaData table)
|
||||
{
|
||||
if(com.kingsrook.qbits.wormaudit.model.WormAuditTables.SETTINGS.equals(table.getName()))
|
||||
{
|
||||
table.withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_DELETE);
|
||||
table.withAuditRules(new com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules()
|
||||
.withAuditLevel(com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel.FIELD));
|
||||
return (table);
|
||||
}
|
||||
if(CORE_TABLES.contains(table.getName()) || table.getName().startsWith("worm_audit_"))
|
||||
{
|
||||
table.withoutCapabilities(Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
|
||||
}
|
||||
if(table.getName().startsWith("worm_audit_"))
|
||||
{
|
||||
table.withoutCapabilities(Capability.TABLE_INSERT);
|
||||
}
|
||||
return (table);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Preserve unrelated failures, map only the documented SQLSTATE and prefix.
|
||||
*******************************************************************************/
|
||||
public static QException map(Exception error)
|
||||
{
|
||||
for(Throwable cause = error; cause != null; cause = cause.getCause())
|
||||
{
|
||||
if(cause instanceof SQLException sql && "P0001".equals(sql.getSQLState())
|
||||
&& sql.getMessage() != null && sql.getMessage().startsWith("worm_audit:"))
|
||||
{
|
||||
return (new QUserFacingException(MESSAGE));
|
||||
}
|
||||
// The PostgreSQL driver prepends ERROR: to server error messages.
|
||||
if(cause instanceof SQLException sql && "P0001".equals(sql.getSQLState())
|
||||
&& sql.getMessage() != null && sql.getMessage().startsWith("ERROR: worm_audit:"))
|
||||
{
|
||||
return (new QUserFacingException(MESSAGE));
|
||||
}
|
||||
}
|
||||
return (error instanceof QException q ? q : new QException("WORM audit operation failed", error));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.model;
|
||||
|
||||
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChildRecordListData;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A standard child-record list for a checkpoint/export/verification link range.
|
||||
** This is a record-screen relation, not an app-home widget. Reads retain user
|
||||
** permissions and use a bounded page with a filtered View All link.
|
||||
*******************************************************************************/
|
||||
public class LinkRangeWidget extends AbstractWidgetRenderer
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Render the inclusive range stored on the parent record.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public RenderWidgetOutput render(RenderWidgetInput input) throws QException
|
||||
{
|
||||
String parent = (String) input.getWidgetMetaData().getDefaultValues().get("parentTable");
|
||||
GetInput get = new GetInput().withPrimaryKey(input.getQueryParams().get("id")).withInputSource(QInputSource.USER);
|
||||
get.setTableName(parent);
|
||||
get.withPermissionEnforcement(PermissionEnforcement.ENFORCE);
|
||||
QRecord record = new GetAction().execute(get).getRecord();
|
||||
if(record == null)
|
||||
{
|
||||
throw (new QNotFoundException("Audit range record was not found"));
|
||||
}
|
||||
Long from = record.getValueLong("from_link_id");
|
||||
Long to = record.getValueLong("to_link_id");
|
||||
QQueryFilter filter = new QQueryFilter()
|
||||
.withCriteria(new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, from == null ? 1L : from))
|
||||
.withCriteria(new QFilterCriteria("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, to == null ? 0L : to))
|
||||
.withOrderBy(new QFilterOrderBy("id", true));
|
||||
CountInput count = new CountInput(WormAuditTables.CHAIN).withFilter(filter).withInputSource(QInputSource.USER);
|
||||
count.withPermissionEnforcement(PermissionEnforcement.ENFORCE);
|
||||
int total = new CountAction().execute(count).getCount();
|
||||
String path = QContext.getQInstance().getTablePath(WormAuditTables.CHAIN);
|
||||
String viewAll = path + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), StandardCharsets.UTF_8);
|
||||
QueryInput query = new QueryInput(WormAuditTables.CHAIN).withFilter(filter.clone().withLimit(100)).withInputSource(QInputSource.USER);
|
||||
query.withShouldTranslatePossibleValues(true);
|
||||
query.withPermissionEnforcement(PermissionEnforcement.ENFORCE);
|
||||
return (new RenderWidgetOutput(new ChildRecordListData("Chain links", new QueryAction().execute(query),
|
||||
QContext.getQInstance().getTable(WormAuditTables.CHAIN), path, viewAll, total)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.model;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditSettings;
|
||||
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Validate the editable singleton, including internal insert-only seeds.
|
||||
*******************************************************************************/
|
||||
public class SettingsCustomizer implements TableCustomizerInterface
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Only the fixed singleton may be inserted by the internal seed path.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> preInsert(InsertInput input, List<QRecord> records, boolean preview) throws QException
|
||||
{
|
||||
if(records.size() != 1 || !Integer.valueOf(1).equals(records.getFirst().getValueInteger("id")))
|
||||
{
|
||||
throw (new QUserFacingException("WORM Audit Settings has one row only (ID 1)."));
|
||||
}
|
||||
if(new com.kingsrook.qbits.wormaudit.chain.AuditStore(input.getTransaction()).get(WormAuditTables.SETTINGS, 1) != null)
|
||||
{
|
||||
throw (new QUserFacingException("WORM Audit Settings already exists. Edit record 1 instead of adding another record."));
|
||||
}
|
||||
return (validate(records));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Validate partial edits without replacing the existing settings.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> preUpdate(UpdateInput input, List<QRecord> records, boolean preview, Optional<List<QRecord>> oldRecords) throws QException
|
||||
{
|
||||
PermissionsHelper.checkTablePermissionThrowing(input, TablePermissionSubType.EDIT);
|
||||
return (validate(records));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reject deletion even for direct core calls where capability checks are off.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> preDelete(DeleteInput input, List<QRecord> records, boolean preview) throws QException
|
||||
{
|
||||
throw (new QUserFacingException("WORM Audit Settings cannot be deleted."));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Refresh local settings after persistence; remote JVMs retain the bounded cache.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> postUpdate(UpdateInput input, List<QRecord> records, Optional<List<QRecord>> oldRecords) throws QException
|
||||
{
|
||||
com.kingsrook.qbits.wormaudit.processes.VerificationRuns.config().invalidateSettings();
|
||||
return (records);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stamp the editor and time on every valid write; core writes FIELD audits.
|
||||
*******************************************************************************/
|
||||
private List<QRecord> validate(List<QRecord> records) throws QException
|
||||
{
|
||||
for(QRecord row : records)
|
||||
{
|
||||
try
|
||||
{
|
||||
WormAuditSettings.validate(row);
|
||||
}
|
||||
catch(IllegalArgumentException e)
|
||||
{
|
||||
throw (new QUserFacingException(e.getMessage()));
|
||||
}
|
||||
row.setValue("modify_date", Instant.now());
|
||||
row.setValue("modified_by", QContext.getQSession() == null || QContext.getQSession().getUser() == null
|
||||
? "system" : QContext.getQSession().getUser().getIdReference());
|
||||
}
|
||||
return (records);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.model;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.CustomPermissionChecker;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reading uses the full table name; editing additionally needs settings access.
|
||||
*******************************************************************************/
|
||||
public class SettingsPermissions implements CustomPermissionChecker
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Enforce table permission and the separate settings permission.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void checkPermissionsThrowing(AbstractActionInput input, MetaDataWithPermissionRules metadata) throws QPermissionDeniedException
|
||||
{
|
||||
var session = QContext.getQSession();
|
||||
boolean edit = input instanceof UpdateInput;
|
||||
if(input instanceof InsertInput || input instanceof DeleteInput || session == null
|
||||
|| !session.hasPermission(WormAuditTables.SETTINGS + (edit ? ".edit" : ".read"))
|
||||
|| (edit && !session.hasPermission("worm_audit.settings")))
|
||||
{
|
||||
throw (new QPermissionDeniedException(edit ? "Editing WORM Audit Settings requires worm_audit_settings.edit and worm_audit.settings."
|
||||
: "WORM Audit Settings requires read access; adding or deleting settings is not allowed."));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.model;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditSettings;
|
||||
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Append-only sidecar tables. Snake-case fields map directly to DDL columns.
|
||||
** Labels also supply the standard QQQ app navigation wording.
|
||||
*******************************************************************************/
|
||||
public final class WormAuditTables
|
||||
{
|
||||
public static final String CHAIN = "worm_audit_chain";
|
||||
public static final String CHECKPOINT = "worm_audit_checkpoint";
|
||||
public static final String EXPORT = "worm_audit_export";
|
||||
public static final String VERIFICATION = "worm_audit_verification";
|
||||
public static final String PURGE = "worm_audit_purge";
|
||||
public static final String SETTINGS = "worm_audit_settings";
|
||||
public static final List<String> ALL = List.of(CHAIN, CHECKPOINT, EXPORT, VERIFICATION, PURGE, SETTINGS);
|
||||
public static final List<String> NAV = List.of(VERIFICATION, EXPORT, CHECKPOINT, CHAIN, PURGE, SETTINGS);
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private WormAuditTables()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Construct full record screens, with no recursive audit writes.
|
||||
*******************************************************************************/
|
||||
public static List<QTableMetaData> tables(WormAuditQBitConfig config)
|
||||
{
|
||||
List<QTableMetaData> tables = new ArrayList<>();
|
||||
tables.add(table(config, CHAIN, "Audit Chain Link", "id:L kind:S audit_id:L detail_ids:T prev_hash:S link_hash:S chained_at:D"));
|
||||
tables.add(table(config, CHECKPOINT, "Checkpoint", "id:L from_link_id:L to_link_id:L link_count:L head_hash:S merkle_root:S algorithm:S key_id:S public_key:T signature:T created_at:D"));
|
||||
tables.add(table(config, EXPORT, "Sealed Export", "id:L from_link_id:L to_link_id:L from_checkpoint_id:L to_checkpoint_id:L bucket_alias:S object_prefix:T object_versions:T manifest_key:T manifest_sha256:S prev_manifest_sha256:S object_count:L byte_count:L lock_mode:S retain_until:D sealed_at:D"));
|
||||
tables.add(table(config, VERIFICATION, "Verification", "id:L kind:S trigger:S started_at:D finished_at:D from_link_id:L to_link_id:L links_checked:L outcome:S break_count:L first_break_link_id:L first_break_audit_id:L first_break_reason:T export_id:L run_by:S"));
|
||||
tables.add(table(config, PURGE, "Purge", "id:L requested_at:D requested_by:S cutoff:D retention_interval:S outcome:S reason:T audits_deleted:L details_deleted:L links_deleted:L anchor_link_id:L anchor_hash:S"));
|
||||
tables.add(settings(config));
|
||||
return (tables);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Editable, FIELD-audited singleton with standard QQQ record sections.
|
||||
*******************************************************************************/
|
||||
public static QTableMetaData settings(WormAuditQBitConfig config)
|
||||
{
|
||||
QTableMetaData table = new QTableMetaData().withName(SETTINGS).withLabel("WORM Audit Settings")
|
||||
.withBackendName(config.getBackendName()).withBackendDetails(new RDBMSTableBackendDetails().withTableName(SETTINGS))
|
||||
.withPrimaryKeyField("id").withRecordLabelFields("id").withRecordLabelFormat("WORM Audit Settings")
|
||||
.withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD))
|
||||
.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_INSERT_EDIT_DELETE_PERMISSIONS)
|
||||
.withPermissionBaseName(SETTINGS).withCustomPermissionChecker(new QCodeReference(SettingsPermissions.class)))
|
||||
.withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_DELETE)
|
||||
.withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(SettingsCustomizer.class))
|
||||
.withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, new QCodeReference(SettingsCustomizer.class))
|
||||
.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, new QCodeReference(SettingsCustomizer.class))
|
||||
.withCustomizer(TableCustomizers.PRE_DELETE_RECORD, new QCodeReference(SettingsCustomizer.class))
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withLabel("ID").withBackendName("id").withIsEditable(false).withDefaultValue(1));
|
||||
var defaults = WormAuditSettings.defaults(config);
|
||||
table.withField(new QFieldMetaData("audit_retention_years", QFieldType.INTEGER).withBackendName("audit_retention_years").withLabel("Audit retention years")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("audit_retention_years")));
|
||||
table.withField(new QFieldMetaData("export_lock_mode", QFieldType.STRING).withBackendName("export_lock_mode").withLabel("Export lock mode")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("export_lock_mode")));
|
||||
table.withField(new QFieldMetaData("export_retention_days", QFieldType.INTEGER).withBackendName("export_retention_days").withLabel("Export retention days")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("export_retention_days")));
|
||||
table.withField(new QFieldMetaData("checkpoint_every_links", QFieldType.INTEGER).withBackendName("checkpoint_every_links").withLabel("Checkpoint every links")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("checkpoint_every_links")));
|
||||
table.withField(new QFieldMetaData("checkpoint_every_minutes", QFieldType.INTEGER).withBackendName("checkpoint_every_minutes").withLabel("Checkpoint every minutes")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("checkpoint_every_minutes")));
|
||||
table.withField(new QFieldMetaData("chain_interval_seconds", QFieldType.INTEGER).withBackendName("chain_interval_seconds").withLabel("Chain interval seconds")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("chain_interval_seconds")));
|
||||
table.withField(new QFieldMetaData("settle_horizon_seconds", QFieldType.INTEGER).withBackendName("settle_horizon_seconds").withLabel("Settle horizon seconds")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("settle_horizon_seconds")));
|
||||
table.withField(new QFieldMetaData("late_detail_lookback_hours", QFieldType.INTEGER).withBackendName("late_detail_lookback_hours").withLabel("Late detail lookback hours")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("late_detail_lookback_hours")));
|
||||
table.withField(new QFieldMetaData("export_cron", QFieldType.STRING).withBackendName("export_cron").withLabel("Export schedule (UTC)")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("export_cron")));
|
||||
table.withField(new QFieldMetaData("verify_cron", QFieldType.STRING).withBackendName("verify_cron").withLabel("Verify schedule (UTC)")
|
||||
.withIsRequired(true).withDefaultValue(defaults.getValue("verify_cron")));
|
||||
table.getField("export_lock_mode").withInlinePossibleValueSource(new QPossibleValueSource().withName(SETTINGS + "_lock_mode")
|
||||
.withType(QPossibleValueSourceType.ENUM).withIdType(QFieldType.STRING).withEnumValues(List.of(
|
||||
new QPossibleValue<>("GOVERNANCE", "GOVERNANCE"), new QPossibleValue<>("COMPLIANCE", "COMPLIANCE"))));
|
||||
for(String field : List.of("export_cron", "verify_cron", "chain_interval_seconds"))
|
||||
{
|
||||
table.getField(field).withHelpContent(new QHelpContent("Takes effect after restart. Cron schedules use UTC and Quartz syntax."));
|
||||
}
|
||||
table.getField("audit_retention_years").withHelpContent(new QHelpContent("Used for the next eligible purge date. The database purge interval changes only through a reviewed migration."));
|
||||
table.withField(new QFieldMetaData("notes", QFieldType.TEXT).withLabel("Notes").withBackendName("notes"));
|
||||
table.withField(new QFieldMetaData("modify_date", QFieldType.DATE_TIME).withLabel("Modified").withBackendName("modify_date").withIsEditable(false));
|
||||
table.withField(new QFieldMetaData("modified_by", QFieldType.STRING).withLabel("Modified by").withBackendName("modified_by").withIsEditable(false));
|
||||
table.withSection(new QFieldSection("retention", "Retention", new QIcon("lock"), Tier.T1,
|
||||
List.of("id", "audit_retention_years", "export_lock_mode", "export_retention_days")).withGridColumns(12));
|
||||
table.withSection(new QFieldSection("cadence", "Chain and checkpoints", new QIcon("link"), Tier.T2,
|
||||
List.of("checkpoint_every_links", "checkpoint_every_minutes", "settle_horizon_seconds", "late_detail_lookback_hours")).withGridColumns(12));
|
||||
table.withSection(new QFieldSection("schedules", "Schedules (after restart)", new QIcon("schedule"), Tier.T2,
|
||||
List.of("chain_interval_seconds", "export_cron", "verify_cron")).withGridColumns(12));
|
||||
table.withSection(new QFieldSection("changes", "Changes", new QIcon("edit_note"), Tier.T2,
|
||||
List.of("notes", "modify_date", "modified_by")).withGridColumns(12));
|
||||
return (table);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Common metadata for a process-written table.
|
||||
*******************************************************************************/
|
||||
private static QTableMetaData table(WormAuditQBitConfig config, String name, String label, String fields)
|
||||
{
|
||||
QTableMetaData table = new QTableMetaData().withName(name).withLabel(navigationLabel(name))
|
||||
.withBackendName(config.getBackendName()).withBackendDetails(new RDBMSTableBackendDetails().withTableName(name))
|
||||
.withPrimaryKeyField("id").withRecordLabelFields("id").withRecordLabelFormat(label + " #%s")
|
||||
.withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.NONE))
|
||||
.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_INSERT_EDIT_DELETE_PERMISSIONS).withPermissionBaseName(name));
|
||||
for(String spec : fields.split(" "))
|
||||
{
|
||||
String[] parts = spec.split(":");
|
||||
QFieldType type = switch(parts[1])
|
||||
{
|
||||
case "L" -> QFieldType.LONG;
|
||||
case "D" -> QFieldType.DATE_TIME;
|
||||
case "T" -> QFieldType.TEXT;
|
||||
default -> QFieldType.STRING;
|
||||
};
|
||||
table.withField(new QFieldMetaData(parts[0], type).withBackendName(parts[0]).withLabel(fieldLabel(parts[0])));
|
||||
}
|
||||
for(String reference : List.of("from_link_id", "to_link_id", "first_break_link_id", "anchor_link_id"))
|
||||
{
|
||||
if(table.getFields().containsKey(reference))
|
||||
{
|
||||
table.getField(reference).withPossibleValueSourceName(CHAIN);
|
||||
}
|
||||
}
|
||||
for(String reference : List.of("from_checkpoint_id", "to_checkpoint_id"))
|
||||
{
|
||||
if(table.getFields().containsKey(reference))
|
||||
{
|
||||
table.getField(reference).withPossibleValueSourceName(CHECKPOINT);
|
||||
}
|
||||
}
|
||||
if(table.getFields().containsKey("export_id"))
|
||||
{
|
||||
table.getField("export_id").withPossibleValueSourceName(EXPORT);
|
||||
}
|
||||
switch(name)
|
||||
{
|
||||
case CHAIN -> table.withRecordLabelFormat("Link #%s");
|
||||
case EXPORT -> table.withRecordLabelFormatAndFields("Export #%s (links %s–%s)", "id", "from_link_id", "to_link_id");
|
||||
case VERIFICATION, PURGE -> table.withRecordLabelFormatAndFields(label + " #%s (%s)", "id", "outcome");
|
||||
default ->
|
||||
{
|
||||
}
|
||||
}
|
||||
if(table.getFields().containsKey("kind"))
|
||||
{
|
||||
table.getField("kind").withInlinePossibleValueSource(new QPossibleValueSource().withName(name + "_kind")
|
||||
.withType(QPossibleValueSourceType.ENUM).withIdType(QFieldType.STRING).withEnumValues(List.of(new QPossibleValue<>("AUDIT", "Audit"),
|
||||
new QPossibleValue<>("LATE_DETAIL", "Late detail"), new QPossibleValue<>("CHAIN", "Chain"),
|
||||
new QPossibleValue<>("SEALED_EXPORT", "Sealed export"))));
|
||||
}
|
||||
if(table.getFields().containsKey("outcome"))
|
||||
{
|
||||
table.getField("outcome").withInlinePossibleValueSource(new QPossibleValueSource().withName(name + "_outcome")
|
||||
.withType(QPossibleValueSourceType.ENUM).withIdType(QFieldType.STRING).withEnumValues(List.of(new QPossibleValue<>("PASS", "Pass"),
|
||||
new QPossibleValue<>("FAIL", "Fail"), new QPossibleValue<>("PURGED", "Purged"), new QPossibleValue<>("REFUSED", "Refused"))));
|
||||
}
|
||||
if(VERIFICATION.equals(name))
|
||||
{
|
||||
table.withSection(new QFieldSection("result", "Result", new QIcon("fact_check"), Tier.T2,
|
||||
List.of("id", "outcome", "break_count", "first_break_link_id", "first_break_audit_id", "first_break_reason")).withGridColumns(12));
|
||||
table.withSection(new QFieldSection("scope", "Scope", new QIcon("link"), Tier.T2,
|
||||
List.of("kind", "trigger", "from_link_id", "to_link_id", "links_checked", "export_id")).withGridColumns(12));
|
||||
table.withSection(new QFieldSection("run", "Run", new QIcon("schedule"), Tier.T2,
|
||||
List.of("started_at", "finished_at", "run_by")).withGridColumns(12));
|
||||
}
|
||||
else
|
||||
{
|
||||
table.withSection(new QFieldSection("record", label, new QIcon("fact_check"), Tier.T1,
|
||||
new ArrayList<>(table.getFields().keySet())).withGridColumns(12));
|
||||
}
|
||||
if(List.of(CHECKPOINT, EXPORT, VERIFICATION).contains(name))
|
||||
{
|
||||
table.withSection(new QFieldSection("links", new QIcon("link"), Tier.T2).withLabel("Chain links")
|
||||
.withWidgetName(name + "_links").withGridColumns(12));
|
||||
}
|
||||
return (table);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Core RC.5 shares the table label with its navigation tree node.
|
||||
*******************************************************************************/
|
||||
public static String navigationLabel(String name)
|
||||
{
|
||||
return (switch(name)
|
||||
{
|
||||
case SETTINGS -> "Settings";
|
||||
case CHAIN -> "Audit Chain";
|
||||
case CHECKPOINT -> "Checkpoints";
|
||||
case EXPORT -> "Sealed Exports";
|
||||
case VERIFICATION -> "Verifications";
|
||||
default -> "Purges";
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Plain record labels independent of physical column names.
|
||||
*******************************************************************************/
|
||||
private static String fieldLabel(String name)
|
||||
{
|
||||
return (switch(name)
|
||||
{
|
||||
case "id" -> "ID";
|
||||
case "kind" -> "Kind";
|
||||
case "audit_id" -> "Audit";
|
||||
case "detail_ids" -> "Detail ids";
|
||||
case "prev_hash" -> "Previous hash";
|
||||
case "link_hash" -> "Link hash";
|
||||
case "chained_at" -> "Chained";
|
||||
case "from_link_id" -> "From link";
|
||||
case "to_link_id" -> "To link";
|
||||
case "link_count" -> "Links";
|
||||
case "head_hash" -> "Head hash";
|
||||
case "merkle_root" -> "Merkle root";
|
||||
case "algorithm" -> "Algorithm";
|
||||
case "key_id" -> "Key ID";
|
||||
case "public_key" -> "Public key";
|
||||
case "signature" -> "Signature";
|
||||
case "created_at" -> "Created";
|
||||
case "from_checkpoint_id" -> "From checkpoint";
|
||||
case "to_checkpoint_id" -> "To checkpoint";
|
||||
case "bucket_alias" -> "Bucket alias";
|
||||
case "object_prefix" -> "Object prefix";
|
||||
case "object_versions" -> "Object versions";
|
||||
case "manifest_key" -> "Manifest key";
|
||||
case "manifest_sha256" -> "Manifest SHA-256";
|
||||
case "prev_manifest_sha256" -> "Previous manifest SHA-256";
|
||||
case "object_count" -> "Objects";
|
||||
case "byte_count" -> "Bytes";
|
||||
case "lock_mode" -> "Lock mode";
|
||||
case "retain_until" -> "Retain until";
|
||||
case "sealed_at" -> "Sealed";
|
||||
case "trigger" -> "Trigger";
|
||||
case "started_at" -> "Started";
|
||||
case "finished_at" -> "Finished";
|
||||
case "links_checked" -> "Links checked";
|
||||
case "outcome" -> "Outcome";
|
||||
case "break_count" -> "Breaks";
|
||||
case "first_break_link_id" -> "First break link";
|
||||
case "first_break_audit_id" -> "First break audit";
|
||||
case "first_break_reason" -> "First break";
|
||||
case "export_id" -> "Sealed export";
|
||||
case "run_by" -> "Run by";
|
||||
case "requested_at" -> "Requested";
|
||||
case "requested_by" -> "Requested by";
|
||||
case "cutoff" -> "Cutoff";
|
||||
case "retention_interval" -> "Retention interval";
|
||||
case "reason" -> "Reason";
|
||||
case "audits_deleted" -> "Audits deleted";
|
||||
case "details_deleted" -> "Details deleted";
|
||||
case "links_deleted" -> "Links deleted";
|
||||
case "anchor_link_id" -> "Anchor link";
|
||||
case "anchor_hash" -> "Anchor hash";
|
||||
default -> throw new IllegalArgumentException("Missing field label: " + name);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,377 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.processes;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedExportVerifier;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedExporter;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedStore;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.UseOtherPermissionNameCustomPermissionChecker;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunInput;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunOutput;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Standard Review/Result actions for sealing and verifying exports.
|
||||
*******************************************************************************/
|
||||
public final class ExportProcesses
|
||||
{
|
||||
public static final String EXPORT = "worm_audit_export_now";
|
||||
public static final String VERIFY = "worm_audit_verify_export";
|
||||
public static final String DAILY = "worm_audit_export_daily";
|
||||
public static final String PERMISSION = "worm_audit.export";
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private ExportProcesses()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Export is a table action; verification requires one or more selected exports.
|
||||
*******************************************************************************/
|
||||
public static List<QProcessMetaData> processes(WormAuditQBitConfig config)
|
||||
{
|
||||
var settings = config.effectiveSettings();
|
||||
Map<String, Serializable> values = new HashMap<>();
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, WormAuditTables.EXPORT);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, WormAuditTables.EXPORT);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION, true);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true);
|
||||
QProcessMetaData export = StreamedETLWithFrontendProcess.defineProcessMetaData(ExportExtract.class, ExportReview.class, ExportLoad.class, new HashMap<>(values))
|
||||
.withName(EXPORT).withLabel("Export now").withTableName(WormAuditTables.EXPORT).withPermissionRules(permission(PERMISSION));
|
||||
QProcessMetaData verify = StreamedETLWithFrontendProcess.defineProcessMetaData(VerifyExtract.class, VerifyReview.class, VerifyLoad.class, new HashMap<>(values))
|
||||
.withName(VERIFY).withLabel("Verify sealed export").withTableName(WormAuditTables.EXPORT).withMinInputRecords(1)
|
||||
.withPermissionRules(permission(WormAuditProcesses.PERMISSION));
|
||||
QProcessMetaData daily = new QProcessMetaData().withName(DAILY).withLabel("Daily sealed audit export")
|
||||
.withStep(new QBackendStepMetaData().withName("export").withCode(new QCodeReference(Daily.class)))
|
||||
.withPermissionRules(permission(PERMISSION)).withSchedule(new QScheduleMetaData().withSchedulerName(config.getSchedulerName())
|
||||
.withCronExpression(settings.export_cron()).withCronTimeZoneId("UTC"));
|
||||
return (List.of(export, verify, daily));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reuse the phase (a) explicit permission-name checker.
|
||||
*******************************************************************************/
|
||||
private static QPermissionRules permission(String name)
|
||||
{
|
||||
return (new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION)
|
||||
.withCustomPermissionChecker(UseOtherPermissionNameCustomPermissionChecker.build(name)));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Plain text summaries must not acquire the framework's default zero count.
|
||||
*******************************************************************************/
|
||||
private static ProcessSummaryLine text(Status status, String message)
|
||||
{
|
||||
ProcessSummaryLine line = new ProcessSummaryLine(status, message);
|
||||
line.setCount(null);
|
||||
return (line);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One synthetic work item allows the first export on an empty export table.
|
||||
*******************************************************************************/
|
||||
public static class ExportExtract extends AbstractExtractStep
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Count the single export operation, not previous export rows.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public Integer doCount(RunBackendStepInput input)
|
||||
{
|
||||
return (1);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Feed one work item through the standard stream.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
getRecordPipe().addRecord(new QRecord().withValue("id", 0L));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Review is read-only; the writer reselects the suffix after confirmation.
|
||||
*******************************************************************************/
|
||||
public static class ExportReview extends AbstractTransformStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private String message;
|
||||
private boolean failed;
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the planned range once without uploading or inserting.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
try
|
||||
{
|
||||
message = SealedExporter.scope(new AuditStore(null)).review();
|
||||
failed = false;
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
message = "Cannot select a contiguous export range";
|
||||
failed = true;
|
||||
}
|
||||
output.getRecords().addAll(input.getRecords());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Show the range, including the explicit no-checkpoint case.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
return (new ArrayList<>(List.of(text(failed ? Status.ERROR : Status.INFO, message == null ? "Nothing to seal: no new checkpoint" : message))));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Seal once after the stream is complete.
|
||||
*******************************************************************************/
|
||||
public static class ExportLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final SealedStore storage;
|
||||
private SealedExporter.Result result;
|
||||
|
||||
/*******************************************************************************
|
||||
** Framework constructor.
|
||||
*******************************************************************************/
|
||||
public ExportLoad()
|
||||
{
|
||||
this(null);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Storage-injected action for unit tests.
|
||||
*******************************************************************************/
|
||||
public ExportLoad(SealedStore storage)
|
||||
{
|
||||
this.storage = storage;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Forward the work item without retaining content.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output)
|
||||
{
|
||||
output.getRecords().addAll(input.getRecords());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The same execution path as the daily schedule.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void postRun(BackendStepPostRunInput input, BackendStepPostRunOutput output)
|
||||
{
|
||||
result = new SealedExporter(VerificationRuns.config(), storage).run("USER", Instant.now());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Result text is value-free even when the provider fails.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
return (result == null ? new ArrayList<>() : new ArrayList<>(List.of(text(result.failed() ? Status.ERROR : Status.OK, result.message()))));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Respect core's selected-record/filter interpretation; no implicit all-export run.
|
||||
*******************************************************************************/
|
||||
public static class VerifyExtract extends ExtractViaQueryStep
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Deterministic selection with full text fields from the export table.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QQueryFilter getQueryFilter(RunBackendStepInput input) throws QException
|
||||
{
|
||||
input.addValue(FIELD_SOURCE_TABLE, WormAuditTables.EXPORT);
|
||||
QQueryFilter filter = super.getQueryFilter(input).clone();
|
||||
filter.setOrderBys(List.of(new QFilterOrderBy("id", true)));
|
||||
return (filter);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Review selected exports without reading their content.
|
||||
*******************************************************************************/
|
||||
public static class VerifyReview extends AbstractTransformStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private long count;
|
||||
|
||||
/*******************************************************************************
|
||||
** Count and forward the selected export records.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output)
|
||||
{
|
||||
count += input.getRecords().size();
|
||||
output.getRecords().addAll(input.getRecords());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One Review summary.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
return (new ArrayList<>(List.of(text(Status.INFO, WormAuditVerifier.number(count) + " sealed exports will be verified"))));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify each selected export and persist its independent result.
|
||||
*******************************************************************************/
|
||||
public static class VerifyLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final SealedStore storage;
|
||||
private final ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
|
||||
/*******************************************************************************
|
||||
** Framework constructor.
|
||||
*******************************************************************************/
|
||||
public VerifyLoad()
|
||||
{
|
||||
this(null);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Storage-injected action for unit tests.
|
||||
*******************************************************************************/
|
||||
public VerifyLoad(SealedStore storage)
|
||||
{
|
||||
this.storage = storage;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Process the current page without retaining exported content.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
for(QRecord selected : input.getRecords())
|
||||
{
|
||||
QRecord row = new AuditStore(null).get(WormAuditTables.EXPORT, selected.getValueLong("id"));
|
||||
if(row == null)
|
||||
{
|
||||
lines.add(text(Status.ERROR, "Selected export " + selected.getValueLong("id") + " is missing"));
|
||||
continue;
|
||||
}
|
||||
var report = new SealedExportVerifier(VerificationRuns.config(), storage).verify(row, "USER");
|
||||
if(report.result().passed())
|
||||
{
|
||||
lines.add(text(Status.OK, WormAuditVerifier.number(report.result().getLinksChecked()) + " links verified in export " + row.getValueLong("id")));
|
||||
}
|
||||
report.result().getBreaks().forEach(b -> lines.add(text(Status.ERROR, b.reason())));
|
||||
if(report.purged() > 0 && report.result().passed())
|
||||
{
|
||||
lines.add(text(Status.INFO, WormAuditVerifier.number(report.purged()) + " links verified from the sealed copy only (purged from the live database)"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exact per-link differences and purged-copy information.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Daily export sends heartbeat even when sealing fails.
|
||||
*******************************************************************************/
|
||||
public static class Daily implements BackendStep
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Run the exporter with the installed host configuration.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output)
|
||||
{
|
||||
var result = new SealedExporter(VerificationRuns.config()).run("SCHEDULE", Instant.now());
|
||||
output.addValue("outcome", result.failed() ? "FAIL" : "PASS");
|
||||
output.addValue("result", result.message());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.processes;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.VerificationResult;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertRecordRef;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertSignal;
|
||||
import com.kingsrook.qqq.backend.core.alerts.QAlerts;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verification run persistence and the core alerts SPI. Both UI and schedule
|
||||
** write exactly one immutable result row; only schedules change alert state.
|
||||
*******************************************************************************/
|
||||
public final class VerificationRuns
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private VerificationRuns()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve this instance's configuration, never a global cross-instance cache.
|
||||
*******************************************************************************/
|
||||
public static WormAuditQBitConfig config()
|
||||
{
|
||||
return (QContext.getQInstance().getQBits().values().stream().map(q -> q.getConfig()).filter(WormAuditQBitConfig.class::isInstance)
|
||||
.map(WormAuditQBitConfig.class::cast).findFirst().orElseThrow(() -> new IllegalStateException("WORM audit qbit is not installed")));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify and persist, sending a heartbeat even if a scheduled run errors.
|
||||
*******************************************************************************/
|
||||
public static VerificationResult run(Set<Long> selection, String trigger) throws QException
|
||||
{
|
||||
boolean scheduled = "SCHEDULE".equals(trigger);
|
||||
try
|
||||
{
|
||||
VerificationResult result = new WormAuditVerifier(config()).verify(selection);
|
||||
QRecord row = save(result, trigger);
|
||||
if(scheduled)
|
||||
{
|
||||
if(result.passed())
|
||||
{
|
||||
QAlerts.resolve("wormaudit.chain.break", "chain", "Audit chain verification passed");
|
||||
}
|
||||
else
|
||||
{
|
||||
QAlerts.raise(new AlertSignal("wormaudit.chain.break").withDedupKey("chain")
|
||||
.withTitle(result.getBreaks().getFirst().reason()).withRecordRef(new AlertRecordRef(WormAuditTables.VERIFICATION, row.getValue("id"))));
|
||||
}
|
||||
}
|
||||
return (result);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if(scheduled)
|
||||
{
|
||||
QAlerts.heartbeat("wormaudit.verify.daily");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Persist identifiers and a value-free first reason, never audit content.
|
||||
*******************************************************************************/
|
||||
public static QRecord save(VerificationResult result, String trigger) throws QException
|
||||
{
|
||||
return (save(result, trigger, null));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Append a chain or sealed-export result using the same immutable schema.
|
||||
*******************************************************************************/
|
||||
public static QRecord save(VerificationResult result, String trigger, Long exportId) throws QException
|
||||
{
|
||||
return (save(result, trigger, exportId, Instant.now()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Clock-injected persistence for genuine historical verification runs.
|
||||
*******************************************************************************/
|
||||
public static QRecord save(VerificationResult result, String trigger, Long exportId, Instant finishedAt) throws QException
|
||||
{
|
||||
String user = QContext.getQSession() == null || QContext.getQSession().getUser() == null ? "system" : QContext.getQSession().getUser().getIdReference();
|
||||
QRecord row = new QRecord().withValue("kind", exportId == null ? "CHAIN" : "SEALED_EXPORT").withValue("export_id", exportId).withValue("trigger", trigger)
|
||||
.withValue("started_at", result.getStartedAt()).withValue("finished_at", finishedAt)
|
||||
.withValue("from_link_id", result.getFromLinkId()).withValue("to_link_id", result.getToLinkId())
|
||||
.withValue("links_checked", result.getLinksChecked()).withValue("outcome", result.passed() ? "PASS" : "FAIL")
|
||||
.withValue("break_count", result.getBreakCount()).withValue("run_by", user);
|
||||
if(!result.passed())
|
||||
{
|
||||
var first = result.getBreaks().getFirst();
|
||||
row.withValue("first_break_link_id", first.linkId()).withValue("first_break_audit_id", first.auditId()).withValue("first_break_reason", first.reason());
|
||||
}
|
||||
return (new AuditStore(null).insert(WormAuditTables.VERIFICATION, List.of(row)).getFirst());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,313 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.processes;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.VerificationResult;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.UseOtherPermissionNameCustomPermissionChecker;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunInput;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunOutput;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.CouldNotFindQueryFilterForExtractStepException;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Standard streamed ETL Review/Result action plus two scheduled-only processes.
|
||||
** No process is an app navigation entry.
|
||||
*******************************************************************************/
|
||||
public final class WormAuditProcesses
|
||||
{
|
||||
public static final String VERIFY = "worm_audit_verify_chain";
|
||||
public static final String CHAIN = "worm_audit_chain_tick";
|
||||
public static final String DAILY = "worm_audit_verify_daily";
|
||||
public static final String PERMISSION = "worm_audit.verify";
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private WormAuditProcesses()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Metadata for table/record verification and scheduled background work.
|
||||
*******************************************************************************/
|
||||
public static List<QProcessMetaData> processes(WormAuditQBitConfig config)
|
||||
{
|
||||
var settings = config.effectiveSettings();
|
||||
Map<String, Serializable> values = new HashMap<>();
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, WormAuditTables.CHAIN);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, WormAuditTables.CHAIN);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION, true);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, "Verify live audit content, link continuity and signed checkpoints.");
|
||||
QProcessMetaData verify = StreamedETLWithFrontendProcess.defineProcessMetaData(Extract.class, Review.class, Load.class, values)
|
||||
.withName(VERIFY).withLabel("Verify audit chain").withTableName(WormAuditTables.CHAIN).withPermissionRules(permission());
|
||||
QProcessMetaData chain = new QProcessMetaData().withName(CHAIN).withLabel("Chain settled audit records")
|
||||
.withStep(new QBackendStepMetaData().withName("chain").withCode(new QCodeReference(ChainTick.class)))
|
||||
.withPermissionRules(permission()).withSchedule(new QScheduleMetaData().withSchedulerName(config.getSchedulerName())
|
||||
.withRepeatSeconds(settings.chain_interval_seconds()).withInitialDelaySeconds(settings.chain_interval_seconds()));
|
||||
QProcessMetaData daily = new QProcessMetaData().withName(DAILY).withLabel("Daily audit chain verification")
|
||||
.withStep(new QBackendStepMetaData().withName("verify").withCode(new QCodeReference(Daily.class)))
|
||||
.withPermissionRules(permission()).withSchedule(new QScheduleMetaData().withSchedulerName(config.getSchedulerName())
|
||||
.withCronExpression(settings.verify_cron()).withCronTimeZoneId("UTC"));
|
||||
return (List.of(verify, chain, daily));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One explicit permission, not process-name access suffixes.
|
||||
*******************************************************************************/
|
||||
private static QPermissionRules permission()
|
||||
{
|
||||
return (new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION)
|
||||
.withCustomPermissionChecker(UseOtherPermissionNameCustomPermissionChecker.build(PERMISSION)));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Use core selection/filter interpretation, with a full-chain default. An
|
||||
** empty frontend filter means a full run, so completeness is still checked.
|
||||
*******************************************************************************/
|
||||
public static class Extract extends ExtractViaQueryStep
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Extract links in id order and make the effective scope available to Load.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QQueryFilter getQueryFilter(RunBackendStepInput input) throws QException
|
||||
{
|
||||
input.addValue(FIELD_SOURCE_TABLE, WormAuditTables.CHAIN);
|
||||
QQueryFilter filter;
|
||||
try
|
||||
{
|
||||
filter = super.getQueryFilter(input).clone();
|
||||
}
|
||||
catch(CouldNotFindQueryFilterForExtractStepException expected)
|
||||
{
|
||||
filter = new QQueryFilter();
|
||||
}
|
||||
filter.setOrderBys(List.of(new QFilterOrderBy("id", true)));
|
||||
return (filter);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Review counts the streamed selection without verifying or writing rows.
|
||||
*******************************************************************************/
|
||||
public static class Review extends AbstractTransformStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private long count;
|
||||
private long min = Long.MAX_VALUE;
|
||||
private long max = Long.MIN_VALUE;
|
||||
private Long checkpoints;
|
||||
|
||||
/*******************************************************************************
|
||||
** Accumulate the scope and forward links to the loader.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
for(QRecord record : input.getRecords())
|
||||
{
|
||||
count++;
|
||||
min = Math.min(min, record.getValueLong("id"));
|
||||
max = Math.max(max, record.getValueLong("id"));
|
||||
}
|
||||
checkpoints = null;
|
||||
output.getRecords().addAll(input.getRecords());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Standard Review line, including the link range and checkpoint count.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
if(checkpoints == null)
|
||||
{
|
||||
try
|
||||
{
|
||||
CountInput input = new CountInput(WormAuditTables.CHECKPOINT).withFilter(new QQueryFilter(
|
||||
new QFilterCriteria("from_link_id", QCriteriaOperator.LESS_THAN_OR_EQUALS, max),
|
||||
new QFilterCriteria("to_link_id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, min)));
|
||||
input.withPermissionBypass("WORM audit internal review");
|
||||
checkpoints = count == 0 ? 0L : new CountAction().execute(input).getCount().longValue();
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
return (new ArrayList<>(List.of(new ProcessSummaryLine(Status.ERROR, "Cannot count overlapping audit checkpoints"))));
|
||||
}
|
||||
}
|
||||
String range = count == 0 ? "none" : WormAuditVerifier.number(min) + "–" + WormAuditVerifier.number(max);
|
||||
return (new ArrayList<>(List.of(new ProcessSummaryLine(Status.INFO, WormAuditVerifier.number(count)
|
||||
+ " chain links will be verified (links " + range + ", " + WormAuditVerifier.number(checkpoints) + " checkpoints)"))));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Load collects selected ids, then performs one snapshot verification in
|
||||
** postRun and appends one result row, including for an empty chain.
|
||||
*******************************************************************************/
|
||||
public static class Load extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private Set<Long> ids;
|
||||
private VerificationResult result;
|
||||
|
||||
/*******************************************************************************
|
||||
** Retain ids only, not audit content, across streamed pages.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
if(selected(new Extract().getQueryFilter(input)))
|
||||
{
|
||||
if(ids == null)
|
||||
{
|
||||
ids = new HashSet<>();
|
||||
}
|
||||
input.getRecords().forEach(r -> ids.add(r.getValueLong("id")));
|
||||
}
|
||||
output.getRecords().addAll(input.getRecords());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Run once after all pages. Explicit filters are selection runs.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void postRun(BackendStepPostRunInput input, BackendStepPostRunOutput output) throws QException
|
||||
{
|
||||
QQueryFilter filter = new Extract().getQueryFilter(input);
|
||||
if(selected(filter) && ids == null)
|
||||
{
|
||||
// Some callers supply the effective selection only to postRun.
|
||||
ids = new HashSet<>();
|
||||
new AuditStore(null).query(WormAuditTables.CHAIN, filter).forEach(r -> ids.add(r.getValueLong("id")));
|
||||
}
|
||||
result = VerificationRuns.run(selected(filter) ? ids : null, "USER");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Empty filters preserve full completeness verification and retain no ids.
|
||||
*******************************************************************************/
|
||||
private boolean selected(QQueryFilter filter)
|
||||
{
|
||||
return ((filter.getCriteria() != null && !filter.getCriteria().isEmpty())
|
||||
|| (filter.getSubFilters() != null && !filter.getSubFilters().isEmpty())
|
||||
|| filter.getLimit() != null || (filter.getSkip() != null && filter.getSkip() > 0));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One ERROR per exact break; warnings remain visible on a passing run.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
if(result == null)
|
||||
{
|
||||
return (lines);
|
||||
}
|
||||
if(result.passed())
|
||||
{
|
||||
lines.add(new ProcessSummaryLine(Status.OK, WormAuditVerifier.number(result.getLinksChecked()) + " links verified"));
|
||||
}
|
||||
result.getBreaks().forEach(b -> lines.add(new ProcessSummaryLine(Status.ERROR, b.reason())));
|
||||
result.getWarnings().forEach(w -> lines.add(new ProcessSummaryLine(Status.WARNING, w)));
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Scheduled writer tick.
|
||||
*******************************************************************************/
|
||||
public static class ChainTick implements BackendStep
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Append settled work under the shared writer lock.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
output.addValue("linksChained", new WormAuditChainer(VerificationRuns.config()).chainOnce().links());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Daily verification, including alerts and heartbeat.
|
||||
*******************************************************************************/
|
||||
public static class Daily implements BackendStep
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Verify the full settled chain.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
output.addValue("outcome", VerificationRuns.run(null, "SCHEDULE").passed() ? "PASS" : "FAIL");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.verify;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Ordered, value-free findings. Break identifiers are separately available for
|
||||
** record navigation and alert correlation without parsing human-readable text.
|
||||
*******************************************************************************/
|
||||
public class VerificationResult
|
||||
{
|
||||
/*******************************************************************************
|
||||
** One break; null ids apply to coverage and checkpoint findings.
|
||||
*******************************************************************************/
|
||||
public record Break(Long linkId, Long auditId, String reason)
|
||||
{
|
||||
}
|
||||
|
||||
private final List<Break> breaks = new ArrayList<>();
|
||||
private final List<String> warnings = new ArrayList<>();
|
||||
private final Instant startedAt;
|
||||
private long breakCount;
|
||||
private long linksChecked;
|
||||
private Long fromLinkId;
|
||||
private Long toLinkId;
|
||||
|
||||
/*******************************************************************************
|
||||
** Use the production clock by default.
|
||||
*******************************************************************************/
|
||||
public VerificationResult()
|
||||
{
|
||||
this(Instant.now());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Bind the verification start to the caller's clock.
|
||||
*******************************************************************************/
|
||||
public VerificationResult(Instant startedAt)
|
||||
{
|
||||
this.startedAt = startedAt;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Add a finding in deterministic verification order.
|
||||
*******************************************************************************/
|
||||
public void add(Long linkId, Long auditId, String reason)
|
||||
{
|
||||
breakCount++;
|
||||
if(breaks.size() < 1000)
|
||||
{
|
||||
breaks.add(new Break(linkId, auditId, reason));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Count additional SQL offenders without retaining their content.
|
||||
*******************************************************************************/
|
||||
public void omitted(long count)
|
||||
{
|
||||
breakCount += count;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Total findings, including diagnostics omitted by the bounded display limit.
|
||||
*******************************************************************************/
|
||||
public long getBreakCount()
|
||||
{
|
||||
return (breakCount);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Keep warnings bounded even when many checkpoint ranges have been purged.
|
||||
*******************************************************************************/
|
||||
public void warn(String warning)
|
||||
{
|
||||
if(warnings.size() < 100)
|
||||
{
|
||||
warnings.add(warning);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Record the scope that was checked.
|
||||
*******************************************************************************/
|
||||
public void scope(long count, Long from, Long to)
|
||||
{
|
||||
linksChecked = count;
|
||||
fromLinkId = from;
|
||||
toLinkId = to;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Ordered findings.
|
||||
*******************************************************************************/
|
||||
public List<Break> getBreaks()
|
||||
{
|
||||
return (List.copyOf(breaks));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Warnings do not break the chain.
|
||||
*******************************************************************************/
|
||||
public List<String> getWarnings()
|
||||
{
|
||||
return (warnings);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether all checked invariants hold.
|
||||
*******************************************************************************/
|
||||
public boolean passed()
|
||||
{
|
||||
return (breakCount == 0);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Number of checked links.
|
||||
*******************************************************************************/
|
||||
public long getLinksChecked()
|
||||
{
|
||||
return (linksChecked);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** First checked link, null for an empty scope.
|
||||
*******************************************************************************/
|
||||
public Long getFromLinkId()
|
||||
{
|
||||
return (fromLinkId);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Last checked link, null for an empty scope.
|
||||
*******************************************************************************/
|
||||
public Long getToLinkId()
|
||||
{
|
||||
return (toLinkId);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verification start time.
|
||||
*******************************************************************************/
|
||||
public Instant getStartedAt()
|
||||
{
|
||||
return (startedAt);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,573 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.verify;
|
||||
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.chain.Hashes;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.checkpoint.Checkpoints;
|
||||
import com.kingsrook.qbits.wormaudit.guard.AppendOnlyGuard;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSTransaction;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Snapshot verification of live content, continuity, exact coverage and signed
|
||||
** checkpoints. Selection runs verify predecessor boundaries but deliberately
|
||||
** omit global completeness; full runs begin at the latest retention anchor.
|
||||
*******************************************************************************/
|
||||
public class WormAuditVerifier
|
||||
{
|
||||
public static final int PAGE_SIZE = 1000;
|
||||
private final WormAuditQBitConfig config;
|
||||
private final int pageSize;
|
||||
|
||||
/*******************************************************************************
|
||||
** Bind trust pins and settle horizon.
|
||||
*******************************************************************************/
|
||||
public WormAuditVerifier(WormAuditQBitConfig config)
|
||||
{
|
||||
this(config, PAGE_SIZE);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Smaller pages support constrained hosts and boundary regression tests.
|
||||
*******************************************************************************/
|
||||
public WormAuditVerifier(WormAuditQBitConfig config, int pageSize)
|
||||
{
|
||||
if(pageSize < 1 || pageSize > PAGE_SIZE)
|
||||
{
|
||||
throw (new IllegalArgumentException("Verification page size must be between 1 and 1000"));
|
||||
}
|
||||
this.config = config;
|
||||
this.pageSize = pageSize;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify all live links and settled coverage.
|
||||
*******************************************************************************/
|
||||
public VerificationResult verify() throws QException
|
||||
{
|
||||
return (verify(null));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Null means full verification; an empty set is an empty selection.
|
||||
*******************************************************************************/
|
||||
public VerificationResult verify(Set<Long> selection) throws QException
|
||||
{
|
||||
return (verify(selection, Instant.now()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Clock-injected verification uses the same horizon and snapshot as production.
|
||||
*******************************************************************************/
|
||||
public VerificationResult verify(Set<Long> selection, Instant now) throws QException
|
||||
{
|
||||
WormAuditChainer.JVM_LOCK.lock();
|
||||
try(QBackendTransaction transaction = QBackendTransaction.openFor(new QueryInput(WormAuditTables.CHAIN)))
|
||||
{
|
||||
try
|
||||
{
|
||||
if(transaction instanceof RDBMSTransaction rdbms)
|
||||
{
|
||||
rdbms.getConnection().setTransactionIsolation(Connection.TRANSACTION_REPEATABLE_READ);
|
||||
}
|
||||
VerificationResult result = verify(new AuditStore(transaction), selection, now);
|
||||
transaction.commit();
|
||||
return (result);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
transaction.rollback();
|
||||
throw (AppendOnlyGuard.map(e));
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
WormAuditChainer.JVM_LOCK.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify one consistent storage snapshot.
|
||||
*******************************************************************************/
|
||||
VerificationResult verify(AuditStore store, Set<Long> selection, Instant now) throws QException
|
||||
{
|
||||
var settings = config.effectiveSettings();
|
||||
VerificationResult result = new VerificationResult(now);
|
||||
if(config.getSigningKey() == null)
|
||||
{
|
||||
result.warn("no signing key configured");
|
||||
}
|
||||
QRecord anchor = store.anchor();
|
||||
long anchorId = anchor.getValueLong("anchor_link_id");
|
||||
long lastId = anchorId;
|
||||
String previous = anchor.getValueString("anchor_hash");
|
||||
long checked = 0;
|
||||
Long first = null;
|
||||
Long last = null;
|
||||
Set<Long> missing = selection == null ? null : new HashSet<>(selection);
|
||||
CheckpointCursor checkpoints = new CheckpointCursor(store, selection, anchor, result);
|
||||
while(true)
|
||||
{
|
||||
List<QRecord> page = store.page(WormAuditTables.CHAIN, lastId, pageSize);
|
||||
if(page.isEmpty())
|
||||
{
|
||||
break;
|
||||
}
|
||||
List<QRecord> selected = page.stream().filter(r -> selection == null || selection.contains(r.getValueLong("id"))).toList();
|
||||
Map<Long, QRecord> headers = AuditStore.index(store.in("audit", "id", selected.stream().map(r -> r.getValueLong("audit_id")).toList()));
|
||||
Set<Long> detailIds = new HashSet<>();
|
||||
for(QRecord link : selected)
|
||||
{
|
||||
try
|
||||
{
|
||||
List<Long> ids = AuditStore.detailIds(link);
|
||||
if(ids.size() <= 250)
|
||||
{
|
||||
detailIds.addAll(ids);
|
||||
}
|
||||
}
|
||||
catch(IllegalArgumentException e)
|
||||
{
|
||||
// Content verification reports malformed coverage with its link id.
|
||||
}
|
||||
}
|
||||
Map<Long, QRecord> details = new HashMap<>();
|
||||
List<Long> ids = new ArrayList<>(detailIds);
|
||||
for(int start = 0; start < ids.size(); start += pageSize)
|
||||
{
|
||||
details.putAll(AuditStore.index(store.in("auditDetail", "id", ids.subList(start, Math.min(start + pageSize, ids.size())))));
|
||||
}
|
||||
for(QRecord link : page)
|
||||
{
|
||||
long id = link.getValueLong("id");
|
||||
long auditId = link.getValueLong("audit_id");
|
||||
if(selection == null || selection.contains(id))
|
||||
{
|
||||
if(missing != null)
|
||||
{
|
||||
missing.remove(id);
|
||||
}
|
||||
checked++;
|
||||
first = first == null ? id : first;
|
||||
last = id;
|
||||
if(selection == null && id != lastId + 1)
|
||||
{
|
||||
result.add(id, auditId, "Missing link ids " + number(lastId + 1) + "–" + number(id - 1) + " (gap)");
|
||||
}
|
||||
if(id != lastId + 1 || previous == null || !previous.equals(link.getValueString("prev_hash")))
|
||||
{
|
||||
result.add(id, auditId, "Break at link " + number(id) + " (audit " + number(auditId) + "): prev-hash discontinuity");
|
||||
}
|
||||
verifyContent(store, link, headers.get(auditId), details, result);
|
||||
}
|
||||
previous = link.getValueString("link_hash");
|
||||
lastId = id;
|
||||
}
|
||||
}
|
||||
// The main scan's page is out of scope before checkpoint pages are read.
|
||||
checkpoints.through(Long.MAX_VALUE);
|
||||
result.scope(checked, first, last);
|
||||
if(missing != null)
|
||||
{
|
||||
missing.stream().sorted().forEach(id -> result.add(id, null, "Missing selected link " + number(id)));
|
||||
}
|
||||
if(selection == null)
|
||||
{
|
||||
Instant cutoff = now.minusSeconds(settings.settle_horizon_seconds());
|
||||
if(store.isRdbms())
|
||||
{
|
||||
store.completeness(cutoff, result);
|
||||
}
|
||||
else if(store.isMemory())
|
||||
{
|
||||
completeness(store, store.all(WormAuditTables.CHAIN), cutoff, result);
|
||||
}
|
||||
else
|
||||
{
|
||||
pagedCompleteness(store, cutoff, result);
|
||||
}
|
||||
}
|
||||
return (result);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Recompute exact stored content. Diagnostics name covered detail ids because
|
||||
** v1 stores a document hash, not per-field hashes or prior values.
|
||||
*******************************************************************************/
|
||||
private void verifyContent(AuditStore store, QRecord link, QRecord audit, Map<Long, QRecord> rows, VerificationResult result) throws QException
|
||||
{
|
||||
long id = link.getValueLong("id");
|
||||
long auditId = link.getValueLong("audit_id");
|
||||
if(List.of("kind", "detail_ids", "prev_hash", "link_hash", "chained_at").stream().anyMatch(field -> link.getValue(field) == null))
|
||||
{
|
||||
result.add(id, auditId, "Break at link " + number(id) + " (audit " + number(auditId) + "): malformed link");
|
||||
return;
|
||||
}
|
||||
if(audit == null)
|
||||
{
|
||||
result.add(id, auditId, "Break at link " + number(id) + " (audit " + number(auditId) + "): audit header is missing");
|
||||
return;
|
||||
}
|
||||
if(audit.getValueInstant("timestamp") == null)
|
||||
{
|
||||
result.add(id, auditId, "Break at link " + number(id) + " (audit " + number(auditId) + "): missing audit timestamp");
|
||||
return;
|
||||
}
|
||||
try
|
||||
{
|
||||
List<Long> details = AuditStore.detailIds(link);
|
||||
String kind = link.getValueString("kind");
|
||||
if(!List.of("AUDIT", "LATE_DETAIL").contains(kind) || details.size() > 250 || ("LATE_DETAIL".equals(kind) && details.isEmpty()))
|
||||
{
|
||||
result.add(id, auditId, "Break at link " + number(id) + ": invalid kind or detail coverage");
|
||||
}
|
||||
if(details.size() > 250)
|
||||
{
|
||||
return;
|
||||
}
|
||||
Set<Long> unique = new HashSet<>();
|
||||
for(long detail : details)
|
||||
{
|
||||
if(!unique.add(detail))
|
||||
{
|
||||
result.add(id, auditId, "audit_detail " + number(detail) + " is covered twice or more by the chain");
|
||||
}
|
||||
}
|
||||
String hash = Hashes.link(link.getValueString("prev_hash"), store.canonical(audit, details, rows));
|
||||
if(!hash.equals(link.getValueString("link_hash")))
|
||||
{
|
||||
String table = store.name("auditTable", audit.getValueLong("auditTableId"));
|
||||
result.add(id, auditId, "Break at link " + number(id) + " (audit " + number(auditId) + ", table " + table + ", record "
|
||||
+ audit.getValueString("recordId") + "): content differs from the chained hash"
|
||||
+ (details.isEmpty() ? "" : " (audit_detail " + details.stream().map(WormAuditVerifier::number).collect(java.util.stream.Collectors.joining(", ")) + ")"));
|
||||
}
|
||||
}
|
||||
catch(QException | IllegalArgumentException e)
|
||||
{
|
||||
result.add(id, auditId, "Break at link " + number(id) + " (audit " + number(auditId) + "): " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Portable completeness for non-SQL backends. Trade repeated scans for bounded
|
||||
** memory; only the memory backend uses the whole-chain coverage index.
|
||||
*******************************************************************************/
|
||||
private void pagedCompleteness(AuditStore store, Instant cutoff, VerificationResult result) throws QException
|
||||
{
|
||||
for(String table : List.of("audit", "auditDetail"))
|
||||
{
|
||||
long after = 0;
|
||||
while(true)
|
||||
{
|
||||
List<QRecord> rows = store.page(table, after, pageSize);
|
||||
if(rows.isEmpty())
|
||||
{
|
||||
break;
|
||||
}
|
||||
Map<Long, Integer> counts = new HashMap<>();
|
||||
for(QRecord row : rows)
|
||||
{
|
||||
QRecord audit = table.equals("audit") ? row : store.get("audit", row.getValueLong("auditId"));
|
||||
if(audit == null || audit.getValueInstant("timestamp") == null || audit.getValueInstant("timestamp").isBefore(cutoff))
|
||||
{
|
||||
counts.put(row.getValueLong("id"), 0);
|
||||
if(table.equals("audit") && audit.getValueInstant("timestamp") == null)
|
||||
{
|
||||
result.add(null, row.getValueLong("id"), "audit " + number(row.getValueLong("id")) + " has no timestamp");
|
||||
}
|
||||
}
|
||||
}
|
||||
long linkAfter = 0;
|
||||
while(!counts.isEmpty())
|
||||
{
|
||||
List<QRecord> links = store.page(WormAuditTables.CHAIN, linkAfter, pageSize);
|
||||
if(links.isEmpty())
|
||||
{
|
||||
break;
|
||||
}
|
||||
for(QRecord link : links)
|
||||
{
|
||||
if(table.equals("audit"))
|
||||
{
|
||||
if("AUDIT".equals(link.getValueString("kind")))
|
||||
{
|
||||
counts.computeIfPresent(link.getValueLong("audit_id"), (id, count) -> count + 1);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
try
|
||||
{
|
||||
AuditStore.detailIds(link).forEach(id -> counts.computeIfPresent(id, (key, count) -> count + 1));
|
||||
}
|
||||
catch(IllegalArgumentException e)
|
||||
{
|
||||
// Malformed coverage was reported by content verification.
|
||||
}
|
||||
}
|
||||
}
|
||||
linkAfter = links.getLast().getValueLong("id");
|
||||
}
|
||||
counts.forEach((id, count) -> coverage(table.equals("audit") ? "audit" : "audit_detail", id, count, result));
|
||||
after = rows.getLast().getValueLong("id");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every settled header and detail must occur once; orphan details also fail.
|
||||
*******************************************************************************/
|
||||
private void completeness(AuditStore store, List<QRecord> links, Instant cutoff, VerificationResult result) throws QException
|
||||
{
|
||||
Map<Long, Integer> headers = new HashMap<>();
|
||||
Map<Long, Integer> details = new HashMap<>();
|
||||
for(QRecord link : links)
|
||||
{
|
||||
if("AUDIT".equals(link.getValueString("kind")))
|
||||
{
|
||||
headers.merge(link.getValueLong("audit_id"), 1, Integer::sum);
|
||||
}
|
||||
try
|
||||
{
|
||||
AuditStore.detailIds(link).forEach(id -> details.merge(id, 1, Integer::sum));
|
||||
}
|
||||
catch(IllegalArgumentException e)
|
||||
{
|
||||
// Invalid coverage was already reported while checking link content.
|
||||
}
|
||||
}
|
||||
Map<Long, QRecord> audits = new HashMap<>();
|
||||
for(QRecord audit : store.all("audit"))
|
||||
{
|
||||
long id = audit.getValueLong("id");
|
||||
audits.put(id, audit);
|
||||
if(audit.getValueInstant("timestamp") == null)
|
||||
{
|
||||
result.add(null, id, "audit " + number(id) + " has no timestamp");
|
||||
}
|
||||
else if(audit.getValueInstant("timestamp").isBefore(cutoff))
|
||||
{
|
||||
coverage("audit", id, headers.getOrDefault(id, 0), result);
|
||||
}
|
||||
}
|
||||
for(QRecord detail : store.all("auditDetail"))
|
||||
{
|
||||
QRecord audit = audits.get(detail.getValueLong("auditId"));
|
||||
if(audit == null || audit.getValueInstant("timestamp") == null || audit.getValueInstant("timestamp").isBefore(cutoff))
|
||||
{
|
||||
coverage("audit_detail", detail.getValueLong("id"), details.getOrDefault(detail.getValueLong("id"), 0), result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Report coverage without printing any audit content.
|
||||
*******************************************************************************/
|
||||
private void coverage(String table, long id, int count, VerificationResult result)
|
||||
{
|
||||
if(count != 1)
|
||||
{
|
||||
result.add(null, table.equals("audit") ? id : null, table + " " + number(id)
|
||||
+ (count == 0 ? " is not covered by the chain" : " is covered twice or more by the chain"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Check pinned signatures even for purged ranges. A partially purged range's
|
||||
** Merkle tree requires its sealed export; never claim to recompute it live.
|
||||
*******************************************************************************/
|
||||
private class CheckpointCursor
|
||||
{
|
||||
private final AuditStore store;
|
||||
private final Set<Long> selection;
|
||||
private final QRecord anchor;
|
||||
private final VerificationResult result;
|
||||
private long after;
|
||||
private QRecord next;
|
||||
|
||||
/*******************************************************************************
|
||||
** Keep only one checkpoint pending as pages complete.
|
||||
*******************************************************************************/
|
||||
CheckpointCursor(AuditStore store, Set<Long> selection, QRecord anchor, VerificationResult result)
|
||||
{
|
||||
this.store = store;
|
||||
this.selection = selection;
|
||||
this.anchor = anchor;
|
||||
this.result = result;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify completed ranges, including missing tails at end of the scan.
|
||||
*******************************************************************************/
|
||||
void through(long lastId) throws QException
|
||||
{
|
||||
while(true)
|
||||
{
|
||||
if(next == null)
|
||||
{
|
||||
List<QRecord> rows = store.page(WormAuditTables.CHECKPOINT, after, 1);
|
||||
if(rows.isEmpty())
|
||||
{
|
||||
return;
|
||||
}
|
||||
next = rows.getFirst();
|
||||
}
|
||||
Long to = next.getValueLong("to_link_id");
|
||||
if(to != null && to > lastId)
|
||||
{
|
||||
return;
|
||||
}
|
||||
check(next);
|
||||
after = next.getValueLong("id");
|
||||
next = null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Re-read at most one capped range of hashes in the same snapshot.
|
||||
*******************************************************************************/
|
||||
private void check(QRecord checkpoint) throws QException
|
||||
{
|
||||
String prefix = "checkpoint " + number(checkpoint.getValueLong("id")) + ": ";
|
||||
if(List.of("from_link_id", "to_link_id", "link_count", "head_hash", "merkle_root", "algorithm", "key_id", "public_key", "signature")
|
||||
.stream().anyMatch(field -> checkpoint.getValue(field) == null))
|
||||
{
|
||||
result.add(null, null, prefix + "malformed checkpoint");
|
||||
return;
|
||||
}
|
||||
long from = checkpoint.getValueLong("from_link_id");
|
||||
long to = checkpoint.getValueLong("to_link_id");
|
||||
long anchorId = anchor.getValueLong("anchor_link_id");
|
||||
if(selection != null && selection.stream().noneMatch(id -> id >= from && id <= to))
|
||||
{
|
||||
return;
|
||||
}
|
||||
var pin = config.getTrustedPublicKeys().get(checkpoint.getValueString("key_id"));
|
||||
if(pin == null)
|
||||
{
|
||||
result.add(null, null, prefix + "untrusted key");
|
||||
}
|
||||
else if(!Checkpoints.verify(checkpoint, pin))
|
||||
{
|
||||
result.add(null, null, prefix + "signature does not verify");
|
||||
}
|
||||
if(from < 1 || to < from || checkpoint.getValueLong("link_count") != to - from + 1
|
||||
|| to - from + 1 > Integer.MAX_VALUE)
|
||||
{
|
||||
result.add(null, null, prefix + "invalid link range or count");
|
||||
return;
|
||||
}
|
||||
if(from <= anchorId)
|
||||
{
|
||||
QRecord head = to > anchorId ? store.get(WormAuditTables.CHAIN, to) : null;
|
||||
if(to > anchorId && (head == null || !Objects.equals(head.getValueString("link_hash"), checkpoint.getValueString("head_hash"))))
|
||||
{
|
||||
result.add(to, null, prefix + "head mismatch");
|
||||
}
|
||||
if(to == anchorId && !Objects.equals(anchor.getValueString("anchor_hash"), checkpoint.getValueString("head_hash")))
|
||||
{
|
||||
result.add(to, null, prefix + "head mismatch with purge anchor");
|
||||
}
|
||||
result.warn(prefix + "purged range; Merkle proof requires the sealed export");
|
||||
return;
|
||||
}
|
||||
List<String> hashes = new ArrayList<>();
|
||||
long cursor = from - 1;
|
||||
String head = null;
|
||||
while(cursor < to)
|
||||
{
|
||||
List<QRecord> page = store.page(WormAuditTables.CHAIN, cursor, (int) Math.min(pageSize, to - cursor));
|
||||
if(page.isEmpty())
|
||||
{
|
||||
break;
|
||||
}
|
||||
for(QRecord link : page)
|
||||
{
|
||||
cursor = link.getValueLong("id");
|
||||
if(cursor <= to)
|
||||
{
|
||||
hashes.add(link.getValueString("link_hash"));
|
||||
}
|
||||
if(cursor == to)
|
||||
{
|
||||
head = link.getValueString("link_hash");
|
||||
}
|
||||
}
|
||||
}
|
||||
if(!Objects.equals(head, checkpoint.getValueString("head_hash")))
|
||||
{
|
||||
result.add(to, null, prefix + "head mismatch");
|
||||
}
|
||||
try
|
||||
{
|
||||
if(hashes.size() != checkpoint.getValueLong("link_count") || hashes.isEmpty()
|
||||
|| hashes.stream().anyMatch(Objects::isNull) || !Hashes.merkle(hashes).equals(checkpoint.getValueString("merkle_root")))
|
||||
{
|
||||
result.add(null, null, prefix + "merkle mismatch");
|
||||
}
|
||||
}
|
||||
catch(IllegalArgumentException e)
|
||||
{
|
||||
result.add(null, null, prefix + "merkle mismatch");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stable human-readable ids matching the process summary contract.
|
||||
*******************************************************************************/
|
||||
public static String number(long value)
|
||||
{
|
||||
return (String.format(Locale.US, "%,d", value));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,353 @@
|
||||
-- liquibase formatted sql
|
||||
|
||||
-- changeset worm-audit:1
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_chain (
|
||||
id bigint PRIMARY KEY,
|
||||
kind text NOT NULL,
|
||||
audit_id bigint NOT NULL,
|
||||
detail_ids text NOT NULL,
|
||||
prev_hash text NOT NULL,
|
||||
link_hash text NOT NULL,
|
||||
chained_at timestamptz NOT NULL,
|
||||
CHECK (id > 0),
|
||||
CHECK (kind IN ('AUDIT', 'LATE_DETAIL')),
|
||||
CHECK (prev_hash ~ '^[0-9a-f]{64}$' AND link_hash ~ '^[0-9a-f]{64}$'),
|
||||
CHECK (detail_ids = '' OR (detail_ids ~ '^(0|[1-9][0-9]*)(,(0|[1-9][0-9]*))*$' AND cardinality(string_to_array(detail_ids, ',')) <= 250))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_checkpoint (
|
||||
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
from_link_id bigint,
|
||||
to_link_id bigint,
|
||||
link_count bigint,
|
||||
head_hash text,
|
||||
merkle_root text,
|
||||
algorithm text,
|
||||
key_id text,
|
||||
public_key text,
|
||||
signature text,
|
||||
created_at timestamptz
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_export (
|
||||
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
from_link_id bigint,
|
||||
to_link_id bigint,
|
||||
from_checkpoint_id bigint,
|
||||
to_checkpoint_id bigint,
|
||||
bucket_alias text,
|
||||
object_prefix text,
|
||||
object_versions text,
|
||||
manifest_key text,
|
||||
manifest_sha256 text,
|
||||
prev_manifest_sha256 text,
|
||||
object_count bigint,
|
||||
byte_count bigint,
|
||||
lock_mode text,
|
||||
retain_until timestamptz,
|
||||
sealed_at timestamptz
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_verification (
|
||||
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
kind text,
|
||||
"trigger" text,
|
||||
started_at timestamptz,
|
||||
finished_at timestamptz,
|
||||
from_link_id bigint,
|
||||
to_link_id bigint,
|
||||
links_checked bigint,
|
||||
outcome text,
|
||||
break_count bigint,
|
||||
first_break_link_id bigint,
|
||||
first_break_audit_id bigint,
|
||||
first_break_reason text,
|
||||
export_id bigint,
|
||||
run_by text
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_purge (
|
||||
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
requested_at timestamptz,
|
||||
requested_by text,
|
||||
cutoff timestamptz,
|
||||
retention_interval text,
|
||||
outcome text,
|
||||
reason text,
|
||||
audits_deleted bigint,
|
||||
details_deleted bigint,
|
||||
links_deleted bigint,
|
||||
anchor_link_id bigint,
|
||||
anchor_hash text
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS worm_audit_chain_audit ON public.worm_audit_chain(audit_id) WHERE kind = 'AUDIT';
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_chain_audit_all ON public.worm_audit_chain(audit_id);
|
||||
|
||||
-- changeset worm-audit:2 splitStatements:false
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_reject() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp SET timezone = 'UTC' AS $$
|
||||
DECLARE
|
||||
old_time timestamptz;
|
||||
function_owner name;
|
||||
stack text;
|
||||
row_identity text := 'all';
|
||||
BEGIN
|
||||
IF TG_OP = 'DELETE' AND current_setting('worm_audit.purging', true) = 'on' THEN
|
||||
SELECT pg_get_userbyid(proowner) INTO function_owner FROM pg_proc
|
||||
WHERE oid = 'public.worm_audit_purge(timestamptz)'::regprocedure;
|
||||
GET DIAGNOSTICS stack = PG_CONTEXT;
|
||||
IF current_user = function_owner AND position('function worm_audit_purge(timestamp with time zone)' in stack) > 0 THEN
|
||||
IF TG_RELID = '${audit}'::regclass THEN
|
||||
old_time := OLD.timestamp;
|
||||
ELSIF TG_RELID = '${audit_detail}'::regclass THEN
|
||||
SELECT timestamp INTO old_time FROM ${audit} WHERE id = OLD.audit_id;
|
||||
ELSIF TG_RELID = 'public.worm_audit_chain'::regclass THEN
|
||||
SELECT timestamp INTO old_time FROM ${audit} WHERE id = OLD.audit_id;
|
||||
END IF;
|
||||
IF old_time < now() - interval '${retention}' THEN
|
||||
RETURN OLD;
|
||||
END IF;
|
||||
END IF;
|
||||
END IF;
|
||||
IF TG_LEVEL = 'ROW' THEN
|
||||
row_identity := OLD.id::text;
|
||||
END IF;
|
||||
RAISE EXCEPTION USING ERRCODE = 'P0001',
|
||||
MESSAGE = format('worm_audit: audit rows are write-once; %s of "%s" is not allowed (id=%s). Expired rows are removed only by worm_audit_purge().',
|
||||
TG_OP, TG_TABLE_NAME, row_identity),
|
||||
HINT = 'see qbit-worm-audit docs/DESIGN.md section 4';
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:3 splitStatements:false
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_lookup_reject() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $$
|
||||
BEGIN
|
||||
IF TG_OP = 'UPDATE' AND NEW.id IS NOT DISTINCT FROM OLD.id AND NEW.name IS NOT DISTINCT FROM OLD.name THEN
|
||||
RETURN NEW;
|
||||
END IF;
|
||||
RAISE EXCEPTION USING ERRCODE = 'P0001',
|
||||
MESSAGE = format('worm_audit: audit rows are write-once; %s of "%s" is not allowed (id=%s). Expired rows are removed only by worm_audit_purge().',
|
||||
TG_OP, TG_TABLE_NAME, OLD.id),
|
||||
HINT = 'see qbit-worm-audit docs/DESIGN.md section 4';
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:4 splitStatements:false
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'worm_audit_purger') THEN
|
||||
CREATE ROLE worm_audit_purger NOLOGIN;
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:5 splitStatements:false
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_purge(p_cutoff timestamptz) RETURNS void
|
||||
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, public, pg_temp SET timezone = 'UTC' AS $$
|
||||
DECLARE
|
||||
boundary bigint;
|
||||
smaller bigint;
|
||||
anchor text;
|
||||
audits_count bigint;
|
||||
details_count bigint;
|
||||
links_count bigint;
|
||||
BEGIN
|
||||
IF p_cutoff IS NULL OR p_cutoff > now() - interval '${retention}' THEN
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome, reason)
|
||||
VALUES(now(), session_user, p_cutoff, '${retention}', 'REFUSED', 'Cutoff is within retention or null');
|
||||
RETURN;
|
||||
END IF;
|
||||
-- Same lock as the Java chainer; table locks also exclude concurrent audit inserts.
|
||||
PERFORM pg_advisory_xact_lock(86753090421);
|
||||
LOCK TABLE ${audit}, ${audit_detail}, public.worm_audit_chain IN SHARE ROW EXCLUSIVE MODE;
|
||||
SELECT coalesce(min(c.id) - 1, (SELECT max(id) FROM public.worm_audit_chain)) INTO boundary
|
||||
FROM public.worm_audit_chain c LEFT JOIN ${audit} a ON a.id = c.audit_id
|
||||
WHERE a.id IS NULL OR a.timestamp IS NULL OR a.timestamp >= p_cutoff;
|
||||
-- Shrink to a prefix closed over every audit's links AND all its stored details.
|
||||
LOOP
|
||||
SELECT min(c.id) - 1 INTO smaller FROM public.worm_audit_chain c
|
||||
WHERE c.id <= boundary AND (
|
||||
EXISTS (SELECT FROM public.worm_audit_chain later WHERE later.audit_id = c.audit_id AND later.id > boundary)
|
||||
OR EXISTS (SELECT FROM ${audit_detail} d WHERE d.audit_id = c.audit_id AND NOT EXISTS (
|
||||
SELECT FROM public.worm_audit_chain covered WHERE covered.audit_id = c.audit_id
|
||||
AND d.id::text = ANY(string_to_array(covered.detail_ids, ',')))));
|
||||
EXIT WHEN smaller IS NULL;
|
||||
boundary := smaller;
|
||||
END LOOP;
|
||||
SELECT link_hash INTO anchor FROM public.worm_audit_chain WHERE id = boundary;
|
||||
IF anchor IS NULL THEN
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome, reason)
|
||||
VALUES(now(), session_user, p_cutoff, '${retention}', 'REFUSED', 'No complete expired chain prefix');
|
||||
RETURN;
|
||||
END IF;
|
||||
PERFORM set_config('worm_audit.purging', 'on', true);
|
||||
DELETE FROM ${audit_detail} d WHERE EXISTS
|
||||
(SELECT FROM public.worm_audit_chain c WHERE c.audit_id = d.audit_id AND c.id <= boundary);
|
||||
GET DIAGNOSTICS details_count = ROW_COUNT;
|
||||
-- Keep ids locally because links must be deleted before their audit headers.
|
||||
WITH removed AS (DELETE FROM public.worm_audit_chain WHERE id <= boundary RETURNING audit_id),
|
||||
removed_audits AS (DELETE FROM ${audit} WHERE id IN (SELECT audit_id FROM removed) RETURNING id)
|
||||
SELECT (SELECT count(*) FROM removed), (SELECT count(*) FROM removed_audits) INTO links_count, audits_count;
|
||||
PERFORM set_config('worm_audit.purging', 'off', true);
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome,
|
||||
audits_deleted, details_deleted, links_deleted, anchor_link_id, anchor_hash)
|
||||
VALUES(now(), session_user, p_cutoff, '${retention}', 'PURGED', audits_count, details_count, links_count, boundary, anchor);
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:6
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON ${audit};
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON ${audit}
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON ${audit};
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON ${audit}
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON ${audit} FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON ${audit} TO ${app_role};
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON ${audit_detail};
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON ${audit_detail}
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON ${audit_detail};
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON ${audit_detail}
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON ${audit_detail} FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON ${audit_detail} TO ${app_role};
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_chain;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_chain
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_chain;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_chain
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_chain FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON public.worm_audit_chain TO ${app_role};
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_checkpoint;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_checkpoint
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_checkpoint;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_checkpoint
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_checkpoint FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON public.worm_audit_checkpoint TO ${app_role};
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_export;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_export
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_export;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_export
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_export FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON public.worm_audit_export TO ${app_role};
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_verification;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_verification
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_verification;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_verification
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_verification FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON public.worm_audit_verification TO ${app_role};
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_purge;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_purge
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_purge;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_purge
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_purge FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON public.worm_audit_purge TO ${app_role};
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON ${audit_table};
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON ${audit_table}
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_lookup_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON ${audit_table};
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON ${audit_table}
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON ${audit_table} FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON ${audit_table} TO ${app_role};
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON ${audit_user};
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON ${audit_user}
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_lookup_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON ${audit_user};
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON ${audit_user}
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON ${audit_user} FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON ${audit_user} TO ${app_role};
|
||||
|
||||
-- Lookups may refresh display metadata, but never identities or names.
|
||||
GRANT UPDATE (label, create_date, modify_date) ON ${audit_table} TO ${app_role};
|
||||
GRANT UPDATE (create_date, modify_date) ON ${audit_user} TO ${app_role};
|
||||
|
||||
REVOKE ALL ON FUNCTION public.worm_audit_purge(timestamptz) FROM PUBLIC;
|
||||
REVOKE ALL ON FUNCTION public.worm_audit_purge(timestamptz) FROM ${app_role};
|
||||
GRANT EXECUTE ON FUNCTION public.worm_audit_purge(timestamptz) TO worm_audit_purger;
|
||||
|
||||
-- changeset worm-audit:7 splitStatements:false
|
||||
DO $$
|
||||
DECLARE
|
||||
table_name text;
|
||||
sequence_name text;
|
||||
BEGIN
|
||||
FOREACH table_name IN ARRAY ARRAY['${audit}', '${audit_detail}', '${audit_table}', '${audit_user}',
|
||||
'public.worm_audit_checkpoint', 'public.worm_audit_export', 'public.worm_audit_verification', 'public.worm_audit_purge'] LOOP
|
||||
sequence_name := pg_get_serial_sequence(table_name, 'id');
|
||||
IF sequence_name IS NOT NULL THEN
|
||||
EXECUTE format('GRANT USAGE ON SEQUENCE %s TO ${app_role}', sequence_name);
|
||||
END IF;
|
||||
END LOOP;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:8 splitStatements:false
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_detail_coverage() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $$
|
||||
BEGIN
|
||||
-- Serialize even direct SQL appends; the GIN index makes overlap checks indexed.
|
||||
PERFORM pg_advisory_xact_lock(86753090421);
|
||||
IF NEW.detail_ids <> '' AND (
|
||||
EXISTS (SELECT FROM unnest(string_to_array(NEW.detail_ids, ',')) AS d(id) GROUP BY d.id HAVING count(*) > 1)
|
||||
OR EXISTS (SELECT FROM public.worm_audit_chain c
|
||||
WHERE string_to_array(c.detail_ids, ',') && string_to_array(NEW.detail_ids, ','))) THEN
|
||||
RAISE EXCEPTION USING ERRCODE = '23505', MESSAGE = 'worm_audit: audit detail is already covered by the chain';
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:9
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_chain_detail_coverage ON public.worm_audit_chain USING gin (string_to_array(detail_ids, ','));
|
||||
DROP TRIGGER IF EXISTS worm_audit_detail_coverage_guard ON public.worm_audit_chain;
|
||||
CREATE TRIGGER worm_audit_detail_coverage_guard BEFORE INSERT ON public.worm_audit_chain
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_detail_coverage();
|
||||
|
||||
-- changeset worm-audit:10
|
||||
-- Editable singleton: deliberately excluded from append-only triggers.
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_settings (
|
||||
id integer PRIMARY KEY CHECK (id = 1),
|
||||
audit_retention_years integer NOT NULL DEFAULT 6 CHECK (audit_retention_years BETWEEN 1 AND 100),
|
||||
export_lock_mode text NOT NULL DEFAULT 'GOVERNANCE' CHECK (export_lock_mode IN ('GOVERNANCE', 'COMPLIANCE')),
|
||||
export_retention_days integer NOT NULL DEFAULT 30 CHECK (export_retention_days BETWEEN 1 AND 36500),
|
||||
checkpoint_every_links integer NOT NULL DEFAULT 10000 CHECK (checkpoint_every_links > 0),
|
||||
checkpoint_every_minutes integer NOT NULL DEFAULT 60 CHECK (checkpoint_every_minutes > 0),
|
||||
chain_interval_seconds integer NOT NULL DEFAULT 30 CHECK (chain_interval_seconds > 0),
|
||||
settle_horizon_seconds integer NOT NULL DEFAULT 60 CHECK (settle_horizon_seconds >= 0),
|
||||
late_detail_lookback_hours integer NOT NULL DEFAULT 24 CHECK (late_detail_lookback_hours > 0),
|
||||
export_cron text NOT NULL DEFAULT '0 0 2 * * ?',
|
||||
verify_cron text NOT NULL DEFAULT '0 0 3 * * ?',
|
||||
notes text,
|
||||
modify_date timestamptz NOT NULL DEFAULT now(),
|
||||
modified_by text NOT NULL DEFAULT current_user
|
||||
);
|
||||
REVOKE DELETE, TRUNCATE ON public.worm_audit_settings FROM ${app_role};
|
||||
GRANT SELECT, INSERT, UPDATE ON public.worm_audit_settings TO ${app_role};
|
||||
INSERT INTO public.worm_audit_settings
|
||||
(id, audit_retention_years, export_lock_mode, export_retention_days, checkpoint_every_links,
|
||||
checkpoint_every_minutes, chain_interval_seconds, settle_horizon_seconds, late_detail_lookback_hours,
|
||||
export_cron, verify_cron)
|
||||
SELECT ${settings_values}
|
||||
WHERE NOT EXISTS (SELECT 1 FROM public.worm_audit_settings WHERE id = 1);
|
||||
@@ -0,0 +1,204 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.KeyPairGenerator;
|
||||
import java.time.Instant;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.wormaudit.canonical.AuditCanonical;
|
||||
import com.kingsrook.qbits.wormaudit.canonical.Jcs;
|
||||
import com.kingsrook.qbits.wormaudit.chain.Hashes;
|
||||
import com.kingsrook.qbits.wormaudit.checkpoint.Checkpoints;
|
||||
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
|
||||
import com.kingsrook.qbits.wormaudit.guard.AppendOnlyGuard;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for canonical and crypto test.
|
||||
*******************************************************************************/
|
||||
class CanonicalAndCryptoTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Verify jcs ordering escaping and invalid types.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void jcsOrderingEscapingAndInvalidTypes()
|
||||
{
|
||||
var value = new LinkedHashMap<String, Object>();
|
||||
value.put("\u20ac", "Euro");
|
||||
value.put("\ud83d\ude00", "Emoji");
|
||||
value.put("\ufb33", "Hebrew");
|
||||
value.put("a", null);
|
||||
String result = Jcs.string(value);
|
||||
assertThat(result)
|
||||
.isEqualTo(
|
||||
"{\"a\":null,\"\u20ac\":\"Euro\",\"\ud83d\ude00\":\"Emoji\",\"\ufb33\":\"Hebrew\"}");
|
||||
assertThat(Jcs.string("\"\\\b\f\n\r\t\u0001/é"))
|
||||
.isEqualTo("\"\\\"\\\\\\b\\f\\n\\r\\t\\u0001/é\"");
|
||||
assertThat(Jcs.string(List.of(Long.MIN_VALUE, Long.MAX_VALUE)))
|
||||
.isEqualTo("[-9223372036854775808,9223372036854775807]");
|
||||
assertThatThrownBy(() -> Jcs.string(1.5)).isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatThrownBy(() -> Jcs.string("\ud800")).isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatThrownBy(() -> Jcs.string("\udc00")).isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatThrownBy(() -> Jcs.string("\ud800x"))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify exact canonical document.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void exactCanonicalDocument()
|
||||
{
|
||||
var audit =
|
||||
new QRecord()
|
||||
.withValue("id", 4211L)
|
||||
.withValue("recordId", 17)
|
||||
.withValue("message", "Record was Edited")
|
||||
.withValue("timestamp", Instant.parse("2026-10-13T21:04:05.123456Z"));
|
||||
var detail =
|
||||
new QRecord()
|
||||
.withValue("id", 9001L)
|
||||
.withValue("fieldName", "title")
|
||||
.withValue("message", "Changed Title")
|
||||
.withValue("oldValue", "hmac:v1:test:old")
|
||||
.withValue("newValue", "hmac:v1:test:new");
|
||||
assertThat(
|
||||
new String(
|
||||
AuditCanonical.bytes(audit, "cms_page", "james", List.of(detail)),
|
||||
StandardCharsets.UTF_8))
|
||||
.isEqualTo(
|
||||
"{\"audit\":{\"clientChannel\":null,\"id\":4211,\"message\":\"Record was"
|
||||
+ " Edited\",\"recordId\":\"17\",\"table\":\"cms_page\",\"timestamp\":\"2026-10-13T21:04:05.123456Z\",\"user\":\"james\"},\"details\":[{\"fieldName\":\"title\",\"id\":9001,\"message\":\"Changed"
|
||||
+ " Title\",\"newValue\":\"hmac:v1:test:new\",\"oldValue\":\"hmac:v1:test:old\"}],\"v\":1}");
|
||||
assertThat(
|
||||
new String(
|
||||
AuditCanonical.bytes(
|
||||
audit,
|
||||
null,
|
||||
null,
|
||||
List.of(
|
||||
new QRecord().withValue("id", 2L),
|
||||
new QRecord().withValue("id", 1L))),
|
||||
StandardCharsets.UTF_8))
|
||||
.containsSubsequence("\"id\":1,", "\"id\":2,");
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify independent hash and merkle vectors.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void independentHashAndMerkleVectors() throws Exception
|
||||
{
|
||||
assertThat(Hashes.link(Hashes.GENESIS, "{}".getBytes(StandardCharsets.UTF_8)))
|
||||
.isEqualTo("72f1d7c888d85ce3159bd8bbf38f3046725d3d4d6cef2b9449d9fb4b58cf7a9c");
|
||||
String a = "11".repeat(32), b = "22".repeat(32), c = "33".repeat(32);
|
||||
assertThat(Hashes.merkle(List.of(a))).isEqualTo(a);
|
||||
assertThat(Hashes.merkle(List.of(a, b, c)))
|
||||
.isEqualTo("2470b93c7954a79f41c0106caeeeae350559c2af06bd9a9ab203b163951f64f7");
|
||||
assertThatThrownBy(() -> Hashes.merkle(List.of()))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify checkpoint pinning and signing.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void checkpointPinningAndSigning() throws Exception
|
||||
{
|
||||
var keys = KeyPairGenerator.getInstance("Ed25519").generateKeyPair();
|
||||
var other = KeyPairGenerator.getInstance("Ed25519").generateKeyPair();
|
||||
var config =
|
||||
new WormAuditQBitConfig()
|
||||
.withSigningKey(
|
||||
() -> new WormAuditQBitConfig.SigningKey("a", keys.getPrivate()))
|
||||
.withTrustedPublicKeys(Map.of("a", keys.getPublic()));
|
||||
var links =
|
||||
List.of(new QRecord().withValue("id", 1L).withValue("link_hash", "11".repeat(32)));
|
||||
var checkpoint = Checkpoints.sign(links, config, Instant.now());
|
||||
assertThat(Checkpoints.verify(checkpoint, keys.getPublic())).isTrue();
|
||||
assertThat(Checkpoints.verify(checkpoint, other.getPublic())).isFalse();
|
||||
assertThat(Checkpoints.verify(checkpoint, null)).isFalse();
|
||||
checkpoint.setValue("signature", "bad");
|
||||
assertThat(Checkpoints.verify(checkpoint, keys.getPublic())).isFalse();
|
||||
checkpoint.setValue("algorithm", "RSA");
|
||||
assertThat(Checkpoints.verify(checkpoint, keys.getPublic())).isFalse();
|
||||
config.withTrustedPublicKeys(Map.of());
|
||||
assertThatThrownBy(() -> Checkpoints.sign(links, config, Instant.now()))
|
||||
.hasMessageContaining("not pinned");
|
||||
config.withTrustedPublicKeys(Map.of("a", other.getPublic()));
|
||||
assertThatThrownBy(() -> Checkpoints.sign(links, config, Instant.now()))
|
||||
.hasMessageContaining("does not match");
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify ddl is reviewed rendering and rejects injection.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void ddlIsReviewedRenderingAndRejectsInjection() throws Exception
|
||||
{
|
||||
String sql = WormAuditDdl.postgres(new WormAuditQBitConfig());
|
||||
assertThat(sql)
|
||||
.isEqualTo(
|
||||
java.nio.file.Files.readString(
|
||||
java.nio.file.Path.of("docs/ddl/worm-audit-postgres.sql")));
|
||||
assertThat(sql).doesNotContain("${");
|
||||
assertThat(sql).contains("GET DIAGNOSTICS stack = PG_CONTEXT", "function worm_audit_purge(timestamp with time zone)", "SET timezone = 'UTC'");
|
||||
// The documented migration contract deliberately uses physical snake_case columns.
|
||||
assertThat(sql).contains("OLD.audit_id", "OLD.timestamp", "NEW.name", "GRANT UPDATE (label, create_date, modify_date)");
|
||||
assertThat(java.nio.file.Files.readString(java.nio.file.Path.of("README.md")))
|
||||
.contains("renders table names only", "audit_detail.audit_id", "audit_user.id/name/create_date/modify_date");
|
||||
assertThat(
|
||||
WormAuditDdl.postgres(
|
||||
new WormAuditQBitConfig()
|
||||
.withRetentionYears(7)
|
||||
.withAppRole("runtime")
|
||||
.withCoreAuditTableNames(Map.of("audit", "history.audit"))))
|
||||
.contains("\"history\".\"audit\"", "7 years", "TO \"runtime\"");
|
||||
assertThatThrownBy(() -> WormAuditDdl.identifier("a; DROP TABLE audit"))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
WormAuditDdl.postgres(
|
||||
new WormAuditQBitConfig().withRetentionYears(0)))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatThrownBy(
|
||||
() -> WormAuditDdl.postgres(new WormAuditQBitConfig().withAppRole("a.b")))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
var sqlError = new java.sql.SQLException("worm_audit: rejected", "P0001");
|
||||
assertThat(AppendOnlyGuard.map(new Exception(sqlError)))
|
||||
.hasMessage(AppendOnlyGuard.MESSAGE);
|
||||
assertThat(
|
||||
AppendOnlyGuard.map(
|
||||
new java.sql.SQLException("ERROR: worm_audit: rejected", "P0001")))
|
||||
.hasMessage(AppendOnlyGuard.MESSAGE);
|
||||
assertThat(AppendOnlyGuard.map(new java.sql.SQLException("other", "P0001")))
|
||||
.hasMessage("WORM audit operation failed");
|
||||
var original = new com.kingsrook.qqq.backend.core.exceptions.QException("original");
|
||||
assertThat(AppendOnlyGuard.map(original)).isSameAs(original);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,347 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import java.security.KeyPairGenerator;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.chain.Hashes;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.OLD;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.STORE;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.core;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.detail;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.remove;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.tamper;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for chain verification test.
|
||||
*******************************************************************************/
|
||||
class ChainVerificationTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Verify signed.
|
||||
*******************************************************************************/
|
||||
static WormAuditQBitConfig signed() throws Exception
|
||||
{
|
||||
var pair = KeyPairGenerator.getInstance("Ed25519").generateKeyPair();
|
||||
return (new WormAuditQBitConfig()
|
||||
.withCheckpointEveryLinks(1)
|
||||
.withSigningKey(() -> new WormAuditQBitConfig.SigningKey("test", pair.getPrivate()))
|
||||
.withTrustedPublicKeys(Map.of("test", pair.getPublic())));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify late headers details and chunk boundaries.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void lateHeadersDetailsAndChunkBoundaries() throws Exception
|
||||
{
|
||||
var config = signed().withBatchSize(1);
|
||||
init(config);
|
||||
audit(20);
|
||||
detail(1, 20);
|
||||
var writer = new WormAuditChainer(config);
|
||||
assertThat(writer.chainOnce().audits()).isEqualTo(1);
|
||||
audit(1);
|
||||
detail(2, 1);
|
||||
detail(3, 20);
|
||||
assertThat(writer.chainOnce().lateDetails()).isEqualTo(1);
|
||||
for(long i = 4; i <= 260; i++)
|
||||
{
|
||||
detail(i, 1);
|
||||
}
|
||||
assertThat(writer.chainOnce().lateDetails()).isEqualTo(257);
|
||||
var links = STORE.all(WormAuditTables.CHAIN);
|
||||
assertThat(links).extracting(r -> r.getValueLong("id")).containsExactly(1L, 2L, 3L, 4L, 5L);
|
||||
assertThat(links).allMatch(r -> AuditStore.detailIds(r).size() <= 250);
|
||||
var result = new WormAuditVerifier(config).verify();
|
||||
assertThat(result.getBreaks()).isEmpty();
|
||||
assertThat(result.getLinksChecked()).isEqualTo(5);
|
||||
assertThat(result.getFromLinkId()).isEqualTo(1);
|
||||
assertThat(result.getToLinkId()).isEqualTo(5);
|
||||
assertThat(writer.chainOnce().links()).isZero();
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify overflow at initial chain and settling and elapsed checkpoint.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void overflowAtInitialChainAndSettlingAndElapsedCheckpoint() throws Exception
|
||||
{
|
||||
var config = signed().withCheckpointEveryLinks(10000).withCheckpointEveryMinutes(1);
|
||||
init(config);
|
||||
audit(1);
|
||||
for(long i = 1; i <= 251; i++)
|
||||
{
|
||||
detail(i, 1);
|
||||
}
|
||||
var writer = new WormAuditChainer(config);
|
||||
assertThat(writer.chainOnce(OLD.plusSeconds(30)).links()).isZero();
|
||||
Instant now = Instant.now();
|
||||
assertThat(writer.chainOnce(now).links()).isEqualTo(2);
|
||||
assertThat(STORE.all(WormAuditTables.CHECKPOINT)).isEmpty();
|
||||
assertThat(writer.chainOnce(now.plusSeconds(61)).checkpoints()).isEqualTo(1);
|
||||
assertThat(new WormAuditVerifier(config).verify().passed()).isTrue();
|
||||
config.withSigningKey(null);
|
||||
assertThat(new WormAuditVerifier(config).verify().getWarnings())
|
||||
.contains("no signing key configured");
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify detects breaks.
|
||||
*******************************************************************************/
|
||||
@ParameterizedTest
|
||||
@ValueSource(
|
||||
strings =
|
||||
{
|
||||
"content",
|
||||
"headerDeleted",
|
||||
"detailDeleted",
|
||||
"inserted",
|
||||
"detailInserted",
|
||||
"gap",
|
||||
"previous",
|
||||
"reorder",
|
||||
"forged",
|
||||
"untrusted",
|
||||
"merkle",
|
||||
"head",
|
||||
"duplicate",
|
||||
"kind",
|
||||
"lookup",
|
||||
"range",
|
||||
"nullCheckpoint",
|
||||
"nullPrevious",
|
||||
"nullTimestamp"
|
||||
})
|
||||
void detectsBreaks(String scenario) throws Exception
|
||||
{
|
||||
var config = signed();
|
||||
init(config);
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
audit(2);
|
||||
detail(2, 2);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
String expected;
|
||||
switch(scenario)
|
||||
{
|
||||
case "content" ->
|
||||
{
|
||||
tamper("auditDetail", 1, "newValue", "tampered");
|
||||
expected =
|
||||
"Break at link 1 (audit 1, table cms_page, record 17): content differs from"
|
||||
+ " the chained hash (audit_detail 1)";
|
||||
}
|
||||
case "headerDeleted" ->
|
||||
{
|
||||
remove("audit", 1);
|
||||
expected = "audit header is missing";
|
||||
}
|
||||
case "detailDeleted" ->
|
||||
{
|
||||
remove("auditDetail", 1);
|
||||
expected = "audit_detail 1 is missing";
|
||||
}
|
||||
case "inserted" ->
|
||||
{
|
||||
audit(3);
|
||||
expected = "audit 3 is not covered";
|
||||
}
|
||||
case "detailInserted" ->
|
||||
{
|
||||
detail(3, 1);
|
||||
expected = "audit_detail 3 is not covered";
|
||||
}
|
||||
case "gap" ->
|
||||
{
|
||||
remove(WormAuditTables.CHAIN, 1);
|
||||
expected = "gap";
|
||||
}
|
||||
case "previous" ->
|
||||
{
|
||||
tamper(WormAuditTables.CHAIN, 2, "prev_hash", Hashes.GENESIS);
|
||||
expected = "prev-hash discontinuity";
|
||||
}
|
||||
case "reorder" ->
|
||||
{
|
||||
tamper(WormAuditTables.CHAIN, 1, "audit_id", 2L);
|
||||
expected = "another audit";
|
||||
}
|
||||
case "forged" ->
|
||||
{
|
||||
tamper(WormAuditTables.CHECKPOINT, 1, "signature", "AA==");
|
||||
expected = "checkpoint 1: signature does not verify";
|
||||
}
|
||||
case "untrusted" ->
|
||||
{
|
||||
config.withTrustedPublicKeys(Map.of());
|
||||
expected = "untrusted key";
|
||||
}
|
||||
case "merkle" ->
|
||||
{
|
||||
tamper(WormAuditTables.CHECKPOINT, 1, "merkle_root", Hashes.GENESIS);
|
||||
expected = "merkle mismatch";
|
||||
}
|
||||
case "head" ->
|
||||
{
|
||||
tamper(WormAuditTables.CHECKPOINT, 1, "head_hash", Hashes.GENESIS);
|
||||
expected = "head mismatch";
|
||||
}
|
||||
case "duplicate" ->
|
||||
{
|
||||
tamper(WormAuditTables.CHAIN, 1, "detail_ids", "1,1");
|
||||
expected = "audit_detail 1 is covered twice";
|
||||
}
|
||||
case "kind" ->
|
||||
{
|
||||
tamper(WormAuditTables.CHAIN, 1, "kind", "INVALID");
|
||||
expected = "invalid kind";
|
||||
}
|
||||
case "lookup" ->
|
||||
{
|
||||
tamper("auditTable", 1, "name", "renamed");
|
||||
expected = "content differs";
|
||||
}
|
||||
case "nullCheckpoint" ->
|
||||
{
|
||||
tamper(WormAuditTables.CHECKPOINT, 1, "signature", null);
|
||||
expected = "malformed checkpoint";
|
||||
}
|
||||
case "nullPrevious" ->
|
||||
{
|
||||
tamper(WormAuditTables.CHAIN, 1, "prev_hash", null);
|
||||
expected = "malformed link";
|
||||
}
|
||||
case "nullTimestamp" ->
|
||||
{
|
||||
tamper("audit", 1, "timestamp", null);
|
||||
expected = "missing audit timestamp";
|
||||
}
|
||||
default ->
|
||||
{
|
||||
tamper(WormAuditTables.CHECKPOINT, 1, "link_count", 22L);
|
||||
expected = "invalid link range";
|
||||
}
|
||||
}
|
||||
var result = new WormAuditVerifier(config).verify();
|
||||
assertThat(result.passed()).isFalse();
|
||||
assertThat(result.getBreaks()).anyMatch(b -> b.reason().contains(expected));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify selection does not run completeness and checks boundary.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void selectionDoesNotRunCompletenessAndChecksBoundary() throws Exception
|
||||
{
|
||||
var config = signed();
|
||||
init(config);
|
||||
audit(1);
|
||||
audit(2);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
audit(3);
|
||||
assertThat(new WormAuditVerifier(config).verify(Set.of(2L)).passed()).isTrue();
|
||||
assertThat(new WormAuditVerifier(config).verify(Set.of()).getLinksChecked()).isZero();
|
||||
assertThat(new WormAuditVerifier(config).verify(Set.of(99L)).getBreaks())
|
||||
.anyMatch(b -> b.reason().contains("Missing selected link 99"));
|
||||
tamper(WormAuditTables.CHAIN, 2, "prev_hash", Hashes.GENESIS);
|
||||
assertThat(new WormAuditVerifier(config).verify(Set.of(2L)).getBreaks())
|
||||
.anyMatch(b -> b.reason().contains("prev-hash"));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify continues after purge anchor including empty live chain.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void continuesAfterPurgeAnchorIncludingEmptyLiveChain() throws Exception
|
||||
{
|
||||
var config = signed();
|
||||
init(config);
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
audit(2);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
var first = STORE.get(WormAuditTables.CHAIN, 1);
|
||||
STORE.insert(
|
||||
WormAuditTables.PURGE,
|
||||
List.of(
|
||||
new com.kingsrook.qqq.backend.core.model.data.QRecord()
|
||||
.withValue("outcome", "PURGED")
|
||||
.withValue("anchor_link_id", 1L)
|
||||
.withValue("anchor_hash", first.getValueString("link_hash"))));
|
||||
remove("auditDetail", 1);
|
||||
remove("audit", 1);
|
||||
remove(WormAuditTables.CHAIN, 1);
|
||||
assertThat(new WormAuditVerifier(config).verify().passed()).isTrue();
|
||||
var last = STORE.get(WormAuditTables.CHAIN, 2);
|
||||
STORE.insert(
|
||||
WormAuditTables.PURGE,
|
||||
List.of(
|
||||
new com.kingsrook.qqq.backend.core.model.data.QRecord()
|
||||
.withValue("outcome", "PURGED")
|
||||
.withValue("anchor_link_id", 2L)
|
||||
.withValue("anchor_hash", last.getValueString("link_hash"))));
|
||||
remove("audit", 2);
|
||||
remove(WormAuditTables.CHAIN, 2);
|
||||
assertThat(new WormAuditVerifier(config).verify().passed()).isTrue();
|
||||
audit(3);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
assertThat(STORE.all(WormAuditTables.CHAIN))
|
||||
.extracting(r -> r.getValueLong("id"))
|
||||
.containsExactly(3L);
|
||||
assertThat(new WormAuditVerifier(config).verify().passed()).isTrue();
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Findings retain bounded diagnostics while recording the complete failure count.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void boundedDiagnosticsPreserveTotalCount() throws Exception
|
||||
{
|
||||
init(new WormAuditQBitConfig());
|
||||
var result = new com.kingsrook.qbits.wormaudit.verify.VerificationResult();
|
||||
for(long id = 1; id <= 1200; id++)
|
||||
{
|
||||
result.add(id, id, "gap");
|
||||
result.warn("warning " + id);
|
||||
}
|
||||
result.omitted(20);
|
||||
assertThat(result.getBreaks()).hasSize(1000);
|
||||
assertThat(result.getWarnings()).hasSize(100);
|
||||
assertThat(result.getBreakCount()).isEqualTo(1220);
|
||||
assertThat(result.passed()).isFalse();
|
||||
assertThat(com.kingsrook.qbits.wormaudit.processes.VerificationRuns.save(result, "USER").getValueLong("break_count")).isEqualTo(1220);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for docker condition test.
|
||||
*******************************************************************************/
|
||||
class DockerConditionTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Verify never skips in ci.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void neverSkipsInCi()
|
||||
{
|
||||
assertThat(RequiresDockerCondition.evaluate(false, null).isDisabled()).isTrue();
|
||||
assertThat(RequiresDockerCondition.evaluate(true, "true").isDisabled()).isFalse();
|
||||
assertThatThrownBy(() -> RequiresDockerCondition.evaluate(false, "false"))
|
||||
.hasMessageContaining("CI is set");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import java.net.URI;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.objectstorage.ObjectStorageQBitConfig;
|
||||
import com.kingsrook.qbits.objectstorage.api.Capability;
|
||||
import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
|
||||
import com.kingsrook.qbits.objectstorage.api.ListRequest;
|
||||
import com.kingsrook.qbits.objectstorage.api.ObjectStorage;
|
||||
import com.kingsrook.qbits.objectstorage.config.AddressingStyle;
|
||||
import com.kingsrook.qbits.objectstorage.config.BucketRole;
|
||||
import com.kingsrook.qbits.objectstorage.config.ChecksumMode;
|
||||
import com.kingsrook.qbits.objectstorage.config.ProviderType;
|
||||
import com.kingsrook.qbits.objectstorage.config.StorageBucketConfig;
|
||||
import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig;
|
||||
import com.kingsrook.qbits.objectstorage.internal.CapabilityProbe;
|
||||
import com.kingsrook.qbits.objectstorage.internal.CapabilityStore;
|
||||
import com.kingsrook.qbits.objectstorage.internal.ObjectStorageRuntime;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.export.ExportFiles;
|
||||
import com.kingsrook.qbits.wormaudit.export.ObjectStorageSealedStore;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedExportVerifier;
|
||||
import com.kingsrook.qbits.wormaudit.export.SealedExporter;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.ExportProcesses;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.testcontainers.containers.GenericContainer;
|
||||
import org.testcontainers.containers.wait.strategy.Wait;
|
||||
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
|
||||
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
||||
import software.amazon.awssdk.core.checksums.RequestChecksumCalculation;
|
||||
import software.amazon.awssdk.core.checksums.ResponseChecksumValidation;
|
||||
import software.amazon.awssdk.core.sync.RequestBody;
|
||||
import software.amazon.awssdk.regions.Region;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
import software.amazon.awssdk.services.s3.model.S3Exception;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.STORE;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.detail;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.tamper;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Real Object Lock acceptance. Missing Docker skips locally and fails in CI.
|
||||
*******************************************************************************/
|
||||
@ExtendWith(RequiresDockerCondition.class)
|
||||
class MinioExportAcceptanceTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Production adapter, GOVERNANCE and COMPLIANCE retention, version preservation and UI findings.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void lockedRoundTripVersionRetentionLiveTamperAndUnsupportedBucket() throws Exception
|
||||
{
|
||||
String image = System.getProperty("qqq.test.minioImage",
|
||||
"cgr.dev/chainguard/minio@sha256:dc455cdd4db6e390b1b26514f15ff37681a777bf8dab41aac445a0259f751e56");
|
||||
try(GenericContainer<?> minio = new GenericContainer<>(image)
|
||||
.withEnv("MINIO_ROOT_USER", "worm-test-access").withEnv("MINIO_ROOT_PASSWORD", "worm-test-secret")
|
||||
.withCommand("server", "/tmp/minio-data").withExposedPorts(9000)
|
||||
.waitingFor(Wait.forHttp("/minio/health/live").forPort(9000).withStartupTimeout(Duration.ofMinutes(2))))
|
||||
{
|
||||
minio.start();
|
||||
String endpoint = "http://" + minio.getHost() + ":" + minio.getMappedPort(9000);
|
||||
try(S3Client s3 = S3Client.builder().endpointOverride(URI.create(endpoint)).region(Region.US_EAST_1).forcePathStyle(true)
|
||||
.credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("worm-test-access", "worm-test-secret")))
|
||||
.requestChecksumCalculation(RequestChecksumCalculation.WHEN_REQUIRED)
|
||||
.responseChecksumValidation(ResponseChecksumValidation.WHEN_REQUIRED).build())
|
||||
{
|
||||
s3.createBucket(b -> b.bucket("worm-locked").objectLockEnabledForBucket(true));
|
||||
s3.createBucket(b -> b.bucket("worm-plain"));
|
||||
var config = ChainVerificationTest.signed();
|
||||
var instance = init(config);
|
||||
AlertsInstanceMetaData.ofOrWithNew(instance).withSink(new QCodeReference(ProcessesAndAlertsTest.Sink.class));
|
||||
ProcessesAndAlertsTest.Sink.raised.clear();
|
||||
System.setProperty("worm.test.access", "worm-test-access");
|
||||
System.setProperty("worm.test.secret", "worm-test-secret");
|
||||
ObjectStorage.configureStandalone(new ObjectStorageQBitConfig().withBackendName("memory")
|
||||
.withAllowedSecretRefPatterns(List.of("prop.worm.test.*"))
|
||||
.withConnection(new StorageConnectionConfig("minio").withProviderType(ProviderType.MINIO).withEndpoint(endpoint)
|
||||
.withRegion("us-east-1").withAddressingStyle(AddressingStyle.PATH).withChecksumMode(ChecksumMode.WHEN_REQUIRED)
|
||||
.withAccessKeyRef("${prop.worm.test.access}").withSecretKeyRef("${prop.worm.test.secret}"))
|
||||
.withBucket(new StorageBucketConfig("worm_audit").withConnection("minio").withBucketName("worm-locked").withRole(BucketRole.BACKUP).withIndexed(false))
|
||||
.withBucket(new StorageBucketConfig("plain").withConnection("minio").withBucketName("worm-plain").withRole(BucketRole.BACKUP).withIndexed(false)));
|
||||
CapabilityProbe.probeConnection("minio", false);
|
||||
assertThat(ObjectStorage.asSystem("test", "1").bucket("worm_audit").capability(Capability.OBJECT_LOCK)).isEqualTo(CapabilityStatus.SUPPORTED);
|
||||
assertThat(ObjectStorage.asSystem("test", "1").bucket("plain").capability(Capability.OBJECT_LOCK)).isEqualTo(CapabilityStatus.UNSUPPORTED);
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
Instant now = Instant.now();
|
||||
var exported = new SealedExporter(config).run("USER", now);
|
||||
assertThat(exported.failed()).as(exported.message()).isFalse();
|
||||
var row = exported.row();
|
||||
var storage = new ObjectStorageSealedStore("worm_audit", "acceptance");
|
||||
assertThat(storage.list(ListRequest.prefix(row.getValueString("object_prefix"))).getItems()).hasSize(3);
|
||||
var verifier = new SealedExportVerifier(config);
|
||||
assertThat(verifier.verify(row, "USER").result().passed()).isTrue();
|
||||
for(String name : ExportFiles.NAMES)
|
||||
{
|
||||
String key = row.getValueString("object_prefix") + name;
|
||||
var head = storage.head(key);
|
||||
assertThat(head.getLockMode()).isEqualTo("GOVERNANCE");
|
||||
assertThat(Duration.between(now.plus(Duration.ofDays(30)), head.getLockRetainUntil()).abs()).isLessThan(Duration.ofSeconds(2));
|
||||
String originalVersion = head.getVersionId();
|
||||
assertThat(originalVersion).isNotBlank();
|
||||
assertThat(ExportFiles.parse(row.getValueString("object_versions").getBytes(java.nio.charset.StandardCharsets.UTF_8)))
|
||||
.containsEntry(name, originalVersion);
|
||||
assertThatThrownBy(() -> s3.deleteObject(b -> b.bucket("worm-locked").key(key).versionId(originalVersion).bypassGovernanceRetention(false)))
|
||||
.isInstanceOf(S3Exception.class);
|
||||
byte[] original = s3.getObjectAsBytes(b -> b.bucket("worm-locked").key(key).versionId(originalVersion)).asByteArray();
|
||||
// S3 overwrites create new versions; they cannot replace the locked version.
|
||||
var replacement = s3.putObject(b -> b.bucket("worm-locked").key(key).objectLockMode("GOVERNANCE")
|
||||
.objectLockRetainUntilDate(row.getValueInstant("retain_until")), RequestBody.fromString("different replacement bytes"));
|
||||
assertThat(replacement.versionId()).isNotEqualTo(originalVersion);
|
||||
assertThat(s3.getObjectAsBytes(b -> b.bucket("worm-locked").key(key).versionId(originalVersion)).asByteArray()).isEqualTo(original);
|
||||
try(var sealed = storage.getVersion(key, originalVersion))
|
||||
{
|
||||
assertThat(sealed.readAllBytes()).isEqualTo(original);
|
||||
}
|
||||
var report = verifier.verify(row, "USER");
|
||||
assertThat(report.result().passed()).isFalse();
|
||||
assertThat(report.result().getLinksChecked()).isEqualTo(1);
|
||||
assertThat(report.result().getBreaks()).allSatisfy(b -> assertThat(b.reason()).contains("was overwritten after sealing"));
|
||||
assertThat(report.result().getBreaks().toString()).contains("object " + name + " was overwritten after sealing (current version "
|
||||
+ replacement.versionId() + ", sealed version " + originalVersion + " is still retained under Object Lock)");
|
||||
}
|
||||
tamper("auditDetail", 1, "newValue", "tampered live value");
|
||||
var load = new ExportProcesses.VerifyLoad();
|
||||
load.runOnePage(new RunBackendStepInput().withRecords(List.of(row)), new RunBackendStepOutput());
|
||||
assertThat(load.getProcessSummary(new RunBackendStepOutput(), true).toString())
|
||||
.contains("ERROR", "link 1 (audit 1): live content differs from the sealed copy");
|
||||
tamper("auditDetail", 1, "newValue", "new");
|
||||
audit(2);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
config.withExportLockMode("COMPLIANCE");
|
||||
var compliance = new SealedExporter(config).run("USER", Instant.now());
|
||||
assertThat(compliance.failed()).isFalse();
|
||||
assertThat(compliance.row().getValueString("lock_mode")).isEqualTo("COMPLIANCE");
|
||||
assertThat(verifier.verify(compliance.row(), "USER").result().passed()).isTrue();
|
||||
for(String name : ExportFiles.NAMES)
|
||||
{
|
||||
String key = compliance.row().getValueString("object_prefix") + name;
|
||||
var head = storage.head(key);
|
||||
assertThat(head.getLockMode()).isEqualTo("COMPLIANCE");
|
||||
assertThatThrownBy(() -> s3.deleteObject(b -> b.bucket("worm-locked").key(key).versionId(head.getVersionId())))
|
||||
.isInstanceOf(S3Exception.class);
|
||||
}
|
||||
audit(3);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
config.withExportBucketAlias("plain");
|
||||
var failed = new SealedExporter(config).run("SCHEDULE", Instant.now());
|
||||
assertThat(failed.failed()).isTrue();
|
||||
assertThat(failed.message()).contains("does not support Object Lock");
|
||||
assertThat(STORE.all(WormAuditTables.EXPORT)).hasSize(2);
|
||||
assertThat(ProcessesAndAlertsTest.Sink.raised).hasSize(1);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ObjectStorageRuntime.reset();
|
||||
CapabilityStore.clearCache();
|
||||
System.clearProperty("worm.test.access");
|
||||
System.clearProperty("worm.test.secret");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import javax.crypto.Mac;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Base64;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.UUID;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.actions.audits.AuditAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.audits.AuditActionCustomizerInterface;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QSupplementalFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.OLD;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.STORE;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.core;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.detail;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for phi canary test.
|
||||
*******************************************************************************/
|
||||
public class PhiCanaryTest
|
||||
{
|
||||
// RC.5 has AiFieldMetaData but not DataClassification; use a test-local marker.
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for phi.
|
||||
*******************************************************************************/
|
||||
public static class Phi extends QSupplementalFieldMetaData
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Verify get type.
|
||||
*******************************************************************************/
|
||||
public String getType()
|
||||
{
|
||||
return ("test-phi");
|
||||
}
|
||||
}
|
||||
|
||||
// Real RC.5 AuditAction customization point: rewrite details BEFORE insertion.
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for token hook.
|
||||
*******************************************************************************/
|
||||
public static class TokenHook implements AuditActionCustomizerInterface
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Verify customize input.
|
||||
*******************************************************************************/
|
||||
public void customizeInput(AuditSingleInput input)
|
||||
{
|
||||
for(QRecord detail : input.getDetails())
|
||||
{
|
||||
var field =
|
||||
QContext.getQInstance()
|
||||
.getTable(input.getAuditTableName())
|
||||
.getField(detail.getValueString("fieldName"));
|
||||
if(field.getSupplementalMetaData("test-phi") != null)
|
||||
{
|
||||
for(String column : List.of("oldValue", "newValue"))
|
||||
{
|
||||
if(detail.getValueString(column) == null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
Mac mac = Mac.getInstance("HmacSHA256");
|
||||
mac.init(
|
||||
new SecretKeySpec(
|
||||
"fixture-key-not-production"
|
||||
.getBytes(StandardCharsets.UTF_8),
|
||||
"HmacSHA256"));
|
||||
detail.setValue(
|
||||
column,
|
||||
"hmac:v1:test:"
|
||||
+ Base64.getUrlEncoder()
|
||||
.withoutPadding()
|
||||
.encodeToString(
|
||||
mac.doFinal(
|
||||
detail.getValueString(column)
|
||||
.getBytes(
|
||||
StandardCharsets
|
||||
.UTF_8))));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
}
|
||||
detail.setValue("message", "Changed Diagnosis");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify token hook keeps every audit and chain value free of canary.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void tokenHookKeepsEveryAuditAndChainValueFreeOfCanary() throws Exception
|
||||
{
|
||||
var config = ChainVerificationTest.signed();
|
||||
var instance = init(config);
|
||||
instance.addTable(
|
||||
new QTableMetaData()
|
||||
.withName("patient")
|
||||
.withBackendName("memory")
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
|
||||
.withField(
|
||||
new QFieldMetaData("diagnosis", QFieldType.STRING)
|
||||
.withSupplementalMetaData(new Phi())));
|
||||
instance.addSupplementalCustomizer(
|
||||
AuditActionCustomizerInterface.CUSTOMIZER_TYPE,
|
||||
new QCodeReference(TokenHook.class));
|
||||
String canary = "CANARY-PHI-" + UUID.randomUUID();
|
||||
var detail =
|
||||
new QRecord()
|
||||
.withValue("fieldName", "diagnosis")
|
||||
.withValue("oldValue", canary)
|
||||
.withValue("newValue", canary)
|
||||
.withValue("message", "Changed Diagnosis from " + canary + " to " + canary);
|
||||
new AuditAction()
|
||||
.execute(
|
||||
new AuditInput()
|
||||
.withAuditSingleInput(
|
||||
new AuditSingleInput()
|
||||
.withAuditTableName("patient")
|
||||
.withAuditUserName("james")
|
||||
.withRecordId(17)
|
||||
.withTimestamp(OLD)
|
||||
.withMessage("Record was Edited")
|
||||
.withDetails(List.of(detail))));
|
||||
assertThat(STORE.all("audit")).hasSize(1);
|
||||
assertThat(STORE.all("auditDetail")).hasSize(1);
|
||||
assertThat(STORE.all("auditDetail").getFirst().getValueString("newValue"))
|
||||
.startsWith("hmac:v1:test:");
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
List<String> values = new ArrayList<>();
|
||||
for(String table :
|
||||
List.of(
|
||||
"audit",
|
||||
"auditDetail",
|
||||
WormAuditTables.CHAIN,
|
||||
WormAuditTables.CHECKPOINT))
|
||||
{
|
||||
STORE.all(table)
|
||||
.forEach(
|
||||
r ->
|
||||
r.getValues().values().stream()
|
||||
.filter(Objects::nonNull)
|
||||
.map(Object::toString)
|
||||
.forEach(values::add));
|
||||
}
|
||||
for(var link : STORE.all(WormAuditTables.CHAIN))
|
||||
{
|
||||
values.add(
|
||||
new String(
|
||||
STORE.canonical(
|
||||
STORE.get("audit", link.getValueLong("audit_id")),
|
||||
AuditStore.detailIds(link)),
|
||||
StandardCharsets.UTF_8));
|
||||
}
|
||||
for(String value : values)
|
||||
{
|
||||
assertThat(value).doesNotContain(canary);
|
||||
// Every longer substring contains a 5-character substring.
|
||||
for(int start = 0; start <= canary.length() - 5; start++)
|
||||
{
|
||||
assertThat(value).doesNotContain(canary.substring(start, start + 5));
|
||||
}
|
||||
}
|
||||
assertThat(new WormAuditVerifier(config).verify().passed()).isTrue();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,597 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.testcontainers.containers.PostgreSQLContainer;
|
||||
import org.testcontainers.junit.jupiter.Container;
|
||||
import org.testcontainers.junit.jupiter.Testcontainers;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.STORE;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.core;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.detail;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.lookups;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for postgres acceptance test.
|
||||
*******************************************************************************/
|
||||
@ExtendWith(RequiresDockerCondition.class)
|
||||
@Testcontainers
|
||||
class PostgresAcceptanceTest
|
||||
{
|
||||
@Container
|
||||
static final PostgreSQLContainer<?> POSTGRES = new PostgreSQLContainer<>("postgres:16-alpine");
|
||||
|
||||
WormAuditQBitConfig config;
|
||||
QInstance instance;
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify owner.
|
||||
*******************************************************************************/
|
||||
Connection owner() throws SQLException
|
||||
{
|
||||
return (DriverManager.getConnection(
|
||||
POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword()));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify app.
|
||||
*******************************************************************************/
|
||||
Connection app() throws SQLException
|
||||
{
|
||||
return (DriverManager.getConnection(POSTGRES.getJdbcUrl(), "app", "fixture"));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify sql.
|
||||
*******************************************************************************/
|
||||
void sql(Connection c, String sql) throws SQLException
|
||||
{
|
||||
try(var s = c.createStatement())
|
||||
{
|
||||
s.execute(sql);
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify count.
|
||||
*******************************************************************************/
|
||||
long count(Connection c, String sql) throws SQLException
|
||||
{
|
||||
try(var s = c.createStatement();
|
||||
var r = s.executeQuery(sql))
|
||||
{
|
||||
r.next();
|
||||
return (r.getLong(1));
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify database.
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void database() throws Exception
|
||||
{
|
||||
config = ChainVerificationTest.signed().withSettleHorizonSeconds(0);
|
||||
try(var c = owner())
|
||||
{
|
||||
sql(
|
||||
c,
|
||||
"DROP SCHEMA public CASCADE; CREATE SCHEMA public; GRANT USAGE ON SCHEMA public"
|
||||
+ " TO PUBLIC");
|
||||
sql(
|
||||
c,
|
||||
"DO $$ BEGIN IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='app') THEN"
|
||||
+ " CREATE ROLE app LOGIN PASSWORD 'fixture'; END IF; END $$");
|
||||
instance = RdbmsTestSupport.instance(POSTGRES.getJdbcUrl(), "app", "fixture", config, true);
|
||||
RdbmsTestSupport.createTables(c, instance, true);
|
||||
String ddl = WormAuditDdl.postgres(config);
|
||||
sql(c, ddl);
|
||||
sql(c, ddl);
|
||||
}
|
||||
QContext.init(instance, new QSession());
|
||||
lookups();
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Home setup probes see dropped/disabled guards, missing purge and runtime ownership.
|
||||
** This test uses the same Docker condition as the existing PostgreSQL suite.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void homeSetupWarnings() throws Exception
|
||||
{
|
||||
var storage = new com.kingsrook.qbits.wormaudit.demo.MemorySealedStore();
|
||||
var clean = com.kingsrook.qbits.wormaudit.app.WormAuditHomeData.load(config, storage, java.time.Instant.now());
|
||||
assertThat(clean.warnings()).contains("Daily verification has never run").noneMatch(w -> w.contains("guard") && w.contains("settings"));
|
||||
try(var connection = owner())
|
||||
{
|
||||
sql(connection, "UPDATE worm_audit_settings SET audit_retention_years=7 WHERE id=1");
|
||||
config.invalidateSettings();
|
||||
assertThat(com.kingsrook.qbits.wormaudit.app.WormAuditHomeData.load(config, storage, java.time.Instant.now()).warnings())
|
||||
.anyMatch(w -> w.contains("purge function's compiled interval (6 years)"));
|
||||
sql(connection, WormAuditDdl.postgres(config));
|
||||
assertThat(count(connection, "SELECT audit_retention_years FROM worm_audit_settings WHERE id=1")).isEqualTo(7);
|
||||
sql(connection, "UPDATE worm_audit_settings SET audit_retention_years=6 WHERE id=1");
|
||||
config.invalidateSettings();
|
||||
sql(connection, "DROP TRIGGER worm_audit_row_guard ON audit_detail");
|
||||
sql(connection, "ALTER TABLE worm_audit_chain DISABLE TRIGGER worm_audit_truncate_guard");
|
||||
sql(connection, "DROP FUNCTION public.worm_audit_purge(timestamptz)");
|
||||
}
|
||||
var missing = com.kingsrook.qbits.wormaudit.app.WormAuditHomeData.load(config, storage, java.time.Instant.now());
|
||||
assertThat(missing.warnings()).contains("Missing or disabled worm_audit_row_guard on audit_detail",
|
||||
"Missing or disabled worm_audit_truncate_guard on worm_audit_chain", "Retention purge function is missing");
|
||||
instance = RdbmsTestSupport.instance(POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword(), config, true);
|
||||
QContext.init(instance, new QSession());
|
||||
assertThat(com.kingsrook.qbits.wormaudit.app.WormAuditHomeData.load(config, storage, java.time.Instant.now()).warnings())
|
||||
.contains("Runtime role owns audit table audit", "Runtime role owns audit table worm_audit_purge");
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify grants triggers lookup rules and spoofed flag.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void grantsTriggersLookupRulesAndSpoofedFlag() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
try(var c = owner();
|
||||
var runtime = app())
|
||||
{
|
||||
assertThatThrownBy(() -> sql(runtime,
|
||||
"INSERT INTO worm_audit_chain(id,kind,audit_id,detail_ids,prev_hash,link_hash,chained_at) "
|
||||
+ "SELECT 99,'LATE_DETAIL',audit_id,detail_ids,prev_hash,link_hash,chained_at FROM worm_audit_chain WHERE id=1"))
|
||||
.isInstanceOf(SQLException.class).satisfies(e -> assertThat(((SQLException) e).getSQLState()).isEqualTo("23505"));
|
||||
|
||||
for(String table :
|
||||
List.of("worm_audit_export", "worm_audit_verification", "worm_audit_purge"))
|
||||
{
|
||||
sql(c, "INSERT INTO " + table + " DEFAULT VALUES");
|
||||
}
|
||||
List<String> tables =
|
||||
List.of(
|
||||
"audit",
|
||||
"audit_detail",
|
||||
"worm_audit_chain",
|
||||
"worm_audit_checkpoint",
|
||||
"worm_audit_export",
|
||||
"worm_audit_verification",
|
||||
"worm_audit_purge");
|
||||
for(String table : tables)
|
||||
{
|
||||
assertThat(
|
||||
count(
|
||||
c,
|
||||
"SELECT has_table_privilege('app','"
|
||||
+ table
|
||||
+ "','UPDATE')::int"))
|
||||
.isZero();
|
||||
assertThat(
|
||||
count(
|
||||
c,
|
||||
"SELECT has_table_privilege('app','"
|
||||
+ table
|
||||
+ "','DELETE')::int"))
|
||||
.isZero();
|
||||
assertThat(
|
||||
count(
|
||||
c,
|
||||
"SELECT has_table_privilege('app','"
|
||||
+ table
|
||||
+ "','TRUNCATE')::int"))
|
||||
.isZero();
|
||||
// PostgreSQL checks grants before firing triggers: an unprivileged app
|
||||
// receives 42501. Temporarily grant DML to prove the trigger defense too.
|
||||
assertThatThrownBy(() -> sql(runtime, "UPDATE " + table + " SET id=id"))
|
||||
.isInstanceOf(SQLException.class)
|
||||
.satisfies(
|
||||
e ->
|
||||
assertThat(((SQLException) e).getSQLState())
|
||||
.isEqualTo("42501"));
|
||||
sql(c, "GRANT UPDATE, DELETE, TRUNCATE ON " + table + " TO app");
|
||||
for(String operation :
|
||||
List.of(
|
||||
"UPDATE " + table + " SET id=id",
|
||||
"DELETE FROM " + table,
|
||||
"TRUNCATE " + table))
|
||||
{
|
||||
assertThatThrownBy(() -> sql(runtime, operation))
|
||||
.isInstanceOf(SQLException.class)
|
||||
.hasMessageContaining("worm_audit:")
|
||||
.satisfies(
|
||||
e ->
|
||||
assertThat(((SQLException) e).getSQLState())
|
||||
.isEqualTo("P0001"));
|
||||
}
|
||||
}
|
||||
sql(runtime, "SELECT set_config('worm_audit.purging','on',false)");
|
||||
assertThatThrownBy(() -> sql(runtime, "DELETE FROM audit"))
|
||||
.hasMessageContaining("worm_audit:");
|
||||
sql(runtime, "UPDATE audit_table SET label='Allowed label'");
|
||||
sql(runtime, "UPDATE audit_user SET modify_date=now()");
|
||||
for(String table : List.of("audit_table", "audit_user"))
|
||||
{
|
||||
sql(c, "GRANT UPDATE, DELETE ON " + table + " TO app");
|
||||
assertThatThrownBy(() -> sql(runtime, "UPDATE " + table + " SET name='renamed'"))
|
||||
.hasMessageContaining("worm_audit:");
|
||||
assertThatThrownBy(() -> sql(runtime, "UPDATE " + table + " SET id=id+1"))
|
||||
.hasMessageContaining("worm_audit:");
|
||||
assertThatThrownBy(() -> sql(runtime, "DELETE FROM " + table))
|
||||
.hasMessageContaining("worm_audit:");
|
||||
assertThatThrownBy(() -> sql(c, "TRUNCATE " + table))
|
||||
.hasMessageContaining("worm_audit:");
|
||||
sql(runtime, "UPDATE " + table + " SET modify_date=now()");
|
||||
}
|
||||
sql(runtime, "UPDATE audit_table SET label='New label'");
|
||||
sql(c, WormAuditDdl.postgres(config));
|
||||
assertThat(
|
||||
count(
|
||||
c,
|
||||
"SELECT"
|
||||
+ " has_function_privilege('app','worm_audit_purge(timestamptz)','EXECUTE')::int"))
|
||||
.isZero();
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify retention refusal purge anchor and real backend tamper.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void retentionRefusalPurgeAnchorAndRealBackendTamper() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
audit(2);
|
||||
detail(2, 2);
|
||||
// A live suffix remains after the expired prefix is removed.
|
||||
try(var c = owner())
|
||||
{
|
||||
sql(
|
||||
c,
|
||||
"INSERT INTO audit(id,audit_table_id,audit_user_id,record_id,message,timestamp)"
|
||||
+ " VALUES(3,1,1,17,'Record was Edited',now()-interval '2 minutes')");
|
||||
}
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).isEmpty();
|
||||
String anchor = STORE.get(WormAuditTables.CHAIN, 2).getValueString("link_hash");
|
||||
try(var c = owner())
|
||||
{
|
||||
sql(
|
||||
c,
|
||||
"DO $$ BEGIN IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='purge_login')"
|
||||
+ " THEN CREATE ROLE purge_login LOGIN PASSWORD 'purger-fixture'; END IF;"
|
||||
+ " END $$; GRANT worm_audit_purger TO purge_login");
|
||||
}
|
||||
try(var purger =
|
||||
DriverManager.getConnection(
|
||||
POSTGRES.getJdbcUrl(), "purge_login", "purger-fixture"))
|
||||
{
|
||||
sql(purger, "SELECT worm_audit_purge(now())");
|
||||
assertThat(STORE.all(WormAuditTables.PURGE).getFirst().getValueString("outcome"))
|
||||
.isEqualTo("REFUSED");
|
||||
sql(purger, "SELECT worm_audit_purge(now()-interval '6 years')");
|
||||
}
|
||||
var purge = STORE.all(WormAuditTables.PURGE).getLast();
|
||||
assertThat(purge.getValueString("outcome")).isEqualTo("PURGED");
|
||||
assertThat(purge.getValueString("requested_by")).isEqualTo("purge_login");
|
||||
assertThat(purge.getValueLong("audits_deleted")).isEqualTo(2);
|
||||
assertThat(purge.getValueLong("details_deleted")).isEqualTo(2);
|
||||
assertThat(purge.getValueLong("links_deleted")).isEqualTo(2);
|
||||
assertThat(purge.getValueString("anchor_hash")).isEqualTo(anchor);
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).isEmpty();
|
||||
detail(3, 3);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
try(var c = owner())
|
||||
{
|
||||
sql(
|
||||
c,
|
||||
"ALTER TABLE audit_detail DISABLE TRIGGER ALL; UPDATE audit_detail SET"
|
||||
+ " new_value='tampered' WHERE id=3; ALTER TABLE audit_detail ENABLE"
|
||||
+ " TRIGGER ALL");
|
||||
}
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks())
|
||||
.anyMatch(
|
||||
b ->
|
||||
b.reason()
|
||||
.contains(
|
||||
"Break at link 4 (audit 3, table cms_page,"
|
||||
+ " record 17)")
|
||||
&& b.reason().contains("audit_detail 3"));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify prefix cannot split an audit and advisory lock skips tick.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void prefixCannotSplitAnAuditAndAdvisoryLockSkipsTick() throws Exception
|
||||
{
|
||||
// This retention fixture deliberately polls a detail for a six-year-old header.
|
||||
setting("late_detail_lookback_hours", 24 * 365 * 10);
|
||||
audit(1);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
try(var c = owner())
|
||||
{
|
||||
sql(
|
||||
c,
|
||||
"INSERT INTO audit(id,audit_table_id,audit_user_id,record_id,message,timestamp)"
|
||||
+ " VALUES(2,1,1,17,'Record was Edited',now()-interval '2 minutes')");
|
||||
}
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
detail(1, 1);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
try(var c = owner())
|
||||
{
|
||||
sql(c, "SELECT worm_audit_purge(now()-interval '6 years')");
|
||||
assertThat(STORE.all(WormAuditTables.PURGE).getLast().getValueString("outcome"))
|
||||
.isEqualTo("REFUSED");
|
||||
assertThat(STORE.all(WormAuditTables.CHAIN)).hasSize(3);
|
||||
c.setAutoCommit(false);
|
||||
sql(c, "SELECT pg_advisory_xact_lock(" + WormAuditChainer.ADVISORY_LOCK + ")");
|
||||
assertThat(new WormAuditChainer(config).chainOnce().skipped()).isTrue();
|
||||
c.rollback();
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify eight writers and three chainers with late commits.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void eightWritersAndThreeChainersWithLateCommits() throws Exception
|
||||
{
|
||||
var done = new AtomicBoolean(false);
|
||||
var executor = Executors.newFixedThreadPool(11);
|
||||
List<Future<?>> writers = new ArrayList<>();
|
||||
List<Future<?>> chainers = new ArrayList<>();
|
||||
try
|
||||
{
|
||||
for(int thread = 0; thread < 8; thread++)
|
||||
{
|
||||
final int writer = thread;
|
||||
writers.add(
|
||||
executor.submit(
|
||||
() ->
|
||||
{
|
||||
try(var c = app())
|
||||
{
|
||||
for(int i = 0; i < 8; i++)
|
||||
{
|
||||
long id = 1000L + writer * 100 + i;
|
||||
c.setAutoCommit(false);
|
||||
sql(
|
||||
c,
|
||||
"INSERT INTO"
|
||||
+ " audit(id,audit_table_id,audit_user_id,record_id,message,timestamp)"
|
||||
+ " VALUES("
|
||||
+ id
|
||||
+ ",1,1,17,'Record was"
|
||||
+ " Edited',now()-interval '2"
|
||||
+ " minutes')");
|
||||
if(i % 2 == 0)
|
||||
{
|
||||
c.commit();
|
||||
}
|
||||
Thread.sleep(5L * (writer + 1));
|
||||
sql(
|
||||
c,
|
||||
"INSERT INTO"
|
||||
+ " audit_detail(id,audit_id,field_name,message,old_value,new_value)"
|
||||
+ " VALUES("
|
||||
+ id
|
||||
+ ","
|
||||
+ id
|
||||
+ ",'title','Changed"
|
||||
+ " Title','old','new')");
|
||||
c.commit();
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}));
|
||||
}
|
||||
for(int i = 0; i < 3; i++)
|
||||
{
|
||||
chainers.add(
|
||||
executor.submit(
|
||||
() ->
|
||||
{
|
||||
QContext.init(instance, new QSession());
|
||||
try
|
||||
{
|
||||
do
|
||||
{
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
Thread.sleep(10);
|
||||
} while(!done.get());
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}));
|
||||
}
|
||||
for(var writer : writers)
|
||||
{
|
||||
writer.get(90, TimeUnit.SECONDS);
|
||||
}
|
||||
done.set(true);
|
||||
for(var chainer : chainers)
|
||||
{
|
||||
chainer.get(90, TimeUnit.SECONDS);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
done.set(true);
|
||||
executor.shutdownNow();
|
||||
}
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
var links = STORE.all(WormAuditTables.CHAIN);
|
||||
for(int i = 0; i < links.size(); i++)
|
||||
{
|
||||
assertThat(links.get(i).getValueLong("id")).isEqualTo(i + 1L);
|
||||
}
|
||||
assertThat(links.stream().filter(r -> "AUDIT".equals(r.getValueString("kind"))).count())
|
||||
.isEqualTo(64);
|
||||
assertThat(links.stream().flatMap(r -> AuditStore.detailIds(r).stream()).toList())
|
||||
.hasSize(64)
|
||||
.doesNotHaveDuplicates();
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).isEmpty();
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Even the function owner cannot authorize direct deletes by setting a GUC.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void ownerCannotSpoofPurgeStack() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
try(var connection = owner())
|
||||
{
|
||||
sql(connection, "ALTER TABLE audit ALTER COLUMN timestamp TYPE timestamp without time zone USING timestamp AT TIME ZONE 'UTC'");
|
||||
sql(connection, "INSERT INTO audit(id,audit_table_id,audit_user_id,timestamp) VALUES"
|
||||
+ "(2,1,1,(now() AT TIME ZONE 'UTC')-interval '6 years'-interval '1 hour')");
|
||||
}
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
try(var connection = owner())
|
||||
{
|
||||
sql(connection, "SELECT set_config('worm_audit.purging','on',false)");
|
||||
for(String table : List.of("audit", "audit_detail", "worm_audit_chain"))
|
||||
{
|
||||
assertThatThrownBy(() -> sql(connection, "DELETE FROM " + table + " WHERE id=1"))
|
||||
.isInstanceOf(SQLException.class).hasMessageContaining("worm_audit:");
|
||||
}
|
||||
assertThat(count(connection, "SELECT count(*) FROM worm_audit_purge")).isZero();
|
||||
// This succeeds only if the PG16 context text matches the stack guard.
|
||||
sql(connection, "SET timezone='America/Chicago'");
|
||||
sql(connection, "SELECT worm_audit_purge(now()-interval '6 years')");
|
||||
assertThat(count(connection, "SELECT count(*) FROM audit")).isZero();
|
||||
assertThat(count(connection, "SELECT count(*) FROM worm_audit_purge WHERE outcome='PURGED'")).isEqualTo(1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A UTC without-tz audit column settles identically in a Chicago JVM/session.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void chainerUsesUtcCutoffInChicago() throws Exception
|
||||
{
|
||||
var original = java.util.TimeZone.getDefault();
|
||||
setting("settle_horizon_seconds", 60);
|
||||
try(var connection = owner())
|
||||
{
|
||||
sql(connection, "ALTER TABLE audit ALTER COLUMN timestamp TYPE timestamp without time zone USING timestamp AT TIME ZONE 'UTC'");
|
||||
sql(connection, "INSERT INTO audit(id,audit_table_id,audit_user_id,timestamp) VALUES"
|
||||
+ "(1,1,1,TIMESTAMP '2026-10-13 12:00:00'),(2,1,1,TIMESTAMP '2026-10-13 12:01:30')");
|
||||
// New runtime connections inherit this non-UTC session setting.
|
||||
sql(connection, "ALTER ROLE app SET timezone='America/Chicago'");
|
||||
java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("America/Chicago"));
|
||||
assertThat(new WormAuditChainer(config).chainOnce(java.time.Instant.parse("2026-10-13T12:02:00Z")).audits()).isEqualTo(1);
|
||||
assertThat(STORE.all(WormAuditTables.CHAIN)).extracting(r -> r.getValueLong("audit_id")).containsExactly(1L);
|
||||
java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("UTC"));
|
||||
assertThat(new WormAuditVerifier(config).verify(java.util.Set.of(1L)).getBreaks()).isEmpty();
|
||||
}
|
||||
finally
|
||||
{
|
||||
java.util.TimeZone.setDefault(original);
|
||||
try(var connection = owner())
|
||||
{
|
||||
sql(connection, "ALTER ROLE app RESET timezone");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** PostgreSQL containment polling includes the lower window boundary only.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void postgresLateDetailWindowAndCompleteness() throws Exception
|
||||
{
|
||||
setting("settle_horizon_seconds", 60);
|
||||
var now = java.time.Instant.now().truncatedTo(java.time.temporal.ChronoUnit.SECONDS);
|
||||
var cutoff = now.minusSeconds(60);
|
||||
try(var connection = owner(); var statement = connection.prepareStatement(
|
||||
"INSERT INTO audit(id,audit_table_id,audit_user_id,timestamp) VALUES(?,1,1,?)"))
|
||||
{
|
||||
for(int id = 1; id <= 4; id++)
|
||||
{
|
||||
long seconds = switch(id)
|
||||
{
|
||||
case 1 -> 86401;
|
||||
case 2 -> 86400;
|
||||
case 3 -> 1;
|
||||
default -> 0;
|
||||
};
|
||||
statement.setInt(1, id);
|
||||
statement.setObject(2, cutoff.minusSeconds(seconds).atOffset(java.time.ZoneOffset.UTC));
|
||||
statement.executeUpdate();
|
||||
}
|
||||
}
|
||||
var writer = new WormAuditChainer(config);
|
||||
assertThat(writer.chainOnce(now).audits()).isEqualTo(3);
|
||||
for(long id = 1; id <= 4; id++)
|
||||
{
|
||||
detail(id, id);
|
||||
}
|
||||
assertThat(writer.chainOnce(now).lateDetails()).isEqualTo(2);
|
||||
assertThat(STORE.all(WormAuditTables.CHAIN).stream().flatMap(r -> AuditStore.detailIds(r).stream()).toList()).containsExactly(2L, 3L);
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).anyMatch(b -> b.reason().contains("audit_detail 1 is not covered"));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The settings row (seeded by the DDL) overrides config: change it there.
|
||||
*******************************************************************************/
|
||||
private void setting(String column, int value) throws Exception
|
||||
{
|
||||
try(var connection = owner())
|
||||
{
|
||||
sql(connection, "UPDATE worm_audit_settings SET " + column + "=" + value + " WHERE id=1");
|
||||
}
|
||||
config.invalidateSettings();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,266 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.VerificationRuns;
|
||||
import com.kingsrook.qbits.wormaudit.processes.WormAuditProcesses;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertResolution;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertSignal;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertSinkInterface;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunInput;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunOutput;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.STORE;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.core;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.detail;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.tamper;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for processes and alerts test.
|
||||
*******************************************************************************/
|
||||
class ProcessesAndAlertsTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for sink.
|
||||
*******************************************************************************/
|
||||
public static class Sink implements AlertSinkInterface
|
||||
{
|
||||
static final List<AlertSignal> raised = new ArrayList<>();
|
||||
static final List<AlertResolution> resolved = new ArrayList<>();
|
||||
static final List<String> beats = new ArrayList<>();
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify raise.
|
||||
*******************************************************************************/
|
||||
public void raise(AlertSignal signal)
|
||||
{
|
||||
raised.add(signal);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify resolve.
|
||||
*******************************************************************************/
|
||||
public void resolve(AlertResolution resolution)
|
||||
{
|
||||
resolved.add(resolution);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify heartbeat.
|
||||
*******************************************************************************/
|
||||
public void heartbeat(String key, Instant at)
|
||||
{
|
||||
beats.add(key);
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** RC.5 validates declared severities, names and daily heartbeat expectations.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void alertTypesAndDailyMonitors() throws Exception
|
||||
{
|
||||
var instance = init(new WormAuditQBitConfig());
|
||||
new com.kingsrook.qqq.backend.core.instances.QInstanceValidator().validate(instance);
|
||||
assertThat(new com.kingsrook.qbits.wormaudit.chain.AuditStore(null).isMemory()).isTrue();
|
||||
var types = AlertsInstanceMetaData.of(instance).getAlertTypes();
|
||||
assertThat(types).containsOnlyKeys("wormaudit.chain.break", "wormaudit.export.failed", "wormaudit.verify.daily", "wormaudit.export.daily");
|
||||
assertThat(types.get("wormaudit.chain.break").getDefaultSeverity())
|
||||
.isEqualTo(com.kingsrook.qqq.backend.core.alerts.AlertSeverity.CRITICAL);
|
||||
assertThat(types.get("wormaudit.export.failed").getDefaultSeverity())
|
||||
.isEqualTo(com.kingsrook.qqq.backend.core.alerts.AlertSeverity.ERROR);
|
||||
types.values().forEach(type ->
|
||||
{
|
||||
assertThat(type.getLabel()).isNotBlank();
|
||||
assertThat(type.getDescription()).isNotBlank();
|
||||
assertThat(AlertsInstanceMetaData.ALERT_TYPE_NAME_PATTERN.matcher(type.getName()).matches()).isTrue();
|
||||
});
|
||||
for(String name : List.of("wormaudit.verify.daily", "wormaudit.export.daily"))
|
||||
{
|
||||
var monitors = types.get(name).getDefaultMonitors();
|
||||
assertThat(monitors).hasSize(1);
|
||||
assertThat(monitors.getFirst().getKind()).isEqualTo(com.kingsrook.qqq.backend.core.alerts.AlertMonitorKind.HEARTBEAT);
|
||||
assertThat(monitors.getFirst().getDefinition()).containsEntry("heartbeatKey", name)
|
||||
.containsEntry("expectedEverySeconds", 86400).containsEntry("graceSeconds", 7200);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify daily raise resolve heartbeat and user result.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void dailyRaiseResolveHeartbeatAndUserResult() throws Exception
|
||||
{
|
||||
var config = ChainVerificationTest.signed();
|
||||
var instance = init(config);
|
||||
Sink.raised.clear();
|
||||
Sink.resolved.clear();
|
||||
Sink.beats.clear();
|
||||
AlertsInstanceMetaData.ofOrWithNew(instance).withSink(new QCodeReference(Sink.class));
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
new WormAuditProcesses.ChainTick()
|
||||
.run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
tamper("auditDetail", 1, "newValue", "broken");
|
||||
var daily = new WormAuditProcesses.Daily();
|
||||
var output = new RunBackendStepOutput();
|
||||
daily.run(new RunBackendStepInput(), output);
|
||||
assertThat(output.getValueString("outcome")).isEqualTo("FAIL");
|
||||
assertThat(Sink.raised).hasSize(1);
|
||||
assertThat(Sink.raised.getFirst().getAlertTypeName()).isEqualTo("wormaudit.chain.break");
|
||||
assertThat(Sink.raised.getFirst().getDedupKey()).isEqualTo("chain");
|
||||
assertThat(Sink.raised.getFirst().getRecordRefs().getFirst().getTableName())
|
||||
.isEqualTo(WormAuditTables.VERIFICATION);
|
||||
assertThat(Sink.raised.getFirst().getTitle()).contains("Break at link 1");
|
||||
var first = STORE.all(WormAuditTables.VERIFICATION).getFirst();
|
||||
assertThat(first.getValueLong("first_break_link_id")).isEqualTo(1);
|
||||
assertThat(first.getValueLong("first_break_audit_id")).isEqualTo(1);
|
||||
assertThat(first.getValueString("trigger")).isEqualTo("SCHEDULE");
|
||||
tamper("auditDetail", 1, "newValue", "new");
|
||||
daily.run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
assertThat(Sink.resolved).hasSize(1);
|
||||
assertThat(Sink.beats)
|
||||
.containsExactly("wormaudit.verify.daily", "wormaudit.verify.daily");
|
||||
assertThat(STORE.all(WormAuditTables.VERIFICATION)).hasSize(2);
|
||||
assertThat(VerificationRuns.run(null, "USER").passed()).isTrue();
|
||||
assertThat(Sink.beats).hasSize(2);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify review and load across pages including failures and empty scope.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void reviewAndLoadAcrossPagesIncludingFailuresAndEmptyScope() throws Exception
|
||||
{
|
||||
var config = new WormAuditQBitConfig();
|
||||
init(config);
|
||||
audit(1);
|
||||
audit(2);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
var links = STORE.all(WormAuditTables.CHAIN);
|
||||
var review = new WormAuditProcesses.Review();
|
||||
var load = new WormAuditProcesses.Load();
|
||||
var output = new RunBackendStepOutput();
|
||||
assertThat(load.getProcessSummary(output, true)).isEmpty();
|
||||
for(var link : links)
|
||||
{
|
||||
var input = new RunBackendStepInput().withRecords(List.of(link));
|
||||
review.runOnePage(input, output);
|
||||
load.runOnePage(input, output);
|
||||
}
|
||||
assertThat(review.getProcessSummary(output, false).toString())
|
||||
.contains("2 chain links will be verified", "0 checkpoints");
|
||||
load.postRun(
|
||||
new BackendStepPostRunInput(new RunBackendStepInput()),
|
||||
new BackendStepPostRunOutput(output));
|
||||
assertThat(STORE.all(WormAuditTables.VERIFICATION)).hasSize(1);
|
||||
assertThat(load.getProcessSummary(output, true).toString())
|
||||
.contains("2 links verified", "no signing key configured");
|
||||
audit(3);
|
||||
var fullWithEmptyFilter = new WormAuditProcesses.Load();
|
||||
fullWithEmptyFilter.runOnePage(new RunBackendStepInput().withRecords(links), output);
|
||||
fullWithEmptyFilter.postRun(new BackendStepPostRunInput(new RunBackendStepInput().addValue("queryFilterJson", "{}")),
|
||||
new BackendStepPostRunOutput(output));
|
||||
assertThat(fullWithEmptyFilter.getProcessSummary(output, true).toString()).contains("audit 3 is not covered by the chain");
|
||||
tamper("audit", 1, "message", "changed");
|
||||
var selected = new WormAuditProcesses.Load();
|
||||
selected.runOnePage(
|
||||
new RunBackendStepInput().withRecords(List.of(links.getFirst())), output);
|
||||
selected.postRun(
|
||||
new BackendStepPostRunInput(new RunBackendStepInput().addValue("recordIds", "1")),
|
||||
new BackendStepPostRunOutput(output));
|
||||
assertThat(selected.getProcessSummary(output, true).toString())
|
||||
.contains("ERROR", "content differs");
|
||||
init(config);
|
||||
var empty = new WormAuditProcesses.Load();
|
||||
empty.postRun(
|
||||
new BackendStepPostRunInput(new RunBackendStepInput()),
|
||||
new BackendStepPostRunOutput(output));
|
||||
assertThat(empty.getProcessSummary(output, true).toString()).contains("0 links verified");
|
||||
assertThat(new WormAuditProcesses.Review().getProcessSummary(output, false).toString())
|
||||
.contains("links none");
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Full loads retain no id collection; review summarizes disjoint streamed pages once.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void reviewRangesAndSelectionOnlyIds() throws Exception
|
||||
{
|
||||
var config = ChainVerificationTest.signed().withCheckpointEveryLinks(2);
|
||||
init(config);
|
||||
for(long id = 1; id <= 4; id++)
|
||||
{
|
||||
audit(id);
|
||||
}
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
var links = STORE.all(WormAuditTables.CHAIN);
|
||||
var review = new WormAuditProcesses.Review();
|
||||
var full = new WormAuditProcesses.Load();
|
||||
var selected = new WormAuditProcesses.Load();
|
||||
var output = new RunBackendStepOutput();
|
||||
var ids = WormAuditProcesses.Load.class.getDeclaredField("ids");
|
||||
ids.setAccessible(true);
|
||||
for(var link : links)
|
||||
{
|
||||
var input = new RunBackendStepInput().withRecords(List.of(link));
|
||||
review.runOnePage(input, output);
|
||||
full.runOnePage(input, output);
|
||||
assertThat(ids.get(full)).isNull();
|
||||
}
|
||||
String summary = review.getProcessSummary(output, false).toString();
|
||||
assertThat(summary).contains("4 chain links", "links 1–4", "2 checkpoints");
|
||||
// A second call reuses the count; later inserts cannot silently change Review.
|
||||
STORE.insert(WormAuditTables.CHECKPOINT, List.of(new com.kingsrook.qqq.backend.core.model.data.QRecord()
|
||||
.withValue("from_link_id", 1L).withValue("to_link_id", 4L)));
|
||||
assertThat(review.getProcessSummary(output, false).toString()).isEqualTo(summary);
|
||||
selected.runOnePage(new RunBackendStepInput().addValue("recordIds", "2").withRecords(List.of(links.get(1))), output);
|
||||
assertThat(ids.get(selected)).isEqualTo(java.util.Set.of(2L));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Docker dependencies must not enable debug wire logs during tests.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void dockerLoggersAreQuiet()
|
||||
{
|
||||
for(String name : List.of("org.testcontainers", "com.github.dockerjava", "org.apache.hc", "tc"))
|
||||
{
|
||||
assertThat(org.apache.logging.log4j.LogManager.getLogger(name).isDebugEnabled()).isFalse();
|
||||
assertThat(org.apache.logging.log4j.LogManager.getLogger(name).isWarnEnabled()).isTrue();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import java.sql.DriverManager;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.STORE;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.core;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.detail;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.lookups;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for rdbms chain test.
|
||||
*******************************************************************************/
|
||||
class RdbmsChainTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Verify real qqq rdbms actions anti join rollback and tamper.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void realQqqRdbmsActionsAntiJoinRollbackAndTamper() throws Exception
|
||||
{
|
||||
String url =
|
||||
"jdbc:h2:mem:worm_"
|
||||
+ UUID.randomUUID()
|
||||
+ ";DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE";
|
||||
var config = ChainVerificationTest.signed();
|
||||
var instance = RdbmsTestSupport.instance(url, "sa", "", config, false);
|
||||
try(var connection = DriverManager.getConnection(url, "sa", ""))
|
||||
{
|
||||
RdbmsTestSupport.createTables(connection, instance, false);
|
||||
QContext.init(instance, new QSession());
|
||||
lookups();
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
var writer = new WormAuditChainer(config);
|
||||
assertThat(writer.chainOnce().links()).isEqualTo(1);
|
||||
assertThat(writer.chainOnce().links()).isZero();
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).isEmpty();
|
||||
// Fail signing after append, verifying the real RDBMS transaction rolls back.
|
||||
var pins = config.getTrustedPublicKeys();
|
||||
config.withTrustedPublicKeys(Map.of());
|
||||
audit(2);
|
||||
assertThatThrownBy(writer::chainOnce)
|
||||
.hasMessageContaining("WORM audit operation failed");
|
||||
assertThat(STORE.all(WormAuditTables.CHAIN)).hasSize(1);
|
||||
config.withTrustedPublicKeys(pins);
|
||||
assertThat(writer.chainOnce().links()).isEqualTo(1);
|
||||
assertThat(STORE.all(WormAuditTables.CHAIN))
|
||||
.extracting(r -> r.getValueLong("id"))
|
||||
.containsExactly(1L, 2L);
|
||||
connection
|
||||
.createStatement()
|
||||
.execute("UPDATE audit_detail SET new_value='changed' WHERE id=1");
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks())
|
||||
.anyMatch(b -> b.reason().contains("audit_detail 1"));
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise multiple keyset pages, capped checkpoints and bounded SQL query counts.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void batchesPagesAndCheckpointRanges() throws Exception
|
||||
{
|
||||
String url = "jdbc:h2:mem:pages_" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE";
|
||||
var config = ChainVerificationTest.signed().withBatchSize(1500).withCheckpointEveryLinks(400).withCheckpointEveryMinutes(1);
|
||||
var instance = RdbmsTestSupport.instance(url, "sa", "", config, false);
|
||||
try(var connection = DriverManager.getConnection(url, "sa", ""); var statement = connection.createStatement())
|
||||
{
|
||||
RdbmsTestSupport.createTables(connection, instance, false);
|
||||
QContext.init(instance, new QSession());
|
||||
lookups();
|
||||
statement.execute("INSERT INTO audit(id,audit_table_id,audit_user_id,record_id,timestamp) SELECT x,1,1,17,CURRENT_TIMESTAMP - INTERVAL '2' MINUTE FROM SYSTEM_RANGE(1,1005) series(x)");
|
||||
statement.execute("INSERT INTO audit_detail(id,audit_id,field_name,new_value) SELECT x,x,'title','new' FROM SYSTEM_RANGE(1,1005) series(x)");
|
||||
statement.execute("SET QUERY_STATISTICS TRUE");
|
||||
var writer = new WormAuditChainer(config);
|
||||
var now = java.time.Instant.now();
|
||||
var counts = writer.chainOnce(now);
|
||||
assertThat(counts.links()).isEqualTo(1005);
|
||||
assertThat(counts.checkpoints()).isEqualTo(2);
|
||||
assertSelectCount(connection, 100);
|
||||
assertThat(STORE.last(WormAuditTables.CHECKPOINT).getValueInstant("created_at")).isCloseTo(now, org.assertj.core.api.Assertions.within(1, java.time.temporal.ChronoUnit.SECONDS));
|
||||
statement.execute("SET QUERY_STATISTICS FALSE");
|
||||
statement.execute("SET QUERY_STATISTICS TRUE");
|
||||
assertThat(writer.chainOnce(now.plusSeconds(61)).checkpoints()).isEqualTo(1);
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).isEmpty();
|
||||
assertSelectCount(connection, 100);
|
||||
assertThat(STORE.all(WormAuditTables.CHECKPOINT)).extracting(r -> r.getValueLong("link_count")).containsExactly(400L, 400L, 205L);
|
||||
statement.execute("UPDATE audit_detail SET new_value='changed' WHERE id=1001");
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).anyMatch(b -> b.reason().contains("audit_detail 1,001"));
|
||||
statement.execute("DELETE FROM worm_audit_chain WHERE id=1000");
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).anyMatch(b -> b.reason().contains("gap"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Assert actual query executions stay proportional to batches, not row count.
|
||||
*******************************************************************************/
|
||||
private void assertSelectCount(java.sql.Connection connection, int limit) throws Exception
|
||||
{
|
||||
try(var statement = connection.createStatement(); var rows = statement.executeQuery(
|
||||
"SELECT COALESCE(SUM(EXECUTION_COUNT),0) FROM INFORMATION_SCHEMA.QUERY_STATISTICS WHERE UPPER(SQL_STATEMENT) LIKE 'SELECT%'"))
|
||||
{
|
||||
rows.next();
|
||||
assertThat(rows.getLong(1)).isLessThan(limit);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Poll only the lookback window and report older omissions with exact bounded ids.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void lateWindowAndCompletenessOffenderLimit() throws Exception
|
||||
{
|
||||
String url = "jdbc:h2:mem:window_" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE";
|
||||
var config = new WormAuditQBitConfig().withBatchSize(500);
|
||||
var instance = RdbmsTestSupport.instance(url, "sa", "", config, false);
|
||||
try(var connection = DriverManager.getConnection(url, "sa", ""); var statement = connection.createStatement())
|
||||
{
|
||||
RdbmsTestSupport.createTables(connection, instance, false);
|
||||
QContext.init(instance, new QSession());
|
||||
lookups();
|
||||
audit(1);
|
||||
statement.execute("INSERT INTO audit(id,audit_table_id,audit_user_id,timestamp) VALUES(2,1,1,CURRENT_TIMESTAMP - INTERVAL '2' MINUTE)");
|
||||
var writer = new WormAuditChainer(config);
|
||||
writer.chainOnce();
|
||||
detail(1, 1);
|
||||
detail(2, 2);
|
||||
assertThat(writer.chainOnce().lateDetails()).isEqualTo(1);
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).anyMatch(b -> b.reason().contains("audit_detail 1 is not covered"));
|
||||
statement.execute("INSERT INTO worm_audit_chain SELECT 99,kind,audit_id,detail_ids,prev_hash,link_hash,chained_at FROM worm_audit_chain WHERE id=3");
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).anyMatch(b -> b.reason().contains("audit_detail 2 is covered twice"));
|
||||
statement.execute("DELETE FROM worm_audit_chain WHERE id=99");
|
||||
statement.execute("INSERT INTO audit_detail(id,audit_id) SELECT x,1 FROM SYSTEM_RANGE(3,107) series(x)");
|
||||
var result = new WormAuditVerifier(config).verify();
|
||||
assertThat(result.getBreakCount()).isEqualTo(106);
|
||||
assertThat(result.getBreaks()).hasSize(100);
|
||||
assertThat(result.getBreaks().getFirst().reason()).contains("audit_detail 1 is not covered");
|
||||
assertThat(result.getBreaks().getLast().reason()).contains("audit_detail 101 is not covered");
|
||||
assertThat(result.getWarnings()).anyMatch(w -> w.contains("106 coverage offenders"));
|
||||
statement.execute("INSERT INTO audit(id,timestamp) SELECT x,CURRENT_TIMESTAMP - INTERVAL '2' MINUTE FROM SYSTEM_RANGE(2000,2104) series(x)");
|
||||
result = new WormAuditVerifier(config).verify();
|
||||
assertThat(result.getBreakCount()).isEqualTo(211);
|
||||
assertThat(result.getBreaks()).hasSize(200);
|
||||
assertThat(result.getBreaks().getFirst().reason()).contains("audit 2,000 is not covered");
|
||||
assertThat(result.getBreaks().get(99).reason()).contains("audit 2,099 is not covered");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Cutoffs are UTC local values for timestamp columns, independent of JVM zone.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void withoutTimezoneCutoffInChicago() throws Exception
|
||||
{
|
||||
var original = java.util.TimeZone.getDefault();
|
||||
String url = "jdbc:h2:mem:timezone_" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE";
|
||||
var config = new WormAuditQBitConfig();
|
||||
var instance = RdbmsTestSupport.instance(url, "sa", "", config, false);
|
||||
try(var connection = DriverManager.getConnection(url, "sa", ""); var statement = connection.createStatement())
|
||||
{
|
||||
RdbmsTestSupport.createTables(connection, instance, false);
|
||||
statement.execute("ALTER TABLE audit ALTER COLUMN timestamp timestamp without time zone");
|
||||
QContext.init(instance, new QSession());
|
||||
lookups();
|
||||
statement.execute("INSERT INTO audit(id,audit_table_id,audit_user_id,timestamp) VALUES(1,1,1,TIMESTAMP '2026-10-13 12:00:00'),(2,1,1,TIMESTAMP '2026-10-13 12:01:30')");
|
||||
java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("America/Chicago"));
|
||||
assertThat(new WormAuditChainer(config).chainOnce(java.time.Instant.parse("2026-10-13T12:02:00Z")).audits()).isEqualTo(1);
|
||||
assertThat(STORE.all(WormAuditTables.CHAIN)).extracting(r -> r.getValueLong("audit_id")).containsExactly(1L);
|
||||
java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("UTC"));
|
||||
assertThat(new WormAuditVerifier(config).verify(java.util.Set.of(1L)).getBreaks()).isEmpty();
|
||||
}
|
||||
finally
|
||||
{
|
||||
java.util.TimeZone.setDefault(original);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.jdbc.SimpleConnectionProvider;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for rdbms test support.
|
||||
*******************************************************************************/
|
||||
class RdbmsTestSupport
|
||||
{
|
||||
// RC.5 supplies the RDBMS actions and a pluggable strategy. Hosts supply their
|
||||
// PostgreSQL quoting strategy; this fixture uses the core actions unchanged.
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for postgres strategy.
|
||||
*******************************************************************************/
|
||||
public static class PostgresStrategy extends BaseRDBMSActionStrategy
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Verify get identifier quote string.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getIdentifierQuoteString()
|
||||
{
|
||||
return ("\"");
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify get insert default values clause.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getInsertDefaultValuesClause()
|
||||
{
|
||||
return ("DEFAULT VALUES");
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify instance.
|
||||
*******************************************************************************/
|
||||
static QInstance instance(
|
||||
String url, String user, String password, WormAuditQBitConfig config, boolean postgres)
|
||||
throws Exception
|
||||
{
|
||||
com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager.resetConnectionProviders();
|
||||
var backend =
|
||||
new RDBMSBackendMetaData()
|
||||
.withName("db")
|
||||
.withVendor(postgres ? "postgresql" : "h2")
|
||||
.withJdbcUrl(url)
|
||||
.withJdbcDriverClassName(
|
||||
postgres ? "org.postgresql.Driver" : "org.h2.Driver")
|
||||
.withUsername(user)
|
||||
.withPassword(password)
|
||||
.withConnectionProvider(new QCodeReference(SimpleConnectionProvider.class));
|
||||
if(postgres)
|
||||
{
|
||||
backend.withActionStrategyCodeReference(new QCodeReference(PostgresStrategy.class));
|
||||
}
|
||||
QInstance instance = WormAuditTestSupport.core(backend);
|
||||
for(var table : instance.getTables().values())
|
||||
{
|
||||
table.withBackendDetails(
|
||||
new RDBMSTableBackendDetails().withTableName(snake(table.getName())));
|
||||
table.getFields().values().forEach(f -> f.withBackendName(snake(f.getName())));
|
||||
}
|
||||
new WormAuditQBitProducer().withQBitConfig(config.withBackendName("db")).produce(instance);
|
||||
return (instance);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify snake.
|
||||
*******************************************************************************/
|
||||
static String snake(String value)
|
||||
{
|
||||
return (value.replaceAll("([a-z])([A-Z])", "$1_$2").toLowerCase(Locale.ROOT));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify create tables.
|
||||
*******************************************************************************/
|
||||
static void createTables(Connection connection, QInstance instance, boolean coreOnly)
|
||||
throws Exception
|
||||
{
|
||||
for(var table : instance.getTables().values())
|
||||
{
|
||||
if(coreOnly && table.getName().startsWith("worm_audit_"))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
List<String> columns = new ArrayList<>();
|
||||
for(var field : table.getFields().values())
|
||||
{
|
||||
String type =
|
||||
switch(field.getType())
|
||||
{
|
||||
case INTEGER -> "integer";
|
||||
case LONG -> "bigint";
|
||||
case DATE_TIME -> "timestamp(6) with time zone";
|
||||
default -> "text";
|
||||
};
|
||||
if(field.getName().equals("id"))
|
||||
{
|
||||
type +=
|
||||
table.getName().equals("worm_audit_chain")
|
||||
? " PRIMARY KEY"
|
||||
: " GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY";
|
||||
}
|
||||
columns.add(
|
||||
WormAuditDdl.identifier(
|
||||
field.getBackendName() == null
|
||||
? field.getName()
|
||||
: field.getBackendName())
|
||||
+ " "
|
||||
+ type);
|
||||
}
|
||||
String name = ((RDBMSTableBackendDetails) table.getBackendDetails()).getTableName();
|
||||
try(var statement = connection.createStatement())
|
||||
{
|
||||
statement.execute(
|
||||
"CREATE TABLE "
|
||||
+ WormAuditDdl.identifier(name)
|
||||
+ " ("
|
||||
+ String.join(",", columns)
|
||||
+ ")");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||