Compare commits

...
9 Commits
Author SHA1 Message Date
james.maes 19ac187886 Merge develop into feature/chain 2026-10-13 20:28:09 -05:00
james.maesandClaude Opus 5.5 f380b9b2a1 feat: append-only audit tables, hash chain, signed checkpoints and verification
Phase (a) of docs/DESIGN.md:
- Postgres DDL template and renderer: write-once triggers on the core
  audit tables and worm_audit_* tables, lookup id/name protection, the
  SECURITY DEFINER retention purge (call-stack guarded, chain prefix only,
  recorded) and the INSERT/SELECT-only grant model;
- capability guard and a clear write-once error;
- RFC 8785 canonical audit content, a single-writer chainer (advisory
  lock, settle horizon, contiguous ids, late arrivals and late details),
  Ed25519 checkpoints with Merkle roots;
- a paged verifier naming the exact break, the Verify audit chain table
  action, and the daily verification with QAlerts break/heartbeat;
- the assistant deny pattern for worm_audit_* (reflective until RC.6);
- tests: Postgres Testcontainers (triggers, grants, purge, tamper,
  concurrency, time zone), PHI canary, chain and checkpoint units.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 20:28:00 -05:00
james.maes 906df343b0 Merge pull request #5 from QRun-IO/feature/design-spec
docs: WORM audit design spec
2026-10-13 19:27:12 -05:00
james.maesandClaude Opus 5.5 d25df1de0a docs: the export policy also needs s3:PutObjectRetention
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 19:23:33 -05:00
james.maesandClaude Opus 5.5 bbac29e4d0 docs: WORM audit design spec
Append-only audit tables enforced by Postgres triggers, a GRANT model and
an audited retention purge; a single-writer hash chain with signed Merkle
checkpoints, verification and daily alerting; sealed JSONL exports to S3
Object Lock (COMPLIANCE, 6 years); canonical content agreed with
qbit-secured-data so only audit tokens are sealed.

Owner decisions (2026-10-12): all recommendations accepted.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 19:20:29 -05:00
james.maes a6f4eb79f6 Merge pull request #2 from QRun-IO/feature/scaffold
chore: repo-standard scaffold
2026-10-13 19:17:03 -05:00
james.maesandClaude Opus 5.5 8ab4fb686d chore: repo-standard scaffold for qbit-worm-audit
Pom on qbit-build-parent 2.0.0 with checkstyle and jacoco gates, Maven
wrapper, CircleCI (orb 0.6.8, configure-package-install pre-step,
GA-only publish_release, no hotfix publishing), release-rc caller
workflow, dependabot, governance files, and the
com.kingsrook.qbits.wormaudit package. Removes the template example.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 19:14:17 -05:00
james.maes 1212aedca6 Merge main into develop (bring develop up to main before the scaffold) 2026-10-13 19:12:10 -05:00
james.maes 87c171387a chore: bump qqq to 0.40.0-SNAPSHOT 2026-01-10 21:05:39 -06:00
60 changed files with 8129 additions and 354 deletions

No files matched your search

+60 -4
View File
@@ -1,9 +1,65 @@
version: 2.1
orbs:
qqq-orb: kingsrook/qqq-orb@0.5.0
qqq-orb: kingsrook/qqq-orb@0.6.8
commands:
configure-package-install:
steps:
- run:
name: Keep package installation unattended
command: |
# These build machines are discarded after the job.
sudo install -d /etc/needrestart/conf.d
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
workflows:
build:
jobs:
- qqq-orb/mvn_test
test_only:
jobs:
- qqq-orb/mvn_test_only:
pre-steps:
- configure-package-install
context: [ qqq-maven-registry-credentials ]
filters:
branches:
ignore: /(develop|release/.*|integration.*)/
tags:
only: []
publish_snapshot:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [ qqq-maven-registry-credentials ]
branch_type: snapshot
filters:
branches:
only:
- develop
publish_release_candidate:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [ qqq-maven-registry-credentials ]
branch_type: release_candidate
filters:
branches:
only:
- /release\/.*/
publish_release:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [ qqq-maven-registry-credentials ]
branch_type: release
filters:
branches:
ignore: /.*/
tags:
only: [ /^v\d+\.\d+\.\d+$/ ]
+20
View File
@@ -0,0 +1,20 @@
# QRun-IO qbit .editorconfig: the Kingsrook layout (3-space indentation, never tabs).
root = true
[*]
charset = utf-8
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
indent_style = space
indent_size = 3
tab_width = 3
[*.md]
trim_trailing_whitespace = false
[*.{yml,yaml}]
indent_size = 2
[*.{cmd,bat}]
end_of_line = crlf
+3
View File
@@ -0,0 +1,3 @@
* text=auto eol=lf
*.cmd text eol=crlf
mvnw text eol=lf
+25
View File
@@ -0,0 +1,25 @@
# Weekly, grouped dependency updates. Minor and patch updates arrive as one PR per ecosystem;
# major updates stay separate so each gets its own review. At most three version-update PRs per
# ecosystem are open at once. Security updates are enabled in the repository settings and are
# not limited by this schedule.
version: 2
updates:
- package-ecosystem: maven
directory: /
target-branch: develop
schedule:
interval: weekly
open-pull-requests-limit: 3
groups:
maven-minor-and-patch:
update-types: [minor, patch]
- package-ecosystem: github-actions
directory: /
target-branch: develop
schedule:
interval: weekly
open-pull-requests-limit: 3
groups:
github-actions:
update-types: [minor, patch]
+31
View File
@@ -0,0 +1,31 @@
# GitHub Release Configuration
# Auto-generates release notes from PR labels
changelog:
exclude:
labels:
- ignore-for-release
- duplicate
- invalid
authors:
- dependabot
- dependabot[bot]
categories:
- title: Breaking Changes
labels:
- breaking-change
- breaking
- title: New Features
labels:
- enhancement
- feature
- title: Bug Fixes
labels:
- bug
- fix
- title: Dependencies
labels:
- dependencies
- title: Other Changes
labels:
- "*"
+25
View File
@@ -0,0 +1,25 @@
# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
# the release line and the release/* push runs CircleCI publish_release_candidate.
name: Release candidate
on:
workflow_dispatch:
inputs:
dry_run:
description: Print the plan without merging or creating branches
type: boolean
default: false
pull_request:
types: [closed]
branches: [develop]
permissions:
contents: write
jobs:
release-rc:
if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
with:
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
+16 -15
View File
@@ -1,20 +1,21 @@
# Maven
target/
.env
.idea
*.iml
*.class
*.log
*.jar
*.war
*.nar
*.ear
# IDE
.idea/
*.iml
.vscode/
.project
.classpath
.settings/
# OS
*.zip
*.tar.gz
*.rar
hs_err_pid*
.DS_Store
Thumbs.db
# Logs
*.log
*.sw*
.flattened-pom.xml
dependency-reduced-pom.xml
/.env.local
/.cache/
.claude/settings.local.json
+3
View File
@@ -0,0 +1,3 @@
wrapperVersion=3.3.4
distributionType=only-script
distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.11/apache-maven-3.9.11-bin.zip
+51
View File
@@ -0,0 +1,51 @@
# AGENTS.md
Guidance for coding agents working in this repository.
## Overview
QBit WORM Audit makes QQQ audit records write-once: database-enforced append-only audit tables (Postgres triggers plus an INSERT/SELECT-only app role), a hash chain over every audit row with signed checkpoints, a verification action and daily job, and sealed JSONL exports to an S3 Object Lock (COMPLIANCE) bucket through qbit-object-storage. Design: `docs/DESIGN.md`.
## Build commands
```bash
./mvnw compile
./mvnw test
./mvnw verify # Checkstyle; JaCoCo: 70% instructions, 90% classes
```
Use the Maven 3.9.11 wrapper (Java 21); Maven 3.10 cannot read the parent POM. On Windows use `./mvnw.cmd`.
## Architecture
Package: `com.kingsrook.qbits.wormaudit`. Entry point: `WormAuditQBitProducer implements QBitMetaDataProducer<WormAuditQBitConfig>`. Config requires the host `backendName`; core audit tables must exist before production.
- `ddl`: reviewed Liquibase PostgreSQL template and safe renderer; public sidecar tables and owner-only retention purge.
- `guard`: immutable QQQ capabilities and value-free SQLSTATE mapping.
- `canonical`, `chain`, `checkpoint`: strict JCS v1, transaction/advisory-locked writer, explicit detail coverage, pinned Ed25519 ranges.
- `verify`: snapshot checks of content, continuity, coverage and checkpoints, starting at the purge anchor.
- `processes`: standard streamed ETL action, scheduled chainer and daily QAlerts producer; never app navigation entries.
- `model`: read-only tables, range record references and standard record child lists. No app-home widgets or export execution in phase (a).
PostgreSQL uses a transaction advisory lock; other backends require a single writer JVM.
Use `./mvnw.cmd -o verify` offline. PostgreSQL tests skip without Docker locally, but fail whenever `CI` is set.
Core RC.5 has no assistant deny API (reflective registration) and disallows underscores in declared alert-type metadata;
signals retain the required `worm_audit.chain.break` name and dispatch through QAlerts without that declaration.
## Invariants and naming
- Audit rows are never updated or deleted by the application; the only removal path is the audited retention purge through a separate database role, after the retention date.
- The chain and the sealed exports hash exactly what core wrote. Never add raw PHI to chain rows, checkpoints, exports, logs or alerts.
- Tables use lowercase snake_case with the `worm_audit_` prefix; backend names equal table names. Permissions use the full `worm_audit` prefix. Processes are table or record actions, never navigation entries.
## Code style
3-space indent, next-line braces, `return (x);`, flower-box Javadoc on classes and methods, fluent `withX()` setters. Follow `checkstyle/` and `CODE_STYLE.md`.
## Dependencies
Parent: `com.kingsrook:qbit-build-parent:2.0.0`. QQQ BOM: `4.1.0-RC.5` (default-active `qqq-snapshot` profile).
## Release flow
CircleCI uses `kingsrook/qqq-orb@0.6.8`: develop publishes SNAPSHOTs, release/* publishes RCs, and only `vX.Y.Z` GA tags publish releases. Hotfix branches run tests only; never push `hotfix/*` or `release/*` by hand. RC automation uses workflow_dispatch or a merged develop PR labelled `release:rc`. Feature branches start from develop and return by PR. Never publish manually.
+14
View File
@@ -0,0 +1,14 @@
# Changelog
All notable changes are documented here, following [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased]
### Added
- Phase (a): PostgreSQL append-only tables/triggers, safe DDL renderer, retention purge with immutable refusal/anchor records, and runtime grants.
- Canonical audit v1 documents, contiguous hash chain with late-detail coverage, pinned Ed25519 checkpoints and binary Merkle roots.
- Snapshot verification, streamed ETL Review/Result action, scheduled chaining/daily verification, and core QAlerts raise/resolve/heartbeat integration.
- Read-only metadata, WORM Audit table navigation, record range references and child lists; reflective RC.6 assistant deny registration.
- Offline memory/H2 acceptance, real pre-insert PHI canary hook, and PostgreSQL Testcontainers acceptance with CI-strict Docker availability.
- Adoption and operations documentation. Export execution and app-home widgets remain later phases.
+47 -11
View File
@@ -1,15 +1,51 @@
# qbit-worm-audit
# CLAUDE.md
## Knowledge base
Guidance for coding agents working in this repository.
Reviewed knowledge for this repo and the wider QQQ platform lives in the second-brain vault:
## Overview
- Platform hub: `R:/Git.Local/KofTwentyTwo/second-brain/knowledge/qqq/qqq-hub.md`
- This repo's dossier: `R:/Git.Local/KofTwentyTwo/second-brain/knowledge/qqq/repos/qbit-worm-audit.md`
(reviewed at commit `294cd30f2d91`, branch `main`, 2026-07-04)
QBit WORM Audit makes QQQ audit records write-once: database-enforced append-only audit tables (Postgres triggers plus an INSERT/SELECT-only app role), a hash chain over every audit row with signed checkpoints, a verification action and daily job, and sealed JSONL exports to an S3 Object Lock (COMPLIANCE) bucket through qbit-object-storage. Design: `docs/DESIGN.md`.
Key facts from the review: this repo is an **un-customized instantiation** of the
`qbit-template-extension` GitHub template — no WORM-audit code exists yet; Maven coordinates are
still `com.kingsrook.qbits:qbit-example-extension`; qqq pin is pre-4.0 (`qqq-backend-core 0.35.0`
on main, `0.40.0-SNAPSHOT` on develop); README's "AGPL-3.0" line contradicts the Apache-2.0
LICENSE. See the dossier's "v4.0 impact" section before starting implementation work here.
## Build commands
```bash
./mvnw compile
./mvnw test
./mvnw verify # Checkstyle; JaCoCo: 70% instructions, 90% classes
```
Use the Maven 3.9.11 wrapper (Java 21); Maven 3.10 cannot read the parent POM. On Windows use `./mvnw.cmd`.
## Architecture
Package: `com.kingsrook.qbits.wormaudit`. Entry point: `WormAuditQBitProducer implements QBitMetaDataProducer<WormAuditQBitConfig>`. Config requires the host `backendName`; core audit tables must exist before production.
- `ddl`: reviewed Liquibase PostgreSQL template and safe renderer; public sidecar tables and owner-only retention purge.
- `guard`: immutable QQQ capabilities and value-free SQLSTATE mapping.
- `canonical`, `chain`, `checkpoint`: strict JCS v1, transaction/advisory-locked writer, explicit detail coverage, pinned Ed25519 ranges.
- `verify`: snapshot checks of content, continuity, coverage and checkpoints, starting at the purge anchor.
- `processes`: standard streamed ETL action, scheduled chainer and daily QAlerts producer; never app navigation entries.
- `model`: read-only tables, range record references and standard record child lists. No app-home widgets or export execution in phase (a).
PostgreSQL uses a transaction advisory lock; other backends require a single writer JVM.
Use `./mvnw.cmd -o verify` offline. PostgreSQL tests skip without Docker locally, but fail whenever `CI` is set.
Core RC.5 has no assistant deny API (reflective registration) and disallows underscores in declared alert-type metadata;
signals retain the required `worm_audit.chain.break` name and dispatch through QAlerts without that declaration.
## Invariants and naming
- Audit rows are never updated or deleted by the application; the only removal path is the audited retention purge through a separate database role, after the retention date.
- The chain and the sealed exports hash exactly what core wrote. Never add raw PHI to chain rows, checkpoints, exports, logs or alerts.
- Tables use lowercase snake_case with the `worm_audit_` prefix; backend names equal table names. Permissions use the full `worm_audit` prefix. Processes are table or record actions, never navigation entries.
## Code style
3-space indent, next-line braces, `return (x);`, flower-box Javadoc on classes and methods, fluent `withX()` setters. Follow `checkstyle/` and `CODE_STYLE.md`.
## Dependencies
Parent: `com.kingsrook:qbit-build-parent:2.0.0`. QQQ BOM: `4.1.0-RC.5` (default-active `qqq-snapshot` profile).
## Release flow
CircleCI uses `kingsrook/qqq-orb@0.6.8`: develop publishes SNAPSHOTs, release/* publishes RCs, and only `vX.Y.Z` GA tags publish releases. Hotfix branches run tests only; never push `hotfix/*` or `release/*` by hand. RC automation uses workflow_dispatch or a merged develop PR labelled `release:rc`. Feature branches start from develop and return by PR. Never publish manually.
+3
View File
@@ -0,0 +1,3 @@
# Code of Conduct
See Project level [Code of Conduct](https://github.com/QRun-IO/qqq/blob/develop/CODE_OF_CONDUCT.md)
+3
View File
@@ -0,0 +1,3 @@
# Code Style Guide
* See Project [Code Style Guide](https://github.com/QRun-IO/qqq/blob/develop/CODE_STYLE.md)
+6
View File
@@ -0,0 +1,6 @@
# Contributing to QBit Secured Data
* See Project [Contributing](https://github.com/QRun-IO/qqq/blob/develop/CONTRIBUTING.md)
* Branches follow GitFlow: `feature/*` from `develop`, pull requests to `develop`, `release/*` for release candidates, `vX.Y.Z` tags for releases.
* Commits are signed and follow Conventional Commits.
* Build and test locally with `./mvnw verify` (Java 21).
+1 -1
View File
@@ -176,7 +176,7 @@
END OF TERMS AND CONDITIONS
Copyright 2022-2025 QRunIO, LLC
Copyright 2024 Kingsrook, LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
+3 -3
View File
@@ -1,6 +1,6 @@
qBit Template - Extension
Copyright 2022-2025 QRunIO, LLC
QBit WORM Audit
Copyright 2026 Kingsrook, LLC
This product includes software developed at QRunIO, LLC (https://qrun.io/).
This product includes software developed at Kingsrook, LLC (https://kingsrook.com/).
Licensed under the Apache License, Version 2.0.
+185 -33
View File
@@ -1,48 +1,200 @@
# QBit Template: Extension
# QBit WORM Audit
Template repository for creating QQQ Extension QBits - infrastructure extensions, customizers, and action handlers.
Phase (a) provides PostgreSQL append-only enforcement, an asynchronous audit hash chain,
Ed25519 checkpoints, a verification action, and daily verification with core QAlerts signals.
It hashes exactly the audit values core stored; it does not read application records or
secured-data keys. Sealed exports and the app home are later phases. The export table and
range references are present as integration hooks; no export job or home widgets are installed.
## What is an Extension QBit?
Requires Java 21, Maven wrapper 3.9.11 and QQQ 4.1.0-RC.5. PostgreSQL provides enforcement;
memory/H2 support tamper-evidence tests without database enforcement.
Extension QBits extend or enhance how QQQ operates at the infrastructure/framework level. They provide:
- New backend implementations
- Authentication mechanisms
- Audit capabilities
- Table customizers
- Action handlers
## Tables and permissions
## Quick Start
All sidecar tables use the configured host backend, snake_case fields, BIGINT identifiers,
record sections, and full table-name permissions. They do not audit themselves. The app lists
Verifications, Sealed Exports, Checkpoints, Audit Chain, and Purges in that order.
1. Click "Use this template" to create your QBit
2. Run `scripts/customize_template.py` to rename the example
3. Implement your extension logic
4. Add tests
5. Publish to Maven repository
| Table | Purpose | Permission |
|---|---|---|
| `worm_audit_verification` | Immutable USER/SCHEDULE results | `worm_audit_verification.read` |
| `worm_audit_export` | Reserved sealed-export catalog | `worm_audit_export.read` |
| `worm_audit_checkpoint` | Signed ranges and Merkle roots | `worm_audit_checkpoint.read` |
| `worm_audit_chain` | Contiguous links and explicit detail coverage | `worm_audit_chain.read` |
| `worm_audit_purge` | Refusals, removal counts and retention anchors | `worm_audit_purge.read` |
## Structure
`worm_audit.verify` authorizes the standard streamed ETL **Verify audit chain** action.
Processes are attached to the chain table or scheduled; none is a navigation entry.
Insert/update/delete capabilities are absent on sidecar tables; update/delete capabilities
are removed from all four core audit tables. Internal appends use an explicit permission bypass.
Checkpoint, export and verification record screens have a bounded chain-link child list;
range endpoints and export/checkpoint references link to their records.
```
src/main/java/com/kingsrook/qbits/example/
├── ExampleExtensionQBitConfig.java # Configuration
├── ExampleExtensionQBitProducer.java # Entry point
├── customizers/
│ └── ExampleTableCustomizer.java # Table customization
└── actions/
└── ExampleActionCustomizer.java # Action handling
The table metadata customizer can configure host ESB publications. Quick Search registration
is host-owned through `QuickSearchQBitConfig.searchableTables`; add all five tables.
Verification inserts use QQQ's record-change listener SPI (as will phase-(b) export inserts),
so an installed ESB publisher receives those events when the host configures its destinations
on these two tables. The qbit defines no destination and configures no chain-link publication.
## Configuration
| Field | Default / contract |
|---|---|
| `backendName` | Required; core and sidecar audit tables must use this backend |
| `retentionYears` | 6; compiled into the migration, not changed by a runtime flag |
| `signingKey` | Optional `Supplier<SigningKey>`; `SigningKey(keyId, PrivateKey)` |
| `trustedPublicKeys` | `Map<String, PublicKey>`; must contain the signing key's public key |
| `chainIntervalSeconds` | 30 |
| `settleHorizonSeconds` | 60 (zero allowed) |
| `batchSize` | 1,000 headers per tick / links per insert batch |
| `lateDetailLookbackHours` | 24; positive hours, older omissions checked by full verification |
| `checkpointEveryLinks` | 10,000; maximum links in each checkpoint |
| `checkpointEveryMinutes` | 60; requires at least one uncheckpointed link |
| `verifyCron` | `0 0 3 * * ?` (03:00 UTC, Quartz syntax) |
| `schedulerName` | `worm_audit`; uses the host scheduler if present, otherwise registers Quartz |
| `coreAuditTableNames` | Overrides keyed by `audit`, `auditDetail`, `auditTable`, `auditUser` |
| `appRole` | `app` |
| `tableMetaDataCustomizer` | Optional; append-only capability guard runs afterward |
A missing signing key produces no checkpoints and adds the warning **no signing key configured**
to verification. It does not itself break the chain. Signing requires a matching public-key pin.
Rotate by adding a new pin and changing the supplier; retain old pins for old checkpoints.
Never store the private key in a database row or checked-in configuration.
## Adopting in another app
Add `com.kingsrook.qbits:qbit-worm-audit:0.1.0-SNAPSHOT`. Define the host's core audit tables
with `AuditsMetaDataProvider` first, then install the qbit:
```java
WormAuditQBitConfig config = new WormAuditQBitConfig()
.withBackendName("main")
.withAppRole("app")
.withSigningKey(() -> new WormAuditQBitConfig.SigningKey("2026-10", privateKey))
.withTrustedPublicKeys(Map.of("2026-10", publicKey));
new WormAuditQBitProducer().withQBitConfig(config).produce(qInstance);
String migration = WormAuditDdl.postgres(config);
```
## Key Characteristics
1. Render and review the migration, then copy it into the host Liquibase changelog.
The [default rendering](docs/ddl/worm-audit-postgres.sql) uses `audit`, `audit_detail`,
`audit_table`, `audit_user`, runtime role `app`, and six years. The renderer resolves table
backend names during production; explicit overrides win. Standalone rendering uses those
defaults. Identifiers are validated and quoted; schema-qualified core table names are supported.
2. Apply the DDL as the schema owner. It creates the five sidecar tables in `public`, functions,
triggers, the NOLOGIN `worm_audit_purger` role, and runtime grants. Core column names must use
the usual snake_case mappings (`audit_id`, `audit_table_id`, etc.). The migration is idempotent. Provision the purger role beforehand or give the migration role permission to create it.
3. Run as a separate runtime role with SELECT/INSERT and sequence USAGE. Do not make it a member
of the purger role or a schema/table owner. Use the host's PostgreSQL RDBMS strategy (RC.5's
base strategy uses MySQL quoting; see the test host strategy in `RdbmsTestSupport`).
4. Start QQQ's standard scheduler with its system session and grant users only the required read
and verification permissions. Point the existing alerts implementation at core `QAlerts`.
5. Configure ESB publications on verification/export in the table customizer. Register all five
tables in the host Quick Search config with `SearchableTableConfig` / `SearchableFieldConfig`.
Index identifiers and status fields; set basepull timestamps to `chained_at`, `created_at`,
`sealed_at`, `finished_at`, and `requested_at` respectively (these tables have no `modifyDate`).
- **No QAppSection** - Pure infrastructure
- **No user-facing tables** (or minimal operational)
- **Wraps existing tables/processes** with new behavior
- **Validates dependencies** on other QQQ components
The RC.6 assistant deny API is registered reflectively when present; absence in RC.5 is a no-op.
RC.5's alert-type metadata validator disallows underscores, so the qbit sends the specified
`worm_audit.chain.break` signal without declaring incompatible alert-type metadata. `QAlerts`
still dispatches it to the sink (or its logging fallback). Every scheduled verification sends
`worm_audit.verify.daily`; FAIL raises with the first break and verification record reference;
PASS resolves dedup key `chain`.
## Documentation
## Chain and verification format
- [Getting Started](docs/00-getting-started.md)
- [Extension Patterns](docs/01-extension-patterns.md)
- JCS v1 uses UTF-16 key ordering, strict surrogate validation and UTC microsecond timestamps.
IDs are integer longs except canonical `recordId`, which is a string. No floating-point values.
- Link hashes concatenate the predecessor's **64 lowercase ASCII hex bytes**, one NUL byte,
and canonical UTF-8 bytes. Genesis is 64 ASCII zeroes.
- Every link names the detail IDs it covered, at most 250. A PostgreSQL insert guard and GIN
overlap index reject duplicate coverage under the same advisory lock. Overflow and later details use
`LATE_DETAIL` links with the same canonical header-and-detail-subset document. The canonical
JSON is never persisted. A v1 content mismatch can identify the link, audit and covered detail
IDs, but cannot identify one changed field among multiple details without original content.
- PostgreSQL chaining uses transaction-scoped advisory lock `86753090421` shared with purge.
Other backends use a JVM-wide lock and require one writer JVM. SQL backends select settled
unchained headers with an anti-join; late commits are picked up on a later tick.
- Merkle leaves are decoded 32-byte link hashes; parents hash the concatenated binary children;
an odd node is promoted unchanged. Ed25519 signs JCS of the seven specified range/trust fields.
- SQL verification uses a repeatable-read snapshot. Full runs check settled header/detail
completeness; selected or filtered runs check their links, predecessor boundaries and
intersecting checkpoints. Full verification currently materializes link metadata in memory.
## Retention operations
Only the ops login in `worm_audit_purger` may execute:
```sql
SELECT public.worm_audit_purge(now() - interval '6 years');
```
A recent/null cutoff or an incomplete expired prefix records REFUSED and returns normally.
Purge locks the audit tables against concurrent inserts, removes only a prefix closed over
all links and details of its audits, and records its last link id/hash. Detail and chain deletion
is allowed only under the function-owner identity, transaction-local purge flag, and compiled
retention cutoff. An app session setting that flag gets no deletion authority.
Lookups cannot change id/name or be deleted/truncated. The runtime role may update only the
lookup label/dates through column grants; table-level UPDATE stays revoked. With normal grants,
PostgreSQL rejects prohibited runtime writes as SQLSTATE 42501 before executing a trigger.
The second layer (triggers) returns `P0001` / `worm_audit:` even if someone grants wider DML
rights. The qbit maps those errors to a value-free user-facing message.
Verification starts from the latest successful purge anchor. Checkpoint signatures still verify
after purge, but Merkle recomputation of a purged/partly purged checkpoint requires its sealed
export and is reported as a warning. Sealed exports are phase (b); preserve external evidence
before exercising retention removal in production. Superusers/owners can disable triggers;
protect signing keys and keep public-key pins outside the database.
## Build and acceptance
```powershell
./mvnw.cmd -o verify
```
Checkstyle and JaCoCo gates (70% instructions, 90% classes) run without Docker. Memory and H2
tests cover canonical vectors, signatures, rollback, exact coverage, late arrivals/details,
verification findings, PHI tokenization, process summaries, permissions, and the alerts SPI.
The PHI stand-in uses RC.5 `AuditActionCustomizerInterface.customizeInput` before insertion;
classification is test-local because RC.5 lacks `DataClassification`.
PostgreSQL acceptance uses `postgres:16-alpine`, applies the DDL twice and tests grants,
triggers, purge, real QQQ RDBMS chaining, superuser tamper, and 8 writers / 3 chainers.
It skips automatically without Docker locally and fails if **any** `CI` environment value is set.
On a Docker-enabled coordinator (image already available if running without network):
```powershell
$env:CI = 'true'
./mvnw.cmd -o '-Dtest=PostgresAcceptanceTest' test
./mvnw.cmd -o verify
```
POSIX equivalents: `CI=true ./mvnw -o -Dtest=PostgresAcceptanceTest test` and
`CI=true ./mvnw -o verify`. No commits, publishing or deployment are part of these commands.
## License
AGPL-3.0 - See [LICENSE](LICENSE)
[Apache License 2.0](LICENSE). Copyright Kingsrook, LLC.
### Physical columns and bounded polling
`WormAuditDdl.postgres(config)` renders table names only. Core physical columns
must be snake_case: `audit.id`, `audit.timestamp`, `audit_detail.id`,
`audit_detail.audit_id`, `audit_table.id/name/label/create_date/modify_date`, and
`audit_user.id/name/create_date/modify_date`. QInstance field backend-name
customizations do not change this migration; adapt the reviewed SQL before
applying it if your schema differs. UTC values in `timestamp without time zone`
and `timestamptz` are both supported.
`lateDetailLookbackHours` defaults to **24** (positive hours). Each PostgreSQL
poll checks uncovered details whose chained header timestamp lies in
`[cutoff - lookback, cutoff)`. The daily full verifier reports omissions outside
that window. Initial header details are always chained, including overflow
chunks. Checkpoints contain at most `checkpointEveryLinks` links; multiple full
ranges can be signed per tick. Verification uses 1,000-link keyset pages.
Full verification retains at most 1,000 exact break diagnostics and 100 warnings,
while persisting the total break count. SQL completeness counts all offenders
and reports the first 100 header ids and first 100 detail ids. This keeps a badly
corrupted chain from exhausting verifier memory.
+4
View File
@@ -0,0 +1,4 @@
# Security Policy
* See Project [Security Policy](https://github.com/QRun-IO/qqq/blob/develop/SECURITY.md)
* Report suspected vulnerabilities privately through GitHub private vulnerability reporting on this repository, never in a public issue.
+246
View File
@@ -0,0 +1,246 @@
<?xml version="1.0"?>
<!--
~ QQQ - Low-code Application Framework for Engineers.
~ Copyright (C) 2021-2022. Kingsrook, LLC
~ 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
~ contact@kingsrook.com
~ https://github.com/Kingsrook/
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ https://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!DOCTYPE module PUBLIC
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
"http://checkstyle.sourceforge.net/dtds/configuration_1_3.dtd">
<!--
See reference at https://checkstyle.sourceforge.io/checks.html
-->
<module name = "Checker">
<property name="charset" value="UTF-8"/>
<property name="severity" value="warning"/>
<property name="fileExtensions" value="java, properties, xml"/>
<!-- Checks for whitespace -->
<!-- See http://checkstyle.sf.net/config_whitespace.html -->
<module name="FileTabCharacter">
<property name="eachLine" value="true"/>
</module>
<!--
<module name="LineLength">
<property name="max" value="100"/>
<property name="ignorePattern" value="^package.*|^import.*|a href|href|http://|https://|ftp://"/>
</module>
-->
<module name="TreeWalker">
<module name="SuppressWarningsHolder"/>
<module name="OuterTypeFilename"/>
<module name="IllegalTokenText">
<property name="tokens" value="STRING_LITERAL, CHAR_LITERAL"/>
<property name="format" value="\\u00(09|0(a|A)|0(c|C)|0(d|D)|22|27|5(C|c))|\\(0(10|11|12|14|15|42|47)|134)"/>
<property name="message" value="Consider using special escape sequence instead of octal value or Unicode escaped value."/>
</module>
<module name="AvoidEscapedUnicodeCharacters">
<property name="allowEscapesForControlCharacters" value="true"/>
<property name="allowByTailComment" value="true"/>
<property name="allowNonPrintableEscapes" value="true"/>
</module>
<module name="AvoidStarImport"/>
<module name="OneTopLevelClass"/>
<module name="NoLineWrap"/>
<module name="EmptyBlock">
<property name="option" value="TEXT"/>
<property name="tokens" value="LITERAL_TRY, LITERAL_FINALLY, LITERAL_IF, LITERAL_ELSE, LITERAL_SWITCH"/>
</module>
<module name="NeedBraces"/>
<module name="LeftCurly">
<property name="option" value="nl"/>
</module>
<module name="RightCurly">
<property name="option" value="alone"/>
</module>
<!--
<module name="WhitespaceAround">
<property name="allowEmptyConstructors" value="true"/>
<property name="allowEmptyMethods" value="true"/>
<property name="allowEmptyTypes" value="true"/>
<property name="allowEmptyLoops" value="true"/>
<message key="ws.notFollowed"
value="WhitespaceAround: ''{0}'' is not followed by whitespace. Empty blocks may only be represented as '{}' when not part of a multi-block statement (4.1.3)"/>
<message key="ws.notPreceded"
value="WhitespaceAround: ''{0}'' is not preceded with whitespace."/>
</module>
-->
<module name="OneStatementPerLine"/>
<module name="MultipleVariableDeclarations"/>
<module name="ArrayTypeStyle"/>
<module name="MissingSwitchDefault"/>
<module name="FallThrough"/>
<module name="UpperEll"/>
<module name="ModifierOrder"/>
<module name="EmptyLineSeparator">
<property name="allowNoEmptyLineBetweenFields" value="true"/>
</module>
<module name="SeparatorWrap">
<property name="id" value="SeparatorWrapDot"/>
<property name="tokens" value="DOT"/>
<property name="option" value="nl"/>
</module>
<module name="SeparatorWrap">
<property name="id" value="SeparatorWrapComma"/>
<property name="tokens" value="COMMA"/>
<property name="option" value="EOL"/>
</module>
<module name="SeparatorWrap">
<!-- ELLIPSIS is EOL until https://github.com/google/styleguide/issues/258 -->
<property name="id" value="SeparatorWrapEllipsis"/>
<property name="tokens" value="ELLIPSIS"/>
<property name="option" value="EOL"/>
</module>
<module name="SeparatorWrap">
<!-- ARRAY_DECLARATOR is EOL until https://github.com/google/styleguide/issues/259 -->
<property name="id" value="SeparatorWrapArrayDeclarator"/>
<property name="tokens" value="ARRAY_DECLARATOR"/>
<property name="option" value="EOL"/>
</module>
<module name="SeparatorWrap">
<property name="id" value="SeparatorWrapMethodRef"/>
<property name="tokens" value="METHOD_REF"/>
<property name="option" value="nl"/>
</module>
<module name="PackageName">
<property name="format" value="^[a-z]+(\.[a-z][a-z0-9]*)*$"/>
<message key="name.invalidPattern"
value="Package name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="TypeName">
<message key="name.invalidPattern"
value="Type name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="MemberName">
<property name="format" value="^[_a-zA-Z]([a-z0-9A-Z][a-zA-Z0-9]*)?$"/>
<message key="name.invalidPattern" value="Member name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="ParameterName">
<property name="format" value="^[a-zA-Z]([a-z0-9A-Z][a-zA-Z0-9]*)?$"/>
<message key="name.invalidPattern" value="Parameter name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="CatchParameterName">
<property name="format" value="^[a-zA-Z]([a-z0-9A-Z][a-zA-Z0-9]*)?$"/>
<message key="name.invalidPattern" value="Catch parameter name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="LocalVariableName">
<property name="tokens" value="VARIABLE_DEF"/>
<property name="format" value="^[a-zA-Z]([a-z0-9A-Z][a-zA-Z0-9]*)?$"/>
<message key="name.invalidPattern" value="Local variable name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="ClassTypeParameterName">
<property name="format" value="(^[A-Z][0-9]?)$|([A-Z][_A-Z0-9]*$)"/>
<message key="name.invalidPattern" value="Class type name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="MethodTypeParameterName">
<property name="format" value="(^[A-Z][0-9]?)$|([A-Z][_A-Z0-9]*$)"/>
<message key="name.invalidPattern" value="Method type name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="InterfaceTypeParameterName">
<property name="format" value="(^[A-Z][0-9]?)$|([A-Z][_A-Z0-9]*$)"/>
<message key="name.invalidPattern" value="Interface type name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="NoFinalizer"/>
<module name="GenericWhitespace">
<message key="ws.followed" value="GenericWhitespace ''{0}'' is followed by whitespace."/>
<message key="ws.preceded" value="GenericWhitespace ''{0}'' is preceded with whitespace."/>
<message key="ws.illegalFollow" value="GenericWhitespace ''{0}'' should followed by whitespace."/>
<message key="ws.notPreceded" value="GenericWhitespace ''{0}'' is not preceded with whitespace."/>
</module>
<module name="Indentation">
<property name="basicOffset" value="3"/>
<property name="braceAdjustment" value="0"/>
<property name="caseIndent" value="3"/>
<property name="throwsIndent" value="6"/>
<property name="lineWrappingIndentation" value="3"/>
<property name="arrayInitIndent" value="6"/>
</module>
<!--
<module name="AbbreviationAsWordInName">
<property name="ignoreFinal" value="false"/>
<property name="allowedAbbreviationLength" value="1"/>
</module>
<module name="OverloadMethodsDeclarationOrder"/>
<module name="VariableDeclarationUsageDistance"/>
<module name="CustomImportOrder">
<property name="sortImportsInGroupAlphabetically" value="true"/>
<property name="separateLineBetweenGroups" value="true"/>
<property name="customImportOrderRules" value="STATIC###THIRD_PARTY_PACKAGE"/>
</module>
-->
<module name="CustomImportOrder">
<property name="customImportOrderRules" value="SPECIAL_IMPORTS###STANDARD_JAVA_PACKAGE###THIRD_PARTY_PACKAGE###STATIC"/>
<property name="specialImportsRegExp" value="^javax\."/>
<property name="standardPackageRegExp" value="^java\."/>
<property name="sortImportsInGroupAlphabetically" value="true"/>
<property name="separateLineBetweenGroups" value="false"/>
</module>
<module name="MethodParamPad"/>
<module name="NoWhitespaceBefore">
<property name="tokens" value="COMMA, SEMI, POST_INC, POST_DEC, DOT, ELLIPSIS, METHOD_REF"/>
<property name="allowLineBreaks" value="true"/>
</module>
<module name="ParenPad"/>
<module name="OperatorWrap">
<property name="option" value="NL"/>
<property name="tokens" value="BAND, BOR, BSR, BXOR, DIV, EQUAL, GE, GT, LAND, LE, LITERAL_INSTANCEOF, LOR, LT, MINUS, MOD, NOT_EQUAL, PLUS, QUESTION, SL, SR, STAR, METHOD_REF "/>
</module>
<module name="AnnotationLocation">
<property name="id" value="AnnotationLocationMostCases"/>
<property name="tokens" value="CLASS_DEF, INTERFACE_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF"/>
</module>
<module name="AnnotationLocation">
<property name="id" value="AnnotationLocationVariables"/>
<property name="tokens" value="VARIABLE_DEF"/>
<property name="allowSamelineMultipleAnnotations" value="true"/>
</module>
<module name="JavadocMethod">
<property name="allowMissingParamTags" value="true"/>
<property name="allowMissingReturnTag" value="true"/>
<property name="allowedAnnotations" value="Override, Test"/>
</module>
<module name="MissingJavadocMethod">
<property name="scope" value="private"/>
</module>
<module name="MissingJavadocType">
<property name="scope" value="private"/>
</module>
<module name="MethodName">
<property name="format" value="^[a-z][a-zA-Z0-9_]*$"/>
<message key="name.invalidPattern"
value="Method name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="EmptyCatchBlock">
<property name="exceptionVariableName" value="expected"/>
</module>
<module name="CommentsIndentation"/>
<!-- this would be nice, but requires the the @inheritDoc javadoc tag... -->
<module name="MissingOverride"/>
</module>
<module name="Header">
<property name="headerFile" value="checkstyle/license.txt"/>
<property name="fileExtensions" value="java"/>
<property name="ignoreLines" value="3"/>
</module>
<module name="SuppressWarningsFilter"/>
</module>
+19
View File
@@ -0,0 +1,19 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-42
View File
@@ -1,42 +0,0 @@
# Getting Started
## Prerequisites
- Java 17+
- Maven 3.8+
- QQQ 0.23.0+
## Creating Your Extension QBit
1. Click "Use this template" on GitHub
2. Clone your new repository
3. Run the customization script:
```bash
python scripts/customize_template.py
```
4. Enter your QBit name when prompted
## Project Structure
```
src/main/java/com/kingsrook/qbits/yourextension/
├── YourExtensionQBitConfig.java
├── YourExtensionQBitProducer.java
├── customizers/
└── actions/
```
## Using Your QBit
```java
new YourExtensionQBitProducer()
.withConfig(new YourExtensionQBitConfig()
.withTargetTableName("order"))
.produce(qInstance, "myExtension");
```
## Next Steps
- Read [Extension Patterns](01-extension-patterns.md)
- Implement your customization logic
- Add tests
-58
View File
@@ -1,58 +0,0 @@
# Extension Patterns
## Table Customizers
Add behavior to existing tables:
```java
public class AuditTableCustomizer
{
public void customize(QTableMetaData table)
{
table.withField(new QFieldMetaData("createdAt", QFieldType.DATE_TIME));
table.withField(new QFieldMetaData("modifiedAt", QFieldType.DATE_TIME));
table.withCustomizer(
TableCustomizers.PRE_INSERT_RECORD,
new QCodeReference(AuditPreInsert.class));
}
}
```
## Action Customizers
Wrap actions with additional behavior:
```java
public class LoggingActionWrapper implements PreInsertRecordInterface
{
@Override
public void preInsertRecord(InsertInput input, QRecord record) throws QException
{
LOG.info("Inserting record", logPair("table", input.getTableName()));
}
}
```
## Config Validation
Validate that required components exist:
```java
@Override
public void validate(QInstance qInstance, List<String> errors)
{
if(qInstance.getTable(targetTableName) == null)
{
errors.add("Target table required: " + targetTableName);
}
}
```
## When to Use Extension QBits
- Adding audit logging to tables
- Implementing authentication/authorization
- Adding encryption to fields
- Custom validation rules
- Performance monitoring
+383
View File
@@ -0,0 +1,383 @@
# QBit WORM Audit: design
Status: approved. **Owner decisions (2026-10-12): all recommendations accepted.** Every open
question in section 12 is decided as recommended.
## 1. Owner request
> "we also need to ensure that our auditing qbit supports WORM audits as well. and the phi/secure
> data qbit needs to ensure that we don't leak phi into audits"
Context: HIPAA §164.312(b) audit controls, §164.312(c)(1) integrity, §164.316(b)(2)(i) six-year
retention of required documentation (audit records are kept to the same horizon), and a
HITRUST-style review that asks three things: (1) can anyone change or delete an audit record,
(2) would we know if they did, and (3) is there a copy nobody can alter?
## 2. Goals and non-goals
Goals:
1. **Append-only at the database level.** No application path, and no ordinary database login,
can UPDATE, DELETE or TRUNCATE audit rows. The only removal is an audited retention purge,
after the retention date, through a separate database role.
2. **Tamper evidence.** Every audit row is linked into a hash chain. Signed checkpoints anchor the
chain. A verification action and a daily job report the exact first broken row.
3. **Sealed copies.** New audit rows and checkpoints are exported on a schedule to an S3 bucket
with Object Lock in COMPLIANCE mode (default six years), and can be verified against the live
chain.
4. **No PHI in the WORM layer.** The chain and the exports seal exactly what core wrote. With
qbit-secured-data that means HMACs for classified values, never plaintext.
Non-goals: changing what core audits (that is core's and qbit-secured-data's job); protecting
against a Postgres superuser *preventing* a change (a superuser can disable triggers; the
chain, the signatures and the Object Lock copies make that change *detectable*, see 4.5).
## 3. Scope: which tables
| Table (QQQ name / kof22 physical) | Treatment |
|---|---|
| `audit` / `audit` | Append-only (UPDATE, DELETE, TRUNCATE rejected). Chained. |
| `auditDetail` / `audit_detail` | Append-only. Chained with its header. |
| `auditTable` / `audit_table` | Lookup. DELETE and TRUNCATE rejected; UPDATE of `id` or `name` rejected (label and dates may change). Names are resolved into the chained content, so a later rename is detected. |
| `auditUser` / `audit_user` | As `audit_table`. |
| `worm_audit_chain`, `worm_audit_checkpoint`, `worm_audit_export`, `worm_audit_verification`, `worm_audit_purge` | The qbit's own tables: all append-only. |
Physical names come from the host's backend (kof22 maps camelCase to snake_case). The DDL is a
template rendered for the host's names (section 5.4).
## 4. Append-only enforcement (Postgres first)
### 4.1 Triggers
One function, `worm_audit_reject()`, attached as:
- `BEFORE UPDATE OR DELETE ... FOR EACH ROW` on every append-only table;
- `BEFORE TRUNCATE ... FOR EACH STATEMENT` on every append-only table;
- on the two lookups, a row trigger that rejects DELETE and any UPDATE that changes `id` or
`name`.
The error is explicit and greppable:
```
ERROR: worm_audit: audit rows are write-once; UPDATE of "audit" is not allowed (id=4211). Expired rows are removed only by worm_audit_purge().
SQLSTATE: P0001 (HINT: see qbit-worm-audit docs/DESIGN.md section 4)
```
In the application the qbit also removes the update and delete capabilities from these tables'
QQQ metadata, so the admin UI and both APIs never offer edit or delete, and a direct call gets
QQQ's standard "table does not support update/delete" error before reaching the database. The
qbit maps a `P0001` with the `worm_audit:` prefix to a `QUserFacingException`: "Audit records are
write-once and cannot be changed or deleted."
### 4.2 The retention purge (the only removal path)
`worm_audit_purge(p_cutoff timestamptz)` is a `SECURITY DEFINER` function owned by the schema
owner. `EXECUTE` is revoked from `PUBLIC` and granted only to the role `worm_audit_purger`, which
the app role is not a member of.
1. It refuses (and records a `REFUSED` row in `worm_audit_purge`, then returns; no exception, so
the record survives) when `p_cutoff > now() - retention`. The retention interval is a constant
compiled into the function by the changeset (default `6 years`), so shortening it is a schema
migration, reviewed and itself audited.
2. It purges only a **chain prefix**: the audits whose chain links all lie at or before the last
link whose audit timestamp is older than the cutoff, together with their details and links.
3. It sets a transaction-local flag (`set_config('worm_audit.purging', 'on', true)`). The reject
trigger allows DELETE only when that flag is on **and** `current_user` is the function owner
**and** the row is older than the retention interval. Setting the flag from another session
does nothing, because `current_user` is not the owner.
4. It writes a `PURGED` row: who (session_user), cutoff, retention, counts, and the **anchor**
(the last purged link's id and hash). Verification continues from the anchor.
The purge runs from an ops runbook (`psql` as `worm_audit_purger`). Version 0.1 has no UI action;
the app home lists purge rows, including refusals.
### 4.3 GRANT model
| Role | Owns | Grants on append-only tables |
|---|---|---|
| `<app>_owner` (Liquibase) | tables, functions, triggers | all (owner) |
| `<app>` (runtime) | nothing | `SELECT, INSERT`; `USAGE` on their sequences. No UPDATE, DELETE or TRUNCATE. |
| `worm_audit_purger` (ops) | nothing | `EXECUTE worm_audit_purge`; nothing else |
When a host uses one role for both migrations and runtime (kof22 today), the triggers still stop
every UPDATE, DELETE and TRUNCATE from the application, but the role can `ALTER TABLE ... DISABLE
TRIGGER`. The docs recommend splitting the roles, and the app home shows a warning when the
runtime role owns the audit tables (checked through `pg_tables.tableowner = current_user`).
### 4.4 Changesets
The qbit ships `src/main/resources/com/kingsrook/qbits/wormaudit/ddl/postgres/worm-audit.sql`,
a Liquibase formatted-SQL template with placeholders for the four core audit table names, the
app role and the retention interval, plus a renderer (`WormAuditDdl.postgres(config)`) and the
rendered default (`docs/ddl/worm-audit-postgres.sql`, snake_case names). Hosts copy the rendered
file into their changelog (kof22: `changesets/features/YYYY-MM-DD-worm-audit.sql`). It creates the
qbit tables, the functions, the triggers, the purger role (`CREATE ROLE ... NOLOGIN` when it is
missing) and the grants. It is idempotent (`CREATE OR REPLACE`, `DROP TRIGGER IF EXISTS`).
### 4.5 Threat model summary
| Actor | Can modify audit rows? | Detected by |
|---|---|---|
| App user, admin UI, QQQ APIs, assistant | No (capabilities removed; triggers) | n/a |
| SQL injection or a bug as the runtime role | No (triggers; no grant) | n/a |
| Owner role (migrations) | Only by disabling triggers | chain verification (daily), alert |
| Postgres superuser rewriting rows and recomputing the chain | Yes | checkpoint signatures (no signing key in the DB); sealed exports in Object Lock |
| Anyone deleting the sealed copies | No (COMPLIANCE mode: not even the bucket root, until expiry) | n/a |
### 4.6 Other databases (documented, not shipped in 0.1)
- **MySQL/MariaDB:** `BEFORE UPDATE`/`BEFORE DELETE` triggers with `SIGNAL SQLSTATE '45000'`;
no TRUNCATE trigger, so revoke `DROP` from the app user; purge through a `SQL SECURITY DEFINER`
procedure.
- **SQL Server:** `INSTEAD OF UPDATE, DELETE` triggers plus `DENY UPDATE, DELETE` to the app user.
- **H2** (tests only): no enforcement; the chain and the exports work on any QQQ RDBMS or memory
backend, so tamper *evidence* is database-independent.
## 5. Tamper evidence
### 5.1 Canonical content (agreed with qbit-secured-data)
One canonical document per audit header: the header plus all of its detail rows, as RFC 8785 JCS
JSON (sorted keys, no insignificant whitespace, UTF-8):
```json
{"audit":{"clientChannel":null,"id":4211,"message":"Record was Edited","recordId":"17","table":"cms_page","timestamp":"2026-10-13T21:04:05.123456Z","user":"james"},
"details":[{"fieldName":"title","id":9001,"message":"Changed Title from ...","newValue":"...","oldValue":"..."}],
"v":1}
```
- Values are exactly as stored: no formatting, no re-masking, no lookups other than resolving
`auditTableId`/`auditUserId` to their names. `details` are sorted by id.
- With qbit-secured-data's core hook (QRun-IO/qqq#1161, `docs/SECURED-DATA.md` §14.2), PII and
PHI values are tokenised inside `DMLAuditAction` before `AuditAction` inserts, in the same
transaction as the data change, so the token is the only form ever written. Stored forms (ASCII,
at most 200 characters) in `oldValue`/`newValue` are
`hmac:v1:<keyId>:<b64url>`, `shred:v1:<auditKeyId>:<b64url>`, `changed:v1`,
`wiped:v1:<ruleId>` and `shredded:v1:<ruleId>` (the last is its own detail row, written when
keys are destroyed, and is chained like any other row). Detail messages for those fields
are replaced with value-free ones ("Changed <Label>"); header messages carry no field values.
The canonical form, the chain and the exports therefore contain tokens only, and verifying the
chain never needs a key. The WORM layer never reads, joins or exports
`secured_data_audit_key` or any other `secured_data_*` table. Crypto-shredding (destroying a
record's key) leaves the HMACs and the chain intact; they simply become unlinkable.
- The canonical JSON is never stored in the database. The chain stores ids and hashes only, so
the qbit adds no new copy of audit content to the database.
### 5.2 The chain: a single asynchronous chainer (recommended)
`link_hash = SHA-256(prev_link_hash || 0x00 || canonical_bytes)`, hex, with the genesis
`prev_link_hash` = 64 zeros.
Core inserts audits from many threads, and headers and details may be inserted in separate
autocommit statements (core passes the caller's transaction, which may be null). Computing the
chain in core's write path would need a global lock on every audited write, and a trigger-based
chain would need the canonical form in SQL. The design instead uses **one chainer**:
- A scheduled job (every 30 s by default) takes `pg_try_advisory_xact_lock` (any other instance
skips that tick; on non-Postgres backends, a single-row lock table with `SELECT ... FOR
UPDATE`). It selects audit headers that are **older than a settle horizon** (default 60 s, so
their details have been committed) and not yet chained (an anti-join on the unique
`worm_audit_chain.audit_id`), in `(timestamp, id)` order. Then it appends links in batches.
- Link ids are assigned by the chainer as `max(id) + 1` inside the lock, not from a sequence, so
the chain is contiguous (a sequence gaps on rollback).
- **Late arrivals** (a header whose id sorts before already-chained ones because its transaction
committed late) are chained when they appear. The chain order is chaining order, not audit id
order, and each link records its `audit_id`.
- **Late details** (a detail inserted after its header was chained) get their own link,
`kind = LATE_DETAIL`, covering exactly those detail ids. Verification flags any detail not
covered by exactly one link.
- Unique constraints back the lock: `UNIQUE (audit_id) WHERE kind = 'AUDIT'`, and the detail
coverage.
Why not per-partition chains: they make each partition's chain shorter but multiply heads,
checkpoints and verification state, and they still need a writer per partition. One chainer
handles thousands of rows a second in batches, which is well above audit volume, and keeps a
single head to sign. The table carries no partition column (YAGNI). A future partitioned mode
would add one.
Window: a row is unchained for at most horizon plus interval (about 90 s). During that window the
triggers still protect it. The chain defends against privileged tampering, which the window does
not change materially.
### 5.3 Signed checkpoints (`worm_audit_checkpoint`)
Every hour, and every 10,000 links, the chainer writes a checkpoint for the links since the last
one: `from_link_id`, `to_link_id`, `head_hash` (the last link hash), `merkle_root` (a binary
SHA-256 Merkle tree over the range's link hashes, so a single exported range can be proven
without the whole chain), `key_id`, `public_key`, `algorithm = Ed25519` and `signature` over the
JCS form of those fields. The signing key comes from config (`WormAuditQBitConfig.signingKey`, a
supplier; kof22: env `WORM_AUDIT_SIGNING_KEY` from Infisical) and never touches the database. The
public key is stored on each checkpoint and pinned in config (`trustedPublicKeys`), so a forged
checkpoint signed with another key fails verification. Rotation adds a new key id. Old
checkpoints verify against their own pinned key.
### 5.4 Verification
**"Verify audit chain"**, a table action on `worm_audit_chain` (and on selected links), is a
standard streamed ETL:
- Extract: the links in scope (all since the purge anchor, or the selection), in id order.
- Transform: recompute each link from the live audit rows (`prev` continuity, canonical content,
`link_hash`); check checkpoint signatures and Merkle roots; for a full run, check completeness
(every settled audit and detail is covered exactly once).
- Load: one `worm_audit_verification` row.
- Review: "1,204,331 chain links will be verified (links 1–1,204,331, 121 checkpoints)."
- Result: "1,204,331 links verified" or, on failure, one line per break, each with the exact row:
"Break at link 88,412 (audit 90,117, table cms_page, record 17): content differs from the
chained hash" / "audit_detail 9,002 is not covered by the chain" / "checkpoint 41: signature
does not verify".
**Daily verification**: a scheduled job runs the same verifier at 03:00. A failure raises
`QAlerts.raise(new AlertSignal("worm_audit.chain.break").withDedupKey("chain")...)` with the first
break in the title and the verification record as the record ref; a later pass resolves it. Every
run sends `QAlerts.heartbeat("worm_audit.verify.daily")`. `QAlerts` is the core producer SPI
(QQQ 4.1.0-RC.5), so the qbit needs no dependency on qbit-alerts. With no alerts implementation
installed the alerts go to the log.
## 6. Sealed exports
### 6.1 Export job
Daily at 02:00 (configurable), and by the table action "Export now" on `worm_audit_export`:
1. Range: links after the last export's `to_link_id`, up to the latest checkpoint. Exports are
contiguous by construction. The app home flags a gap (a range not covered) and lag.
2. Objects under `<prefix>/<yyyy>/<mm>/<dd>/<from>-<to>/`:
- `links.jsonl`: one line per link: `{"link":{id,kind,auditId,detailIds,prevHash,linkHash},"content":<canonical document>}`;
- `checkpoints.jsonl`: the checkpoints in range, with signatures;
- `manifest.json`: format version, range, counts, the SHA-256 of each file, `head_hash`,
the previous export's manifest SHA-256 (the exports form their own chain), qbit version.
3. Each object is put through qbit-object-storage (`SystemStorage`, bucket alias from config,
default `worm_audit`) with `LockMode.COMPLIANCE` and `retainUntil = now + retentionYears`. The
export refuses to run, and alerts `worm_audit.export.failed`, when the bucket does not report
the `OBJECT_LOCK` capability or the put does not echo the lock.
4. On success it inserts one `worm_audit_export` row (insert-only; a failed export leaves no row,
and its alert carries the reason).
### 6.2 Restore and verify a sealed export
"Verify sealed export", a record action on `worm_audit_export` (standard ETL), downloads the
objects. It checks the manifest hashes, the lock mode and retain-until of each object, the link
hashes recomputed from the exported content, the checkpoint signatures, and the export's link to
the previous manifest. Then it compares each exported link and content document with the live
chain and the live audit rows. Result lines name each difference ("link 88,412: live content
differs from the sealed copy (sealed 2026-10-14)"). This is also the restore tool: the exported
content is the canonical audit content, so a verified export is evidence of the original rows
after a live tamper. A purged range stays verifiable from the sealed copy alone. Writing restored
rows back into a live table is out of scope (it would need the purge role and break append-only).
## 7. Tables
All tables use the host's RDBMS backend, are append-only (section 4), are audited by nothing
(auditing the audit tables would recurse), have full record screens, and are searchable in Quick
Search. ESB table events are on for `worm_audit_verification` and `worm_audit_export` (the
events integrations care about); chain links are high-volume and publish none.
- **`worm_audit_chain`** [Audit Chain Link]: `id` (contiguous), `kind` (AUDIT, LATE_DETAIL),
`audit_id`, `detail_ids` (comma list, at most 250 per link), `prev_hash`, `link_hash`,
`chained_at`. Unique `(audit_id) where kind = 'AUDIT'`.
- **`worm_audit_checkpoint`** [Checkpoint]: `id`, `from_link_id`, `to_link_id`, `link_count`,
`head_hash`, `merkle_root`, `algorithm`, `key_id`, `public_key`, `signature`, `created_at`.
- **`worm_audit_export`** [Sealed Export]: `id`, `from_link_id`, `to_link_id`,
`from_checkpoint_id`, `to_checkpoint_id`, `bucket_alias`, `object_prefix`, `manifest_key`,
`manifest_sha256`, `prev_manifest_sha256`, `object_count`, `byte_count`, `lock_mode`,
`retain_until`, `sealed_at`.
- **`worm_audit_verification`** [Verification]: `id`, `kind` (CHAIN, SEALED_EXPORT), `trigger`
(SCHEDULE, USER), `started_at`, `finished_at`, `from_link_id`, `to_link_id`, `links_checked`,
`outcome` (PASS, FAIL), `break_count`, `first_break_link_id`, `first_break_audit_id`,
`first_break_reason`, `export_id`, `run_by`.
- **`worm_audit_purge`** [Purge]: `id`, `requested_at`, `requested_by`, `cutoff`,
`retention_interval`, `outcome` (PURGED, REFUSED), `reason`, `audits_deleted`,
`details_deleted`, `links_deleted`, `anchor_link_id`, `anchor_hash`.
Nav (the app "WORM Audit"): Home, Verifications, Sealed Exports, Checkpoints, Audit Chain, Purges.
Checkpoints list their links as a child list. Exports and verifications link their checkpoints
and links.
## 8. Permissions and the assistant wall
- Table permissions use the full name (`worm_audit_chain.read`, ...). There are no insert, edit or
delete permissions to grant: the capabilities are absent.
- Process permissions: `worm_audit.verify`, `worm_audit.export`. The home is visible with read on
any `worm_audit_*` table.
- The assistant channel is denied `worm_audit_*`. The qbit registers
`ClientChannelsMetaData.ofOrWithNew(qInstance).withDeniedPattern("assistant", "worm_audit_*")`
(core, QQQ 4.1.0-RC.6, #1141). On RC.5 it does so reflectively when the class is present. A
one-line core PR also adds `worm_audit_*` to `ASSISTANT_DEFAULT_DENIED_PATTERNS`, so the wall
holds even if the qbit is configured without its producer. Core's `audit*` pattern already
covers the core audit tables.
## 9. App home
A real home that owns the data (no duplicate dashboard), built from standard widgets, served
through `/qqq/v1/widget` like every QQQ widget:
- **Chain status**: links chained, last link time, unchained backlog (with the settle horizon),
last verification (time, outcome, links), open breaks with the first break's exact row and a
link to the verification record.
- **Export status**: last sealed export (range, time, retain-until), lag in links and hours,
gaps, and bucket lock state (COMPLIANCE, retention).
- **Retention**: configured years, the purge anchor, the oldest live audit, the next eligible
purge date.
- **Recent purges**: the last 10 purge rows, refusals included.
- **Setup warnings**: triggers missing on any table (checked via `pg_trigger`), runtime role owns
the audit tables, signing key or trusted keys missing, bucket without Object Lock.
The home must be readable at every width (390 to 1920 px) in light and dark mode, with no inner
horizontal scroll. Screenshots go in `docs/screenshots/`. If a custom endpoint is ever needed, it
registers as a `MiddlewareSpecExtension` under `/qqq/v1/qbit/worm-audit/...`
(docs/UI-API-EXTENSIONS.md). The design needs none.
## 10. Configuration (`WormAuditQBitConfig`)
`retentionYears` (6), `signingKey` (supplier of an Ed25519 private key), `trustedPublicKeys`
(key id to public key), `chainIntervalSeconds` (30), `settleHorizonSeconds` (60),
`checkpointEveryLinks` (10,000), `checkpointEveryMinutes` (60), `exportBucketAlias` (`worm_audit`),
`exportPrefix` (`worm-audit`), `exportCron` (02:00 daily), `verifyCron` (03:00 daily),
`coreAuditTableNames` (physical names, defaulting to QQQ's), `tableMetaDataCustomizer`.
## 11. Tests (acceptance)
1. **Postgres Testcontainers** (`postgres:16`): the rendered DDL applies twice (idempotent);
UPDATE, DELETE and TRUNCATE on `audit`, `audit_detail` and each `worm_audit_*` table fail with
the `worm_audit:` message; lookup renames fail and label changes succeed; the runtime role has
no UPDATE/DELETE grant; `worm_audit_purge` refuses a recent cutoff (and records it), purges an
expired prefix as the purger role, records the anchor, and verification passes from the
anchor; setting `worm_audit.purging` from the app session does not allow DELETE.
2. **Chain**: chain, verify (PASS); a forced tamper (triggers disabled as superuser, one
`audit_detail.new_value` changed) is detected, naming that exact link, audit and detail; a
deleted row, an inserted row, a reordered link and a forged checkpoint are each detected.
3. **Concurrency**: 8 threads inserting audits (with late commits and details in separate
statements) while 3 chainers run. The chain is contiguous, every audit and detail is covered
exactly once, and verification passes.
4. **Exports (MinIO Testcontainers, bucket with Object Lock)**: export, then verify the round
trip; every object has COMPLIANCE retention; deleting or overwriting an object version is
refused by MinIO; a tampered live row is reported by "Verify sealed export"; a bucket without
Object Lock makes the export refuse and alert.
5. **PHI**: a field classified PHI (`AiFieldMetaData.withClassification(DataClassification.PHI)`,
read through `AiDataClassifications.ofField`) is written with the value `CANARY-PHI-<random>`.
With secured-data's audit token hook (a stand-in producing the same `hmac:v1:` form in the
qbit's tests until the core hook ships), the audit header, the details (old, new, message),
the canonical content and the export JSONL contain neither the canary nor any substring of it
longer than 4 characters.
6. **Alerts**: a break raises `worm_audit.chain.break` through `QAlerts` (captured by a test
sink); a later pass resolves it.
7. Coverage gates: 70% instructions, 90% classes.
## 12. Decisions (all accepted as recommended, 2026-10-12)
| # | Question | Decision |
|---|---|---|
| 1 | Chain construction | A single asynchronous chainer with a settle horizon, contiguous link ids and late-arrival links (5.2), not synchronous or per-partition chains. |
| 2 | Chain storage | A sidecar table (`worm_audit_chain`), not columns on core tables, so core's schema is untouched and no UPDATE of audit rows is ever needed. |
| 3 | Checkpoints | Hourly or every 10,000 links; Ed25519; Merkle root per range. |
| 4 | Lookup tables | DELETE and id/name changes rejected; names resolved into the content. |
| 5 | Purge | SQL function plus a purger role only, chain-prefix only, no UI action in 0.1. |
| 6 | Export | Daily JSONL plus manifest, COMPLIANCE, six years, contiguous ranges, the exports form their own chain. |
| 7 | Databases | Postgres enforced; MySQL and SQL Server documented; evidence on any backend. |
## 13. Phases
- **(a)** DDL and renderer, capabilities guard, chain and chainer, checkpoints, verification
action and daily job with alerts, the assistant deny pattern; tests 1–3, 5 and 6.
- **(b)** Sealed exports and the export verifier through qbit-object-storage; test 4.
- **(c)** App home and setup warnings, screenshots; kof22 adoption (draft Website-Backend PR, a
feature changeset) and the infra proposal for an Object Lock bucket (section 14).
## 14. Adoption infrastructure (kof22, proposal only)
- **Bucket:** `kof22-worm-audit` on the kof22 MinIO, created by `minio-bucket-init` with
`mc mb --with-lock --ignore-existing` and `mc retention set --default COMPLIANCE 6y`. Object
Lock can only be enabled at bucket creation.
- **Policy:** `kof22-app` gets `s3:PutObject`, `s3:GetObject`, `s3:ListBucket`,
`s3:GetObjectRetention`, `s3:PutObjectRetention` (needed to set the lock on each put),
`s3:GetBucketObjectLockConfiguration` on that bucket. No delete, and no
`s3:BypassGovernanceRetention` (irrelevant under COMPLIANCE anyway).
- **Off-site:** COMPLIANCE on a single homelab MinIO stops every API deletion, but not someone
destroying the volume. Recommend extending `media-offsite-copy` to replicate the bucket to a B2
bucket with Object Lock (COMPLIANCE, 6 years).
- **Secrets:** `WORM_AUDIT_SIGNING_KEY` (Ed25519 PKCS#8, base64) generated in Praetor
`terraform/infisical-config` and delivered by ESO. The public key goes in the backend config.
- **Database:** split the kof22 owner and runtime roles when convenient (4.3). Until then the
home shows the owner warning.
These are owner-run (Praetor and cluster-gitops are applied by the owner). The qbit only
documents them.
+325
View File
@@ -0,0 +1,325 @@
-- liquibase formatted sql
-- changeset worm-audit:1
CREATE TABLE IF NOT EXISTS public.worm_audit_chain (
id bigint PRIMARY KEY,
kind text NOT NULL,
audit_id bigint NOT NULL,
detail_ids text NOT NULL,
prev_hash text NOT NULL,
link_hash text NOT NULL,
chained_at timestamptz NOT NULL,
CHECK (id > 0),
CHECK (kind IN ('AUDIT', 'LATE_DETAIL')),
CHECK (prev_hash ~ '^[0-9a-f]{64}$' AND link_hash ~ '^[0-9a-f]{64}$'),
CHECK (detail_ids = '' OR (detail_ids ~ '^(0|[1-9][0-9]*)(,(0|[1-9][0-9]*))*$' AND cardinality(string_to_array(detail_ids, ',')) <= 250))
);
CREATE TABLE IF NOT EXISTS public.worm_audit_checkpoint (
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
from_link_id bigint,
to_link_id bigint,
link_count bigint,
head_hash text,
merkle_root text,
algorithm text,
key_id text,
public_key text,
signature text,
created_at timestamptz
);
CREATE TABLE IF NOT EXISTS public.worm_audit_export (
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
from_link_id bigint,
to_link_id bigint,
from_checkpoint_id bigint,
to_checkpoint_id bigint,
bucket_alias text,
object_prefix text,
manifest_key text,
manifest_sha256 text,
prev_manifest_sha256 text,
object_count bigint,
byte_count bigint,
lock_mode text,
retain_until timestamptz,
sealed_at timestamptz
);
CREATE TABLE IF NOT EXISTS public.worm_audit_verification (
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
kind text,
"trigger" text,
started_at timestamptz,
finished_at timestamptz,
from_link_id bigint,
to_link_id bigint,
links_checked bigint,
outcome text,
break_count bigint,
first_break_link_id bigint,
first_break_audit_id bigint,
first_break_reason text,
export_id bigint,
run_by text
);
CREATE TABLE IF NOT EXISTS public.worm_audit_purge (
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
requested_at timestamptz,
requested_by text,
cutoff timestamptz,
retention_interval text,
outcome text,
reason text,
audits_deleted bigint,
details_deleted bigint,
links_deleted bigint,
anchor_link_id bigint,
anchor_hash text
);
CREATE UNIQUE INDEX IF NOT EXISTS worm_audit_chain_audit ON public.worm_audit_chain(audit_id) WHERE kind = 'AUDIT';
CREATE INDEX IF NOT EXISTS worm_audit_chain_audit_all ON public.worm_audit_chain(audit_id);
-- changeset worm-audit:2 splitStatements:false
CREATE OR REPLACE FUNCTION public.worm_audit_reject() RETURNS trigger
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp SET timezone = 'UTC' AS $$
DECLARE
old_time timestamptz;
function_owner name;
stack text;
row_identity text := 'all';
BEGIN
IF TG_OP = 'DELETE' AND current_setting('worm_audit.purging', true) = 'on' THEN
SELECT pg_get_userbyid(proowner) INTO function_owner FROM pg_proc
WHERE oid = 'public.worm_audit_purge(timestamptz)'::regprocedure;
GET DIAGNOSTICS stack = PG_CONTEXT;
IF current_user = function_owner AND position('function worm_audit_purge(timestamp with time zone)' in stack) > 0 THEN
IF TG_RELID = '"audit"'::regclass THEN
old_time := OLD.timestamp;
ELSIF TG_RELID = '"audit_detail"'::regclass THEN
SELECT timestamp INTO old_time FROM "audit" WHERE id = OLD.audit_id;
ELSIF TG_RELID = 'public.worm_audit_chain'::regclass THEN
SELECT timestamp INTO old_time FROM "audit" WHERE id = OLD.audit_id;
END IF;
IF old_time < now() - interval '6 years' THEN
RETURN OLD;
END IF;
END IF;
END IF;
IF TG_LEVEL = 'ROW' THEN
row_identity := OLD.id::text;
END IF;
RAISE EXCEPTION USING ERRCODE = 'P0001',
MESSAGE = format('worm_audit: audit rows are write-once; %s of "%s" is not allowed (id=%s). Expired rows are removed only by worm_audit_purge().',
TG_OP, TG_TABLE_NAME, row_identity),
HINT = 'see qbit-worm-audit docs/DESIGN.md section 4';
END;
$$;
-- changeset worm-audit:3 splitStatements:false
CREATE OR REPLACE FUNCTION public.worm_audit_lookup_reject() RETURNS trigger
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $$
BEGIN
IF TG_OP = 'UPDATE' AND NEW.id IS NOT DISTINCT FROM OLD.id AND NEW.name IS NOT DISTINCT FROM OLD.name THEN
RETURN NEW;
END IF;
RAISE EXCEPTION USING ERRCODE = 'P0001',
MESSAGE = format('worm_audit: audit rows are write-once; %s of "%s" is not allowed (id=%s). Expired rows are removed only by worm_audit_purge().',
TG_OP, TG_TABLE_NAME, OLD.id),
HINT = 'see qbit-worm-audit docs/DESIGN.md section 4';
END;
$$;
-- changeset worm-audit:4 splitStatements:false
DO $$
BEGIN
IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'worm_audit_purger') THEN
CREATE ROLE worm_audit_purger NOLOGIN;
END IF;
END;
$$;
-- changeset worm-audit:5 splitStatements:false
CREATE OR REPLACE FUNCTION public.worm_audit_purge(p_cutoff timestamptz) RETURNS void
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, public, pg_temp SET timezone = 'UTC' AS $$
DECLARE
boundary bigint;
smaller bigint;
anchor text;
audits_count bigint;
details_count bigint;
links_count bigint;
BEGIN
IF p_cutoff IS NULL OR p_cutoff > now() - interval '6 years' THEN
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome, reason)
VALUES(now(), session_user, p_cutoff, '6 years', 'REFUSED', 'Cutoff is within retention or null');
RETURN;
END IF;
-- Same lock as the Java chainer; table locks also exclude concurrent audit inserts.
PERFORM pg_advisory_xact_lock(86753090421);
LOCK TABLE "audit", "audit_detail", public.worm_audit_chain IN SHARE ROW EXCLUSIVE MODE;
SELECT coalesce(min(c.id) - 1, (SELECT max(id) FROM public.worm_audit_chain)) INTO boundary
FROM public.worm_audit_chain c LEFT JOIN "audit" a ON a.id = c.audit_id
WHERE a.id IS NULL OR a.timestamp IS NULL OR a.timestamp >= p_cutoff;
-- Shrink to a prefix closed over every audit's links AND all its stored details.
LOOP
SELECT min(c.id) - 1 INTO smaller FROM public.worm_audit_chain c
WHERE c.id <= boundary AND (
EXISTS (SELECT FROM public.worm_audit_chain later WHERE later.audit_id = c.audit_id AND later.id > boundary)
OR EXISTS (SELECT FROM "audit_detail" d WHERE d.audit_id = c.audit_id AND NOT EXISTS (
SELECT FROM public.worm_audit_chain covered WHERE covered.audit_id = c.audit_id
AND d.id::text = ANY(string_to_array(covered.detail_ids, ',')))));
EXIT WHEN smaller IS NULL;
boundary := smaller;
END LOOP;
SELECT link_hash INTO anchor FROM public.worm_audit_chain WHERE id = boundary;
IF anchor IS NULL THEN
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome, reason)
VALUES(now(), session_user, p_cutoff, '6 years', 'REFUSED', 'No complete expired chain prefix');
RETURN;
END IF;
PERFORM set_config('worm_audit.purging', 'on', true);
DELETE FROM "audit_detail" d WHERE EXISTS
(SELECT FROM public.worm_audit_chain c WHERE c.audit_id = d.audit_id AND c.id <= boundary);
GET DIAGNOSTICS details_count = ROW_COUNT;
-- Keep ids locally because links must be deleted before their audit headers.
WITH removed AS (DELETE FROM public.worm_audit_chain WHERE id <= boundary RETURNING audit_id),
removed_audits AS (DELETE FROM "audit" WHERE id IN (SELECT audit_id FROM removed) RETURNING id)
SELECT (SELECT count(*) FROM removed), (SELECT count(*) FROM removed_audits) INTO links_count, audits_count;
PERFORM set_config('worm_audit.purging', 'off', true);
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome,
audits_deleted, details_deleted, links_deleted, anchor_link_id, anchor_hash)
VALUES(now(), session_user, p_cutoff, '6 years', 'PURGED', audits_count, details_count, links_count, boundary, anchor);
END;
$$;
-- changeset worm-audit:6
DROP TRIGGER IF EXISTS worm_audit_row_guard ON "audit";
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON "audit"
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON "audit";
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON "audit"
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
REVOKE UPDATE, DELETE, TRUNCATE ON "audit" FROM "app";
GRANT SELECT, INSERT ON "audit" TO "app";
DROP TRIGGER IF EXISTS worm_audit_row_guard ON "audit_detail";
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON "audit_detail"
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON "audit_detail";
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON "audit_detail"
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
REVOKE UPDATE, DELETE, TRUNCATE ON "audit_detail" FROM "app";
GRANT SELECT, INSERT ON "audit_detail" TO "app";
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_chain;
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_chain
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_chain;
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_chain
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_chain FROM "app";
GRANT SELECT, INSERT ON public.worm_audit_chain TO "app";
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_checkpoint;
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_checkpoint
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_checkpoint;
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_checkpoint
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_checkpoint FROM "app";
GRANT SELECT, INSERT ON public.worm_audit_checkpoint TO "app";
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_export;
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_export
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_export;
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_export
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_export FROM "app";
GRANT SELECT, INSERT ON public.worm_audit_export TO "app";
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_verification;
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_verification
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_verification;
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_verification
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_verification FROM "app";
GRANT SELECT, INSERT ON public.worm_audit_verification TO "app";
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_purge;
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_purge
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_purge;
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_purge
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_purge FROM "app";
GRANT SELECT, INSERT ON public.worm_audit_purge TO "app";
DROP TRIGGER IF EXISTS worm_audit_row_guard ON "audit_table";
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON "audit_table"
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_lookup_reject();
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON "audit_table";
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON "audit_table"
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
REVOKE UPDATE, DELETE, TRUNCATE ON "audit_table" FROM "app";
GRANT SELECT, INSERT ON "audit_table" TO "app";
DROP TRIGGER IF EXISTS worm_audit_row_guard ON "audit_user";
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON "audit_user"
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_lookup_reject();
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON "audit_user";
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON "audit_user"
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
REVOKE UPDATE, DELETE, TRUNCATE ON "audit_user" FROM "app";
GRANT SELECT, INSERT ON "audit_user" TO "app";
-- Lookups may refresh display metadata, but never identities or names.
GRANT UPDATE (label, create_date, modify_date) ON "audit_table" TO "app";
GRANT UPDATE (create_date, modify_date) ON "audit_user" TO "app";
REVOKE ALL ON FUNCTION public.worm_audit_purge(timestamptz) FROM PUBLIC;
REVOKE ALL ON FUNCTION public.worm_audit_purge(timestamptz) FROM "app";
GRANT EXECUTE ON FUNCTION public.worm_audit_purge(timestamptz) TO worm_audit_purger;
-- changeset worm-audit:7 splitStatements:false
DO $$
DECLARE
table_name text;
sequence_name text;
BEGIN
FOREACH table_name IN ARRAY ARRAY['"audit"', '"audit_detail"', '"audit_table"', '"audit_user"',
'public.worm_audit_checkpoint', 'public.worm_audit_export', 'public.worm_audit_verification', 'public.worm_audit_purge'] LOOP
sequence_name := pg_get_serial_sequence(table_name, 'id');
IF sequence_name IS NOT NULL THEN
EXECUTE format('GRANT USAGE ON SEQUENCE %s TO "app"', sequence_name);
END IF;
END LOOP;
END;
$$;
-- changeset worm-audit:8 splitStatements:false
CREATE OR REPLACE FUNCTION public.worm_audit_detail_coverage() RETURNS trigger
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $$
BEGIN
-- Serialize even direct SQL appends; the GIN index makes overlap checks indexed.
PERFORM pg_advisory_xact_lock(86753090421);
IF NEW.detail_ids <> '' AND (
EXISTS (SELECT FROM unnest(string_to_array(NEW.detail_ids, ',')) AS d(id) GROUP BY d.id HAVING count(*) > 1)
OR EXISTS (SELECT FROM public.worm_audit_chain c
WHERE string_to_array(c.detail_ids, ',') && string_to_array(NEW.detail_ids, ','))) THEN
RAISE EXCEPTION USING ERRCODE = '23505', MESSAGE = 'worm_audit: audit detail is already covered by the chain';
END IF;
RETURN NEW;
END;
$$;
-- changeset worm-audit:9
CREATE INDEX IF NOT EXISTS worm_audit_chain_detail_coverage ON public.worm_audit_chain USING gin (string_to_array(detail_ids, ','));
DROP TRIGGER IF EXISTS worm_audit_detail_coverage_guard ON public.worm_audit_chain;
CREATE TRIGGER worm_audit_detail_coverage_guard BEFORE INSERT ON public.worm_audit_chain
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_detail_coverage();
Vendored Executable
+295
View File
@@ -0,0 +1,295 @@
#!/bin/sh
# ----------------------------------------------------------------------------
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# ----------------------------------------------------------------------------
# ----------------------------------------------------------------------------
# Apache Maven Wrapper startup batch script, version 3.3.4
#
# Optional ENV vars
# -----------------
# JAVA_HOME - location of a JDK home dir, required when download maven via java source
# MVNW_REPOURL - repo url base for downloading maven distribution
# MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven
# MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output
# ----------------------------------------------------------------------------
set -euf
[ "${MVNW_VERBOSE-}" != debug ] || set -x
# OS specific support.
native_path() { printf %s\\n "$1"; }
case "$(uname)" in
CYGWIN* | MINGW*)
[ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")"
native_path() { cygpath --path --windows "$1"; }
;;
esac
# set JAVACMD and JAVACCMD
set_java_home() {
# For Cygwin and MinGW, ensure paths are in Unix format before anything is touched
if [ -n "${JAVA_HOME-}" ]; then
if [ -x "$JAVA_HOME/jre/sh/java" ]; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD="$JAVA_HOME/jre/sh/java"
JAVACCMD="$JAVA_HOME/jre/sh/javac"
else
JAVACMD="$JAVA_HOME/bin/java"
JAVACCMD="$JAVA_HOME/bin/javac"
if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then
echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2
echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2
return 1
fi
fi
else
JAVACMD="$(
'set' +e
'unset' -f command 2>/dev/null
'command' -v java
)" || :
JAVACCMD="$(
'set' +e
'unset' -f command 2>/dev/null
'command' -v javac
)" || :
if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then
echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2
return 1
fi
fi
}
# hash string like Java String::hashCode
hash_string() {
str="${1:-}" h=0
while [ -n "$str" ]; do
char="${str%"${str#?}"}"
h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296))
str="${str#?}"
done
printf %x\\n $h
}
verbose() { :; }
[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; }
die() {
printf %s\\n "$1" >&2
exit 1
}
trim() {
# MWRAPPER-139:
# Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds.
# Needed for removing poorly interpreted newline sequences when running in more
# exotic environments such as mingw bash on Windows.
printf "%s" "${1}" | tr -d '[:space:]'
}
scriptDir="$(dirname "$0")"
scriptName="$(basename "$0")"
# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties
while IFS="=" read -r key value; do
case "${key-}" in
distributionUrl) distributionUrl=$(trim "${value-}") ;;
distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;;
esac
done <"$scriptDir/.mvn/wrapper/maven-wrapper.properties"
[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties"
case "${distributionUrl##*/}" in
maven-mvnd-*bin.*)
MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/
case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in
*AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;;
:Darwin*x86_64) distributionPlatform=darwin-amd64 ;;
:Darwin*arm64) distributionPlatform=darwin-aarch64 ;;
:Linux*x86_64*) distributionPlatform=linux-amd64 ;;
*)
echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2
distributionPlatform=linux-amd64
;;
esac
distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip"
;;
maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;;
*) MVN_CMD="mvn${scriptName#mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;;
esac
# apply MVNW_REPOURL and calculate MAVEN_HOME
# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-<version>,maven-mvnd-<version>-<platform>}/<hash>
[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}"
distributionUrlName="${distributionUrl##*/}"
distributionUrlNameMain="${distributionUrlName%.*}"
distributionUrlNameMain="${distributionUrlNameMain%-bin}"
MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}"
MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")"
exec_maven() {
unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || :
exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD"
}
if [ -d "$MAVEN_HOME" ]; then
verbose "found existing MAVEN_HOME at $MAVEN_HOME"
exec_maven "$@"
fi
case "${distributionUrl-}" in
*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;;
*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;;
esac
# prepare tmp dir
if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then
clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; }
trap clean HUP INT TERM EXIT
else
die "cannot create temp dir"
fi
mkdir -p -- "${MAVEN_HOME%/*}"
# Download and Install Apache Maven
verbose "Couldn't find MAVEN_HOME, downloading and installing it ..."
verbose "Downloading from: $distributionUrl"
verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName"
# select .zip or .tar.gz
if ! command -v unzip >/dev/null; then
distributionUrl="${distributionUrl%.zip}.tar.gz"
distributionUrlName="${distributionUrl##*/}"
fi
# verbose opt
__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR=''
[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v
# normalize http auth
case "${MVNW_PASSWORD:+has-password}" in
'') MVNW_USERNAME='' MVNW_PASSWORD='' ;;
has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;;
esac
if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then
verbose "Found wget ... using wget"
wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl"
elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then
verbose "Found curl ... using curl"
curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl"
elif set_java_home; then
verbose "Falling back to use Java to download"
javaSource="$TMP_DOWNLOAD_DIR/Downloader.java"
targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName"
cat >"$javaSource" <<-END
public class Downloader extends java.net.Authenticator
{
protected java.net.PasswordAuthentication getPasswordAuthentication()
{
return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() );
}
public static void main( String[] args ) throws Exception
{
setDefault( new Downloader() );
java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() );
}
}
END
# For Cygwin/MinGW, switch paths to Windows format before running javac and java
verbose " - Compiling Downloader.java ..."
"$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java"
verbose " - Running Downloader.java ..."
"$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")"
fi
# If specified, validate the SHA-256 sum of the Maven distribution zip file
if [ -n "${distributionSha256Sum-}" ]; then
distributionSha256Result=false
if [ "$MVN_CMD" = mvnd.sh ]; then
echo "Checksum validation is not supported for maven-mvnd." >&2
echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2
exit 1
elif command -v sha256sum >/dev/null; then
if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c - >/dev/null 2>&1; then
distributionSha256Result=true
fi
elif command -v shasum >/dev/null; then
if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then
distributionSha256Result=true
fi
else
echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2
echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2
exit 1
fi
if [ $distributionSha256Result = false ]; then
echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2
echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2
exit 1
fi
fi
# unzip and move
if command -v unzip >/dev/null; then
unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip"
else
tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar"
fi
# Find the actual extracted directory name (handles snapshots where filename != directory name)
actualDistributionDir=""
# First try the expected directory name (for regular distributions)
if [ -d "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" ]; then
if [ -f "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/bin/$MVN_CMD" ]; then
actualDistributionDir="$distributionUrlNameMain"
fi
fi
# If not found, search for any directory with the Maven executable (for snapshots)
if [ -z "$actualDistributionDir" ]; then
# enable globbing to iterate over items
set +f
for dir in "$TMP_DOWNLOAD_DIR"/*; do
if [ -d "$dir" ]; then
if [ -f "$dir/bin/$MVN_CMD" ]; then
actualDistributionDir="$(basename "$dir")"
break
fi
fi
done
set -f
fi
if [ -z "$actualDistributionDir" ]; then
verbose "Contents of $TMP_DOWNLOAD_DIR:"
verbose "$(ls -la "$TMP_DOWNLOAD_DIR")"
die "Could not find Maven distribution directory in extracted archive"
fi
verbose "Found extracted Maven distribution directory: $actualDistributionDir"
printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$actualDistributionDir/mvnw.url"
mv -- "$TMP_DOWNLOAD_DIR/$actualDistributionDir" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME"
clean || :
exec_maven "$@"
Vendored
+189
View File
@@ -0,0 +1,189 @@
<# : batch portion
@REM ----------------------------------------------------------------------------
@REM Licensed to the Apache Software Foundation (ASF) under one
@REM or more contributor license agreements. See the NOTICE file
@REM distributed with this work for additional information
@REM regarding copyright ownership. The ASF licenses this file
@REM to you under the Apache License, Version 2.0 (the
@REM "License"); you may not use this file except in compliance
@REM with the License. You may obtain a copy of the License at
@REM
@REM http://www.apache.org/licenses/LICENSE-2.0
@REM
@REM Unless required by applicable law or agreed to in writing,
@REM software distributed under the License is distributed on an
@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@REM KIND, either express or implied. See the License for the
@REM specific language governing permissions and limitations
@REM under the License.
@REM ----------------------------------------------------------------------------
@REM ----------------------------------------------------------------------------
@REM Apache Maven Wrapper startup batch script, version 3.3.4
@REM
@REM Optional ENV vars
@REM MVNW_REPOURL - repo url base for downloading maven distribution
@REM MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven
@REM MVNW_VERBOSE - true: enable verbose log; others: silence the output
@REM ----------------------------------------------------------------------------
@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0)
@SET __MVNW_CMD__=
@SET __MVNW_ERROR__=
@SET __MVNW_PSMODULEP_SAVE=%PSModulePath%
@SET PSModulePath=
@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @(
IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B)
)
@SET PSModulePath=%__MVNW_PSMODULEP_SAVE%
@SET __MVNW_PSMODULEP_SAVE=
@SET __MVNW_ARG0_NAME__=
@SET MVNW_USERNAME=
@SET MVNW_PASSWORD=
@IF NOT "%__MVNW_CMD__%"=="" ("%__MVNW_CMD__%" %*)
@echo Cannot start maven from wrapper >&2 && exit /b 1
@GOTO :EOF
: end batch / begin powershell #>
$ErrorActionPreference = "Stop"
if ($env:MVNW_VERBOSE -eq "true") {
$VerbosePreference = "Continue"
}
# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties
$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl
if (!$distributionUrl) {
Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties"
}
switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) {
"maven-mvnd-*" {
$USE_MVND = $true
$distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip"
$MVN_CMD = "mvnd.cmd"
break
}
default {
$USE_MVND = $false
$MVN_CMD = $script -replace '^mvnw','mvn'
break
}
}
# apply MVNW_REPOURL and calculate MAVEN_HOME
# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-<version>,maven-mvnd-<version>-<platform>}/<hash>
if ($env:MVNW_REPOURL) {
$MVNW_REPO_PATTERN = if ($USE_MVND -eq $False) { "/org/apache/maven/" } else { "/maven/mvnd/" }
$distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace "^.*$MVNW_REPO_PATTERN",'')"
}
$distributionUrlName = $distributionUrl -replace '^.*/',''
$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$',''
$MAVEN_M2_PATH = "$HOME/.m2"
if ($env:MAVEN_USER_HOME) {
$MAVEN_M2_PATH = "$env:MAVEN_USER_HOME"
}
if (-not (Test-Path -Path $MAVEN_M2_PATH)) {
New-Item -Path $MAVEN_M2_PATH -ItemType Directory | Out-Null
}
$MAVEN_WRAPPER_DISTS = $null
if ((Get-Item $MAVEN_M2_PATH).Target[0] -eq $null) {
$MAVEN_WRAPPER_DISTS = "$MAVEN_M2_PATH/wrapper/dists"
} else {
$MAVEN_WRAPPER_DISTS = (Get-Item $MAVEN_M2_PATH).Target[0] + "/wrapper/dists"
}
$MAVEN_HOME_PARENT = "$MAVEN_WRAPPER_DISTS/$distributionUrlNameMain"
$MAVEN_HOME_NAME = ([System.Security.Cryptography.SHA256]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join ''
$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME"
if (Test-Path -Path "$MAVEN_HOME" -PathType Container) {
Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME"
Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD"
exit $?
}
if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) {
Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl"
}
# prepare tmp dir
$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile
$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir"
$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null
trap {
if ($TMP_DOWNLOAD_DIR.Exists) {
try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null }
catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" }
}
}
New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null
# Download and Install Apache Maven
Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..."
Write-Verbose "Downloading from: $distributionUrl"
Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName"
$webclient = New-Object System.Net.WebClient
if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) {
$webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD)
}
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null
# If specified, validate the SHA-256 sum of the Maven distribution zip file
$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum
if ($distributionSha256Sum) {
if ($USE_MVND) {
Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties."
}
Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash
if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) {
Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property."
}
}
# unzip and move
Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null
# Find the actual extracted directory name (handles snapshots where filename != directory name)
$actualDistributionDir = ""
# First try the expected directory name (for regular distributions)
$expectedPath = Join-Path "$TMP_DOWNLOAD_DIR" "$distributionUrlNameMain"
$expectedMvnPath = Join-Path "$expectedPath" "bin/$MVN_CMD"
if ((Test-Path -Path $expectedPath -PathType Container) -and (Test-Path -Path $expectedMvnPath -PathType Leaf)) {
$actualDistributionDir = $distributionUrlNameMain
}
# If not found, search for any directory with the Maven executable (for snapshots)
if (!$actualDistributionDir) {
Get-ChildItem -Path "$TMP_DOWNLOAD_DIR" -Directory | ForEach-Object {
$testPath = Join-Path $_.FullName "bin/$MVN_CMD"
if (Test-Path -Path $testPath -PathType Leaf) {
$actualDistributionDir = $_.Name
}
}
}
if (!$actualDistributionDir) {
Write-Error "Could not find Maven distribution directory in extracted archive"
}
Write-Verbose "Found extracted Maven distribution directory: $actualDistributionDir"
Rename-Item -Path "$TMP_DOWNLOAD_DIR/$actualDistributionDir" -NewName $MAVEN_HOME_NAME | Out-Null
try {
Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null
} catch {
if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) {
Write-Error "fail to move MAVEN_HOME"
}
} finally {
try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null }
catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" }
}
Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD"
+209 -23
View File
@@ -1,42 +1,137 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<!--
~ QQQ - Low-code Application Framework for Engineers.
~ Copyright (C) 2021-2026. Kingsrook, LLC
~ 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
~ contact@kingsrook.com
~ https://github.com/Kingsrook/
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ https://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<groupId>com.kingsrook.qbits</groupId>
<artifactId>qbit-example-extension</artifactId>
<version>0.1.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>QBit Example Extension</name>
<description>Example Extension QBit - rename this for your QBit</description>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<properties>
<revision>0.1.0-SNAPSHOT</revision>
<maven.compiler.source>21</maven.compiler.source>
<maven.compiler.target>21</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<qqq.version>0.35.0</qqq.version>
<jacoco.version>0.8.15</jacoco.version>
<h2.version>2.2.224</h2.version>
<testcontainers.version>2.0.5</testcontainers.version>
</properties>
<parent>
<groupId>com.kingsrook</groupId>
<artifactId>qbit-build-parent</artifactId>
<version>2.0.0</version>
</parent>
<groupId>com.kingsrook.qbits</groupId>
<artifactId>qbit-worm-audit</artifactId>
<version>${revision}</version>
<packaging>jar</packaging>
<name>QBit WORM Audit</name>
<description>QBit for write-once (WORM) audit in QQQ applications: database-enforced append-only audit tables, hash-chained tamper evidence with signed checkpoints, verification, and sealed exports to S3 Object Lock</description>
<url>https://github.com/QRun-IO/qbit-worm-audit</url>
<modelVersion>4.0.0</modelVersion>
<licenses>
<license>
<name>Apache License, Version 2.0</name>
<url>https://www.apache.org/licenses/LICENSE-2.0</url>
<distribution>repo</distribution>
</license>
</licenses>
<developers>
<developer>
<id>kingsrook</id>
<name>Kingsrook Team</name>
<email>contact@kingsrook.com</email>
<organization>Kingsrook, LLC</organization>
<organizationUrl>https://kingsrook.com</organizationUrl>
</developer>
</developers>
<scm>
<connection>scm:git:git://github.com/QRun-IO/qbit-worm-audit.git</connection>
<developerConnection>scm:git:ssh://github.com:QRun-IO/qbit-worm-audit.git</developerConnection>
<url>https://github.com/QRun-IO/qbit-worm-audit</url>
<tag>HEAD</tag>
</scm>
<dependencies>
<dependency>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-backend-core</artifactId>
<version>${qqq.version}</version>
</dependency>
<!-- The memory backend is included in qqq-backend-core. -->
<dependency>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-backend-module-rdbms</artifactId>
</dependency>
<dependency>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-middleware-javalin</artifactId>
</dependency>
<dependency>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-frontend-next</artifactId>
<version>1.0.0-RC.14</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<version>${h2.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>42.7.14</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers-postgresql</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers-minio</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers-junit-jupiter</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<version>5.10.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<version>3.24.2</version>
<version>3.27.7</version>
<scope>test</scope>
</dependency>
</dependencies>
@@ -45,14 +140,105 @@
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.11.0</version>
<artifactId>maven-checkstyle-plugin</artifactId>
<version>3.3.1</version>
<dependencies>
<dependency>
<groupId>com.puppycrawl.tools</groupId>
<artifactId>checkstyle</artifactId>
<version>10.12.6</version>
</dependency>
</dependencies>
<configuration>
<configLocation>checkstyle/config.xml</configLocation>
<headerLocation>checkstyle/license.txt</headerLocation>
<encoding>UTF-8</encoding>
<consoleOutput>true</consoleOutput>
<failsOnError>false</failsOnError>
<failOnViolation>true</failOnViolation>
<violationSeverity>warning</violationSeverity>
<includeTestSourceDirectory>false</includeTestSourceDirectory>
</configuration>
<executions>
<execution>
<id>validate</id>
<phase>validate</phase>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>3.1.2</version>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>${jacoco.version}</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>verify</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>check</id>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.70</minimum>
</limit>
<limit>
<counter>CLASS</counter>
<value>COVEREDRATIO</value>
<minimum>0.90</minimum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
<profiles>
<!-- QQQ 4.1 APIs until qbit-build-parent imports the 4.1 BOM (as in qbit-quick-search and qbit-cdn, ADR-0007). -->
<profile>
<id>qqq-snapshot</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<!-- Latest QQQ RC on Maven Central (2026-10-13). -->
<qqq.snapshot.version>4.1.0-RC.5</qqq.snapshot.version>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-bom-pom</artifactId>
<version>${qqq.snapshot.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
</profile>
</profiles>
</project>
@@ -1,75 +0,0 @@
/*******************************************************************************
** Configuration for the Example Extension QBit.
*******************************************************************************/
package com.kingsrook.qbits.example;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig;
/*******************************************************************************
** Configuration class for the Example Extension QBit.
*******************************************************************************/
public class ExampleExtensionQBitConfig implements QBitConfig
{
private String targetTableName;
private Boolean enableFeatureX = true;
/*******************************************************************************
** Validate configuration before QBit is produced.
*******************************************************************************/
@Override
public void validate(QInstance qInstance, List<String> errors)
{
if(targetTableName != null && qInstance.getTable(targetTableName) == null)
{
errors.add("Target table not found: " + targetTableName);
}
}
/*******************************************************************************
** Getter for targetTableName
*******************************************************************************/
public String getTargetTableName()
{
return targetTableName;
}
/*******************************************************************************
** Fluent setter for targetTableName
*******************************************************************************/
public ExampleExtensionQBitConfig withTargetTableName(String targetTableName)
{
this.targetTableName = targetTableName;
return this;
}
/*******************************************************************************
** Getter for enableFeatureX
*******************************************************************************/
public Boolean getEnableFeatureX()
{
return enableFeatureX;
}
/*******************************************************************************
** Fluent setter for enableFeatureX
*******************************************************************************/
public ExampleExtensionQBitConfig withEnableFeatureX(Boolean enableFeatureX)
{
this.enableFeatureX = enableFeatureX;
return this;
}
}
@@ -1,66 +0,0 @@
/*******************************************************************************
** Producer for the Example Extension QBit.
*******************************************************************************/
package com.kingsrook.qbits.example;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitProducer;
/*******************************************************************************
** Producer class for the Example Extension QBit.
*******************************************************************************/
public class ExampleExtensionQBitProducer implements QBitProducer
{
public static final String GROUP_ID = "com.kingsrook.qbits";
public static final String ARTIFACT_ID = "qbit-example-extension";
public static final String VERSION = "0.1.0";
private ExampleExtensionQBitConfig config;
/*******************************************************************************
** Produce this QBit into the given QInstance.
*******************************************************************************/
@Override
public void produce(QInstance qInstance, String namespace) throws QException
{
QBitMetaData qBitMetaData = new QBitMetaData()
.withGroupId(GROUP_ID)
.withArtifactId(ARTIFACT_ID)
.withVersion(VERSION)
.withNamespace(namespace)
.withConfig(config);
qInstance.addQBit(qBitMetaData);
if(config != null && config.getTargetTableName() != null)
{
applyTableCustomizations(qInstance, config.getTargetTableName());
}
}
/*******************************************************************************
** Apply customizations to the target table.
*******************************************************************************/
private void applyTableCustomizations(QInstance qInstance, String tableName)
{
// Implement your extension logic here
}
/*******************************************************************************
** Fluent setter for config
*******************************************************************************/
public ExampleExtensionQBitProducer withConfig(ExampleExtensionQBitConfig config)
{
this.config = config;
return this;
}
}
@@ -1,23 +0,0 @@
/*******************************************************************************
** Example table customizer.
*******************************************************************************/
package com.kingsrook.qbits.example.customizers;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
/*******************************************************************************
** Example table customizer that adds behavior to existing tables.
*******************************************************************************/
public class ExampleTableCustomizer
{
/*******************************************************************************
** Apply customizations to the given table.
*******************************************************************************/
public void customize(QTableMetaData table)
{
// Example: Add audit tracking fields
// table.withField(new QFieldMetaData("createdAt", QFieldType.DATE_TIME));
}
}
@@ -0,0 +1,361 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit;
import java.security.PrivateKey;
import java.security.PublicKey;
import java.util.Map;
import java.util.function.Supplier;
import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
/*******************************************************************************
** Host configuration. Private signing material never enters metadata records.
*******************************************************************************/
public class WormAuditQBitConfig implements QBitConfig
{
public static final int DEFAULT_RETENTION_YEARS = 6;
/*******************************************************************************
** Signing identity; the matching public key must be pinned in trustedPublicKeys.
*******************************************************************************/
public record SigningKey(String keyId, PrivateKey privateKey)
{
}
private String backendName = null;
private int retentionYears = 6;
private Supplier<SigningKey> signingKey = null;
private Map<String, PublicKey> trustedPublicKeys = Map.of();
private int chainIntervalSeconds = 30;
private int settleHorizonSeconds = 60;
private int batchSize = 1000;
private int lateDetailLookbackHours = 24;
private int checkpointEveryLinks = 10000;
private int checkpointEveryMinutes = 60;
private String verifyCron = "0 0 3 * * ?";
private String schedulerName = "worm_audit";
private Map<String, String> coreAuditTableNames = Map.of();
private String appRole = "app";
private MetaDataCustomizerInterface<QTableMetaData> tableMetaDataCustomizer = null;
/*******************************************************************************
** Read backendName.
*******************************************************************************/
public String getBackendName()
{
return (backendName);
}
/*******************************************************************************
** Configure backendName.
*******************************************************************************/
public WormAuditQBitConfig withBackendName(String backendName)
{
this.backendName = backendName;
return (this);
}
/*******************************************************************************
** Read retentionYears.
*******************************************************************************/
public int getRetentionYears()
{
return (retentionYears);
}
/*******************************************************************************
** Configure retentionYears.
*******************************************************************************/
public WormAuditQBitConfig withRetentionYears(int retentionYears)
{
this.retentionYears = retentionYears;
return (this);
}
/*******************************************************************************
** Read signingKey.
*******************************************************************************/
public Supplier<SigningKey> getSigningKey()
{
return (signingKey);
}
/*******************************************************************************
** Configure signingKey.
*******************************************************************************/
public WormAuditQBitConfig withSigningKey(Supplier<SigningKey> signingKey)
{
this.signingKey = signingKey;
return (this);
}
/*******************************************************************************
** Read trustedPublicKeys.
*******************************************************************************/
public Map<String, PublicKey> getTrustedPublicKeys()
{
return (trustedPublicKeys);
}
/*******************************************************************************
** Configure trustedPublicKeys.
*******************************************************************************/
public WormAuditQBitConfig withTrustedPublicKeys(Map<String, PublicKey> trustedPublicKeys)
{
this.trustedPublicKeys = trustedPublicKeys;
return (this);
}
/*******************************************************************************
** Read chainIntervalSeconds.
*******************************************************************************/
public int getChainIntervalSeconds()
{
return (chainIntervalSeconds);
}
/*******************************************************************************
** Configure chainIntervalSeconds.
*******************************************************************************/
public WormAuditQBitConfig withChainIntervalSeconds(int chainIntervalSeconds)
{
this.chainIntervalSeconds = chainIntervalSeconds;
return (this);
}
/*******************************************************************************
** Read settleHorizonSeconds.
*******************************************************************************/
public int getSettleHorizonSeconds()
{
return (settleHorizonSeconds);
}
/*******************************************************************************
** Configure settleHorizonSeconds.
*******************************************************************************/
public WormAuditQBitConfig withSettleHorizonSeconds(int settleHorizonSeconds)
{
this.settleHorizonSeconds = settleHorizonSeconds;
return (this);
}
/*******************************************************************************
** Read the bounded late-detail polling window in hours.
*******************************************************************************/
public int getLateDetailLookbackHours()
{
return (lateDetailLookbackHours);
}
/*******************************************************************************
** Configure late-detail polling; older omissions are reported by verification.
*******************************************************************************/
public WormAuditQBitConfig withLateDetailLookbackHours(int lateDetailLookbackHours)
{
this.lateDetailLookbackHours = lateDetailLookbackHours;
return (this);
}
/*******************************************************************************
** Read batchSize.
*******************************************************************************/
public int getBatchSize()
{
return (batchSize);
}
/*******************************************************************************
** Configure batchSize.
*******************************************************************************/
public WormAuditQBitConfig withBatchSize(int batchSize)
{
this.batchSize = batchSize;
return (this);
}
/*******************************************************************************
** Read checkpointEveryLinks.
*******************************************************************************/
public int getCheckpointEveryLinks()
{
return (checkpointEveryLinks);
}
/*******************************************************************************
** Configure checkpointEveryLinks.
*******************************************************************************/
public WormAuditQBitConfig withCheckpointEveryLinks(int checkpointEveryLinks)
{
this.checkpointEveryLinks = checkpointEveryLinks;
return (this);
}
/*******************************************************************************
** Read checkpointEveryMinutes.
*******************************************************************************/
public int getCheckpointEveryMinutes()
{
return (checkpointEveryMinutes);
}
/*******************************************************************************
** Configure checkpointEveryMinutes.
*******************************************************************************/
public WormAuditQBitConfig withCheckpointEveryMinutes(int checkpointEveryMinutes)
{
this.checkpointEveryMinutes = checkpointEveryMinutes;
return (this);
}
/*******************************************************************************
** Read verifyCron.
*******************************************************************************/
public String getVerifyCron()
{
return (verifyCron);
}
/*******************************************************************************
** Configure verifyCron.
*******************************************************************************/
public WormAuditQBitConfig withVerifyCron(String verifyCron)
{
this.verifyCron = verifyCron;
return (this);
}
/*******************************************************************************
** Read schedulerName.
*******************************************************************************/
public String getSchedulerName()
{
return (schedulerName);
}
/*******************************************************************************
** Configure schedulerName.
*******************************************************************************/
public WormAuditQBitConfig withSchedulerName(String schedulerName)
{
this.schedulerName = schedulerName;
return (this);
}
/*******************************************************************************
** Read coreAuditTableNames.
*******************************************************************************/
public Map<String, String> getCoreAuditTableNames()
{
return (coreAuditTableNames);
}
/*******************************************************************************
** Configure coreAuditTableNames.
*******************************************************************************/
public WormAuditQBitConfig withCoreAuditTableNames(Map<String, String> coreAuditTableNames)
{
this.coreAuditTableNames = coreAuditTableNames;
return (this);
}
/*******************************************************************************
** Read appRole.
*******************************************************************************/
public String getAppRole()
{
return (appRole);
}
/*******************************************************************************
** Configure appRole.
*******************************************************************************/
public WormAuditQBitConfig withAppRole(String appRole)
{
this.appRole = appRole;
return (this);
}
/*******************************************************************************
** Read tableMetaDataCustomizer.
*******************************************************************************/
@Override
public MetaDataCustomizerInterface<QTableMetaData> getTableMetaDataCustomizer()
{
return (tableMetaDataCustomizer);
}
/*******************************************************************************
** Configure tableMetaDataCustomizer.
*******************************************************************************/
public WormAuditQBitConfig withTableMetaDataCustomizer(MetaDataCustomizerInterface<QTableMetaData> tableMetaDataCustomizer)
{
this.tableMetaDataCustomizer = tableMetaDataCustomizer;
return (this);
}
/*******************************************************************************
** Reject invalid runtime limits before producing metadata.
*******************************************************************************/
public void validate()
{
if(backendName == null || backendName.isBlank() || retentionYears < 1 || chainIntervalSeconds < 1
|| lateDetailLookbackHours < 1 || settleHorizonSeconds < 0 || batchSize < 1 || checkpointEveryLinks < 1 || checkpointEveryMinutes < 1)
{
throw (new IllegalArgumentException("backendName is required and WORM audit limits must be positive (settle horizon may be zero)"));
}
}
}
@@ -0,0 +1,213 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit;
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
import com.kingsrook.qbits.wormaudit.guard.AppendOnlyGuard;
import com.kingsrook.qbits.wormaudit.model.LinkRangeWidget;
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
import com.kingsrook.qbits.wormaudit.processes.WormAuditProcesses;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaDataProducer;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.quartz.QuartzSchedulerMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
/*******************************************************************************
** Produces append-only audit metadata, verification actions and schedules.
*******************************************************************************/
public class WormAuditQBitProducer implements QBitMetaDataProducer<WormAuditQBitConfig>
{
public static final String GROUP_ID = "com.kingsrook.qbits";
public static final String ARTIFACT_ID = "qbit-worm-audit";
public static final String VERSION = "0.1.0";
private WormAuditQBitConfig qBitConfig;
private String namespace;
/*******************************************************************************
** Fluent setter for the config.
*******************************************************************************/
public WormAuditQBitProducer withQBitConfig(WormAuditQBitConfig config)
{
this.qBitConfig = config;
return (this);
}
/*******************************************************************************
** Getter for the config.
*******************************************************************************/
@Override
public WormAuditQBitConfig getQBitConfig()
{
return (qBitConfig);
}
/*******************************************************************************
** Fluent setter for the namespace.
*******************************************************************************/
public WormAuditQBitProducer withNamespace(String namespace)
{
this.namespace = namespace;
return (this);
}
/*******************************************************************************
** Getter for the namespace.
*******************************************************************************/
@Override
public String getNamespace()
{
return (namespace);
}
/*******************************************************************************
** Identity and configuration of this qbit.
*******************************************************************************/
@Override
public QBitMetaData getQBitMetaData()
{
return (new QBitMetaData().withGroupId(GROUP_ID).withArtifactId(ARTIFACT_ID).withVersion(VERSION).withNamespace(namespace).withConfig(qBitConfig));
}
/*******************************************************************************
** Package-scanned producers have no config: skip them.
*******************************************************************************/
@Override
public boolean isEnabled()
{
return (qBitConfig != null);
}
/*******************************************************************************
** Register phase (a) tables, processes and schedules on the host backend.
*******************************************************************************/
@Override
public MetaDataProducerMultiOutput produce(QInstance qInstance) throws QException
{
qBitConfig.validate();
if(qInstance.getBackend(qBitConfig.getBackendName()) == null)
{
throw (new IllegalArgumentException("Configured audit backend is missing"));
}
WormAuditDdl.resolveNames(qBitConfig, qInstance);
for(String core : AppendOnlyGuard.CORE_TABLES)
{
if(!qBitConfig.getBackendName().equals(qInstance.getTable(core).getBackendName()))
{
throw (new IllegalArgumentException("Core and WORM audit tables must share one backend"));
}
}
QBitMetaData qBit = getQBitMetaData();
MetaDataProducerMultiOutput output = new MetaDataProducerMultiOutput().with(qBit);
AppendOnlyGuard guard = new AppendOnlyGuard();
for(QTableMetaData table : WormAuditTables.tables(qBitConfig))
{
if(qBitConfig.getTableMetaDataCustomizer() != null)
{
table = qBitConfig.getTableMetaDataCustomizer().customizeMetaData(qInstance, table);
}
guard.customizeMetaData(qInstance, table);
table.setSourceQBitName(qBit.getName());
output.add(table);
}
for(String core : AppendOnlyGuard.CORE_TABLES)
{
guard.customizeMetaData(qInstance, qInstance.getTable(core));
}
for(String name : WormAuditTables.ALL)
{
output.add(QPossibleValueSource.newForTable(name));
}
for(String name : java.util.List.of(WormAuditTables.CHECKPOINT, WormAuditTables.EXPORT, WormAuditTables.VERIFICATION))
{
output.add(new QWidgetMetaData().withName(name + "_links").withLabel("Chain links").withType("childRecordList")
.withCodeReference(new QCodeReference(LinkRangeWidget.class)).withDefaultValue("parentTable", name));
}
for(var process : WormAuditProcesses.processes(qBitConfig))
{
process.setSourceQBitName(qBit.getName());
output.add(process);
}
if(qInstance.getSchedulers() == null || !qInstance.getSchedulers().containsKey(qBitConfig.getSchedulerName()))
{
qInstance.addScheduler(new QuartzSchedulerMetaData().withName(qBitConfig.getSchedulerName()));
}
QAppMetaData app = new QAppMetaData().withName("worm_audit").withLabel("WORM Audit").withIcon(new QIcon("policy"));
for(String name : WormAuditTables.NAV)
{
QTableMetaData table = output.getEach(QTableMetaData.class).stream().filter(t -> t.getName().equals(name)).findFirst().orElseThrow();
app.withSectionOfChildren(new QAppSection().withName(name).withLabel(table.getLabel()), table);
}
output.add(app);
// RC.5 rejects underscores in declared alert type names. QAlerts still
// dispatches the specified worm_audit.chain.break signal to installed sinks.
output.addSelfToInstance(qInstance);
denyAssistant(qInstance);
return (new MetaDataProducerMultiOutput());
}
/*******************************************************************************
** RC.5 compatibility: replace with a direct call when the qbit builds on RC.6 (qqq#1141, #1163).
*******************************************************************************/
private void denyAssistant(QInstance instance) throws QException
{
try
{
Class<?> type = Class.forName("com.kingsrook.qqq.backend.core.model.metadata.permissions.ClientChannelsMetaData");
Object metadata = type.getMethod("ofOrWithNew", QInstance.class).invoke(null, instance);
type.getMethod("withDeniedPattern", String.class, String.class).invoke(metadata, "assistant", "worm_audit_*");
}
catch(ClassNotFoundException expected)
{
// The assistant wall API is absent in RC.5.
}
catch(ReflectiveOperationException e)
{
throw (new QException("Cannot register the WORM audit assistant deny pattern", e));
}
}
}
@@ -0,0 +1,74 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit.canonical;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
/*******************************************************************************
** v1 audit content, exactly as stored. Subsets use the same document schema for
** LATE_DETAIL links; no canonical content is persisted by this qbit.
*******************************************************************************/
public final class AuditCanonical
{
private static final DateTimeFormatter TIMESTAMP = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'").withZone(ZoneOffset.UTC);
/*******************************************************************************
** Utility class.
*******************************************************************************/
private AuditCanonical()
{
}
/*******************************************************************************
** Resolve names outside this function, without looking up application data.
*******************************************************************************/
public static byte[] bytes(QRecord audit, String table, String user, List<QRecord> details)
{
Map<String, Object> header = new LinkedHashMap<>();
header.put("clientChannel", audit.getValueString("clientChannel"));
header.put("id", audit.getValueLong("id"));
header.put("message", audit.getValueString("message"));
header.put("recordId", audit.getValueString("recordId"));
header.put("table", table);
header.put("timestamp", TIMESTAMP.format(audit.getValueInstant("timestamp")));
header.put("user", user);
List<Map<String, Object>> rows = details.stream().sorted(Comparator.comparing(r -> r.getValueLong("id"))).map(detail ->
{
Map<String, Object> row = new LinkedHashMap<>();
row.put("id", detail.getValueLong("id"));
for(String field : List.of("fieldName", "message", "newValue", "oldValue"))
{
row.put(field, detail.getValueString(field));
}
return (row);
}).toList();
return (Jcs.bytes(Map.of("audit", header, "details", rows, "v", 1)));
}
}
@@ -0,0 +1,130 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit.canonical;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
/*******************************************************************************
** RFC 8785 serialization of the audit schema: strings, integer longs, null,
** objects and arrays. Java String ordering is UTF-16 code-unit ordering. Reject
** malformed surrogate pairs instead of silently replacing them during UTF-8 encoding.
*******************************************************************************/
public final class Jcs
{
/*******************************************************************************
** Utility class.
*******************************************************************************/
private Jcs()
{
}
/*******************************************************************************
** Encode a document with no insignificant whitespace.
*******************************************************************************/
public static byte[] bytes(Object value)
{
return (string(value).getBytes(StandardCharsets.UTF_8));
}
/*******************************************************************************
** Serialize only the value types supported by the v1 schema.
*******************************************************************************/
public static String string(Object value)
{
if(value == null)
{
return ("null");
}
if(value instanceof String text)
{
return (quote(text));
}
if(value instanceof Long || value instanceof Integer)
{
return (value.toString());
}
if(value instanceof List<?> list)
{
return ("[" + String.join(",", list.stream().map(Jcs::string).toList()) + "]");
}
if(value instanceof Map<?, ?> map)
{
TreeMap<String, Object> sorted = new TreeMap<>();
map.forEach((key, item) -> sorted.put((String) key, item));
return ("{" + String.join(",", sorted.entrySet().stream().map(e -> quote(e.getKey()) + ":" + string(e.getValue())).toList()) + "}");
}
throw (new IllegalArgumentException("Unsupported canonical value type"));
}
/*******************************************************************************
** JSON escaping mandated by JCS, including lowercase control escapes.
*******************************************************************************/
private static String quote(String text)
{
StringBuilder out = new StringBuilder("\"");
for(int i = 0; i < text.length(); i++)
{
char c = text.charAt(i);
switch(c)
{
case '"' -> out.append("\\\"");
case '\\' -> out.append("\\\\");
case '\b' -> out.append("\\b");
case '\f' -> out.append("\\f");
case '\n' -> out.append("\\n");
case '\r' -> out.append("\\r");
case '\t' -> out.append("\\t");
default ->
{
if(c < 32)
{
out.append(String.format(java.util.Locale.ROOT, "\\u%04x", (int) c));
}
else if(Character.isHighSurrogate(c))
{
if(i + 1 >= text.length() || !Character.isLowSurrogate(text.charAt(i + 1)))
{
throw (new IllegalArgumentException("Unpaired surrogate in canonical string"));
}
out.append(c).append(text.charAt(++i));
}
else if(Character.isLowSurrogate(c))
{
throw (new IllegalArgumentException("Unpaired surrogate in canonical string"));
}
else
{
out.append(c);
}
}
}
}
return (out.append('"').toString());
}
}
@@ -0,0 +1,479 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit.chain;
import java.io.Serializable;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.Timestamp;
import java.sql.Types;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qbits.wormaudit.canonical.AuditCanonical;
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
import com.kingsrook.qbits.wormaudit.guard.AppendOnlyGuard;
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
import com.kingsrook.qbits.wormaudit.verify.VerificationResult;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSTransaction;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails;
/*******************************************************************************
** QQQ storage shared by chainer, verifier and future export code. All reads use
** the caller's transaction; only insert operations exist here. Bypass applies
** to internal catalog work after process permission checks, never user requests.
*******************************************************************************/
public class AuditStore
{
private final QBackendTransaction transaction;
private final Map<String, String> names = new LinkedHashMap<>(16, 0.75F, true);
private Boolean timestampWithZone;
/*******************************************************************************
** Bind operations to one snapshot/transaction, or the memory backend.
*******************************************************************************/
public AuditStore(QBackendTransaction transaction)
{
this.transaction = transaction;
}
/*******************************************************************************
** Read in deterministic id order, explicitly fetching TEXT fields.
*******************************************************************************/
public List<QRecord> query(String table, QQueryFilter filter) throws QException
{
QueryInput input = new QueryInput(table).withFilter(filter).withTransaction(transaction).withShouldFetchHeavyFields(true);
input.withPermissionBypass("WORM audit internal read");
return (new QueryAction().execute(input).getRecords());
}
/*******************************************************************************
** All records ordered by id.
*******************************************************************************/
public List<QRecord> all(String table) throws QException
{
return (query(table, new QQueryFilter().withOrderBy(new QFilterOrderBy("id", true))));
}
/*******************************************************************************
** An equality query.
*******************************************************************************/
public List<QRecord> where(String table, String field, Serializable value) throws QException
{
// Core lookup keys are INTEGER while audit and chain keys are LONG.
// Memory equality is type-sensitive; normalize to the declared field type.
if(value instanceof Number number && QContext.getQInstance().getTable(table).getField(field).getType()
== com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType.INTEGER)
{
value = Math.toIntExact(number.longValue());
}
return (query(table, new QQueryFilter(new QFilterCriteria(field, QCriteriaOperator.EQUALS, value)).withOrderBy(new QFilterOrderBy("id", true))));
}
/*******************************************************************************
** Return a row or null if a tamper removed it.
*******************************************************************************/
public QRecord get(String table, long id) throws QException
{
return (where(table, "id", id).stream().findFirst().orElse(null));
}
/*******************************************************************************
** Insert a batch, propagating failures so a transaction can roll back.
*******************************************************************************/
public List<QRecord> insert(String table, List<QRecord> rows) throws QException
{
if(rows.isEmpty())
{
return (rows);
}
try
{
if(WormAuditTables.ALL.contains(table) && QContext.getQInstance().getBackendForTable(table)
instanceof com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData)
{
// RC.5 binds Instant as a UTC LocalDateTime, which shifts timestamptz
// in non-UTC sessions. Its Timestamp path preserves the actual instant.
rows.forEach(row -> row.getValues().replaceAll((field, value) -> value instanceof Instant instant ? Timestamp.from(instant) : value));
}
InsertInput input = new InsertInput(table).withRecords(rows).withTransaction(transaction).withOmitDmlAudit(true);
input.withPermissionBypass("WORM audit internal append");
List<QRecord> inserted = new InsertAction().execute(input).getRecords();
for(QRecord row : inserted)
{
if(row.getErrors() != null && !row.getErrors().isEmpty())
{
// Never echo a rejected record value into logs or alerts.
throw (new QException("WORM audit append failed for " + table));
}
}
return (inserted);
}
catch(Exception e)
{
throw (AppendOnlyGuard.map(e));
}
}
/*******************************************************************************
** Parse explicit coverage; duplicates are retained so verification sees them.
*******************************************************************************/
public static List<Long> detailIds(QRecord link)
{
String ids = link.getValueString("detail_ids");
return (ids == null || ids.isEmpty() ? List.of() : java.util.Arrays.stream(ids.split(",")).map(Long::valueOf).toList());
}
/*******************************************************************************
** Resolve a lookup name without touching application tables.
*******************************************************************************/
public String name(String table, Long id) throws QException
{
if(id == null)
{
return (null);
}
String key = table + ":" + id;
if(names.containsKey(key))
{
return (names.get(key));
}
QRecord row = get(table, id);
if(row == null)
{
throw (new QException("Missing " + table + " " + id));
}
if(names.size() >= 2000)
{
names.remove(names.keySet().iterator().next());
}
names.put(key, row.getValueString("name"));
return (row.getValueString("name"));
}
/*******************************************************************************
** Canonical bytes for the exact detail ids recorded on a link.
*******************************************************************************/
public byte[] canonical(QRecord audit, List<Long> ids) throws QException
{
return (canonical(audit, ids, index(in("auditDetail", "id", ids))));
}
/*******************************************************************************
** Settled unchained headers. RDBMS uses an anti-join under the writer lock;
** memory uses QQQ queries and the same ordering and cutoff semantics.
*******************************************************************************/
public List<QRecord> unchained(Instant cutoff, int limit) throws QException
{
if(isRdbms())
{
String sql = "SELECT a." + column("audit", "id") + " FROM " + physical("audit") + " a WHERE a." + column("audit", "timestamp")
+ " < ? AND NOT EXISTS (SELECT 1 FROM worm_audit_chain c WHERE c.audit_id=a." + column("audit", "id")
+ " AND c.kind='AUDIT') ORDER BY a." + column("audit", "timestamp") + ", a." + column("audit", "id") + " LIMIT " + limit;
Map<Long, QRecord> rows = index(in("audit", "id", sqlIds(sql, cutoff)));
return (rows.values().stream().sorted(Comparator.comparing((QRecord r) -> r.getValueInstant("timestamp"))
.thenComparing(r -> r.getValueLong("id"))).toList());
}
var covered = all(WormAuditTables.CHAIN).stream().filter(r -> "AUDIT".equals(r.getValueString("kind"))).map(r -> r.getValueLong("audit_id")).collect(java.util.stream.Collectors.toSet());
return (all("audit").stream().filter(r -> r.getValueInstant("timestamp").isBefore(cutoff) && !covered.contains(r.getValueLong("id")))
.sorted(Comparator.comparing((QRecord r) -> r.getValueInstant("timestamp")).thenComparing(r -> r.getValueLong("id"))).limit(limit).toList());
}
/*******************************************************************************
** Latest successful retention anchor, or genesis.
*******************************************************************************/
public QRecord anchor() throws QException
{
List<QRecord> rows = query(WormAuditTables.PURGE, new QQueryFilter(new QFilterCriteria("outcome", QCriteriaOperator.EQUALS, "PURGED"))
.withOrderBy(new QFilterOrderBy("id", false)).withLimit(1));
return (rows.isEmpty() ? new QRecord().withValue("anchor_link_id", 0L).withValue("anchor_hash", Hashes.GENESIS) : rows.getFirst());
}
/*******************************************************************************
** Whether SQL anti-joins are available in this snapshot.
*******************************************************************************/
public boolean isRdbms()
{
return (transaction instanceof RDBMSTransaction);
}
/*******************************************************************************
** Fetch one keyset page, never using an offset or a whole-chain read.
*******************************************************************************/
public List<QRecord> page(String table, long after, int limit) throws QException
{
return (query(table, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, after))
.withOrderBy(new QFilterOrderBy("id", true)).withLimit(limit)));
}
/*******************************************************************************
** Fetch just the tail row.
*******************************************************************************/
public QRecord last(String table) throws QException
{
List<QRecord> rows = query(table, new QQueryFilter().withOrderBy(new QFilterOrderBy("id", false)).withLimit(1));
return (rows.isEmpty() ? null : rows.getFirst());
}
/*******************************************************************************
** One batched IN query, with memory-backend integer key normalization.
*******************************************************************************/
public List<QRecord> in(String table, String field, Collection<Long> ids) throws QException
{
if(ids.isEmpty())
{
return (List.of());
}
boolean integer = QContext.getQInstance().getTable(table).getField(field).getType()
== com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType.INTEGER;
List<Serializable> values = ids.stream().distinct().map(id -> integer ? (Serializable) Math.toIntExact(id) : id).toList();
return (query(table, new QQueryFilter(new QFilterCriteria(field, QCriteriaOperator.IN, values)).withOrderBy(new QFilterOrderBy("id", true))));
}
/*******************************************************************************
** Index one bounded batch by primary key.
*******************************************************************************/
public static Map<Long, QRecord> index(List<QRecord> rows)
{
return (rows.stream().collect(Collectors.toMap(r -> r.getValueLong("id"), r -> r)));
}
/*******************************************************************************
** Canonicalize already fetched content; a missing detail never triggers a query.
*******************************************************************************/
public byte[] canonical(QRecord audit, List<Long> ids, Map<Long, QRecord> details) throws QException
{
List<QRecord> rows = new ArrayList<>();
for(long id : ids)
{
QRecord detail = details.get(id);
if(detail == null || !audit.getValueLong("id").equals(detail.getValueLong("auditId")))
{
throw (new QException("audit_detail " + id + " is missing or belongs to another audit"));
}
rows.add(detail);
}
return (AuditCanonical.bytes(audit, name("auditTable", audit.getValueLong("auditTableId")), name("auditUser", audit.getValueLong("auditUserId")), rows));
}
/*******************************************************************************
** Poll uncovered details in a bounded time window; memory retains its simple fallback.
*******************************************************************************/
public List<QRecord> lateDetails(Instant cutoff, int hours, int limit) throws QException
{
if(isRdbms())
{
String sql = "SELECT d." + column("auditDetail", "id") + " FROM " + physical("auditDetail") + " d JOIN " + physical("audit")
+ " a ON a." + column("audit", "id") + "=d." + column("auditDetail", "auditId")
+ " WHERE a." + column("audit", "timestamp") + " < ? AND a." + column("audit", "timestamp") + " >= ?"
+ " AND EXISTS (SELECT 1 FROM worm_audit_chain h WHERE h.kind='AUDIT' AND h.audit_id=a." + column("audit", "id") + ")"
+ " AND NOT EXISTS (SELECT 1 FROM worm_audit_chain c WHERE " + containsDetail() + ") ORDER BY d." + column("auditDetail", "id") + " LIMIT " + limit;
return (in("auditDetail", "id", sqlIds(sql, cutoff, cutoff.minusSeconds(hours * 3600L))));
}
List<QRecord> links = all(WormAuditTables.CHAIN);
Set<Long> headers = links.stream().filter(r -> "AUDIT".equals(r.getValueString("kind"))).map(r -> r.getValueLong("audit_id")).collect(Collectors.toSet());
Set<Long> covered = new HashSet<>();
links.forEach(link -> covered.addAll(detailIds(link)));
Set<Long> settled = all("audit").stream().filter(a -> headers.contains(a.getValueLong("id")) && a.getValueInstant("timestamp").isBefore(cutoff))
.map(a -> a.getValueLong("id")).collect(Collectors.toSet());
return (all("auditDetail").stream().filter(d -> settled.contains(d.getValueLong("auditId")) && !covered.contains(d.getValueLong("id"))).toList());
}
/*******************************************************************************
** SQL completeness counts all offenders but returns only the first 100 of each type.
*******************************************************************************/
public void completeness(Instant cutoff, VerificationResult result) throws QException
{
String auditId = "a." + column("audit", "id");
String time = "a." + column("audit", "timestamp");
String headers = "SELECT " + auditId + " AS id, (SELECT count(*) FROM worm_audit_chain c WHERE c.kind='AUDIT' AND c.audit_id=" + auditId
+ ") AS coverage FROM " + physical("audit") + " a WHERE " + time + " < ? OR " + time + " IS NULL";
String details = "SELECT d." + column("auditDetail", "id") + " AS id, (SELECT count(*) FROM worm_audit_chain c WHERE " + containsDetail()
+ ") AS coverage FROM " + physical("auditDetail") + " d LEFT JOIN " + physical("audit") + " a ON " + auditId + "=d."
+ column("auditDetail", "auditId") + " WHERE " + time + " < ? OR " + time + " IS NULL";
reportCoverage(headers, "audit", cutoff, result);
reportCoverage(details, "audit_detail", cutoff, result);
}
/*******************************************************************************
** Keep exact offender ids and total counts without materializing the result set.
*******************************************************************************/
private void reportCoverage(String source, String table, Instant cutoff, VerificationResult result) throws QException
{
String sql = "SELECT id, coverage, count(*) OVER() AS total FROM (" + source + ") offenders WHERE coverage <> 1 ORDER BY id LIMIT 100";
try(PreparedStatement statement = ((RDBMSTransaction) transaction).getConnection().prepareStatement(sql))
{
bindCutoff(statement, 1, cutoff);
long total = 0;
int reported = 0;
try(ResultSet rows = statement.executeQuery())
{
while(rows.next())
{
long id = rows.getLong(1);
total = rows.getLong(3);
result.add(null, table.equals("audit") ? id : null, table + " " + com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier.number(id)
+ (rows.getLong(2) == 0 ? " is not covered by the chain" : " is covered twice or more by the chain"));
reported++;
}
}
result.omitted(total - reported);
if(total > reported)
{
result.warn(table + ": " + total + " coverage offenders; first 100 ids reported");
}
}
catch(Exception e)
{
throw (AppendOnlyGuard.map(e));
}
}
/*******************************************************************************
** PostgreSQL's containment expression matches the GIN expression index.
*******************************************************************************/
private String containsDetail() throws QException
{
try
{
String id = "d." + column("auditDetail", "id");
if("PostgreSQL".equals(((RDBMSTransaction) transaction).getConnection().getMetaData().getDatabaseProductName()))
{
return ("string_to_array(c.detail_ids, ',') @> ARRAY[" + id + "::text]");
}
return ("POSITION(',' || CAST(" + id + " AS varchar) || ',' IN ',' || c.detail_ids || ',') > 0");
}
catch(Exception e)
{
throw (AppendOnlyGuard.map(e));
}
}
/*******************************************************************************
** Read ids from an anti-join, binding all instants according to column type.
*******************************************************************************/
private List<Long> sqlIds(String sql, Instant... cutoffs) throws QException
{
try(PreparedStatement statement = ((RDBMSTransaction) transaction).getConnection().prepareStatement(sql))
{
for(int i = 0; i < cutoffs.length; i++)
{
bindCutoff(statement, i + 1, cutoffs[i]);
}
List<Long> ids = new ArrayList<>();
try(ResultSet rows = statement.executeQuery())
{
while(rows.next())
{
ids.add(rows.getLong(1));
}
}
return (ids);
}
catch(Exception e)
{
throw (AppendOnlyGuard.map(e));
}
}
/*******************************************************************************
** UTC local timestamps for without-tz columns; offset timestamps for timestamptz.
*******************************************************************************/
private void bindCutoff(PreparedStatement statement, int index, Instant cutoff) throws Exception
{
if(timestampWithZone == null)
{
try(var probe = ((RDBMSTransaction) transaction).getConnection().createStatement();
var rows = probe.executeQuery("SELECT " + column("audit", "timestamp") + " FROM " + physical("audit") + " WHERE 1=0"))
{
var metadata = rows.getMetaData();
timestampWithZone = metadata.getColumnType(1) == Types.TIMESTAMP_WITH_TIMEZONE
|| "timestamptz".equalsIgnoreCase(metadata.getColumnTypeName(1));
}
}
statement.setObject(index, timestampWithZone ? cutoff.atOffset(ZoneOffset.UTC) : LocalDateTime.ofInstant(cutoff, ZoneOffset.UTC));
}
/*******************************************************************************
** Resolve and quote physical core table names.
*******************************************************************************/
private String physical(String table)
{
return (WormAuditDdl.identifier(((RDBMSTableBackendDetails) QContext.getQInstance().getTable(table).getBackendDetails()).getTableName()));
}
/*******************************************************************************
** Resolve and quote physical field names for runtime queries.
*******************************************************************************/
private String column(String table, String field)
{
String name = QContext.getQInstance().getTable(table).getField(field).getBackendName();
return (WormAuditDdl.identifier(name == null ? field : name));
}
}
@@ -0,0 +1,99 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit.chain;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.HexFormat;
import java.util.List;
/*******************************************************************************
** SHA-256 primitives. Link predecessors are the 64 lowercase ASCII hex bytes;
** Merkle leaves are the decoded 32-byte link hashes, with odd nodes promoted.
*******************************************************************************/
public final class Hashes
{
public static final String GENESIS = "0".repeat(64);
/*******************************************************************************
** Utility class.
*******************************************************************************/
private Hashes()
{
}
/*******************************************************************************
** SHA-256 digest of concatenated byte arrays.
*******************************************************************************/
public static byte[] digest(byte[]... parts)
{
try
{
MessageDigest digest = MessageDigest.getInstance("SHA-256");
for(byte[] part : parts)
{
digest.update(part);
}
return (digest.digest());
}
catch(NoSuchAlgorithmException e)
{
throw (new IllegalStateException("JDK lacks SHA-256", e));
}
}
/*******************************************************************************
** Link hash over predecessor, NUL delimiter and canonical bytes.
*******************************************************************************/
public static String link(String previous, byte[] canonical)
{
return (HexFormat.of().formatHex(digest(previous.getBytes(StandardCharsets.US_ASCII), new byte[] { 0 }, canonical)));
}
/*******************************************************************************
** Binary Merkle root; an empty range cannot be checkpointed.
*******************************************************************************/
public static String merkle(List<String> hashes)
{
if(hashes.isEmpty())
{
throw (new IllegalArgumentException("A Merkle range must contain links"));
}
List<byte[]> level = hashes.stream().map(HexFormat.of()::parseHex).toList();
while(level.size() > 1)
{
List<byte[]> next = new ArrayList<>();
for(int i = 0; i < level.size(); i += 2)
{
next.add(i + 1 == level.size() ? level.get(i) : digest(level.get(i), level.get(i + 1)));
}
level = next;
}
return (HexFormat.of().formatHex(level.getFirst()));
}
}
@@ -0,0 +1,224 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit.chain;
import java.sql.ResultSet;
import java.sql.Statement;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
import com.kingsrook.qbits.wormaudit.checkpoint.Checkpoints;
import com.kingsrook.qbits.wormaudit.guard.AppendOnlyGuard;
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSTransaction;
/*******************************************************************************
** Single asynchronous writer. PostgreSQL uses a transaction-scoped advisory
** lock shared with the retention function. Other backends use a JVM-wide lock:
** those deployments must run their chainer in one JVM only.
*******************************************************************************/
public class WormAuditChainer
{
public static final long ADVISORY_LOCK = 86753090421L;
public static final ReentrantLock JVM_LOCK = new ReentrantLock();
private final WormAuditQBitConfig config;
/*******************************************************************************
** Counts describe committed work; a contended writer tick is skipped.
*******************************************************************************/
public record Counts(int audits, int lateDetails, int links, int checkpoints, boolean skipped)
{
}
/*******************************************************************************
** Bind host configuration.
*******************************************************************************/
public WormAuditChainer(WormAuditQBitConfig config)
{
this.config = config;
}
/*******************************************************************************
** Run a writer tick under one transaction; link ids cannot gap on rollback.
*******************************************************************************/
public Counts chainOnce() throws QException
{
return (chainOnce(Instant.now()));
}
/*******************************************************************************
** Clock-injected tick for deterministic settling and checkpoint tests.
*******************************************************************************/
public Counts chainOnce(Instant now) throws QException
{
config.validate();
JVM_LOCK.lock();
try(QBackendTransaction transaction = QBackendTransaction.openFor(new QueryInput(WormAuditTables.CHAIN)))
{
try
{
if(transaction instanceof RDBMSTransaction rdbms && "PostgreSQL".equals(rdbms.getConnection().getMetaData().getDatabaseProductName()))
{
try(Statement statement = rdbms.getConnection().createStatement(); ResultSet result = statement.executeQuery("SELECT pg_try_advisory_xact_lock(" + ADVISORY_LOCK + ")"))
{
result.next();
if(!result.getBoolean(1))
{
return (new Counts(0, 0, 0, 0, true));
}
}
}
Counts counts = append(new AuditStore(transaction), now);
transaction.commit();
return (counts);
}
catch(Exception e)
{
transaction.rollback();
throw (AppendOnlyGuard.map(e));
}
}
finally
{
JVM_LOCK.unlock();
}
}
/*******************************************************************************
** Append settled headers and uncovered detail chunks, then checkpoint.
*******************************************************************************/
private Counts append(AuditStore store, Instant now) throws Exception
{
QRecord tail = store.last(WormAuditTables.CHAIN);
QRecord anchor = store.anchor();
long nextId = tail == null ? anchor.getValueLong("anchor_link_id") + 1 : tail.getValueLong("id") + 1;
String previous = tail == null ? anchor.getValueString("anchor_hash") : tail.getValueString("link_hash");
List<QRecord> appended = new ArrayList<>();
Instant cutoff = now.minusSeconds(config.getSettleHorizonSeconds());
List<QRecord> audits = store.unchained(cutoff, config.getBatchSize());
Map<Long, QRecord> details = AuditStore.index(store.in("auditDetail", "auditId", audits.stream().map(r -> r.getValueLong("id")).toList()));
Map<Long, List<QRecord>> grouped = details.values().stream().collect(Collectors.groupingBy(r -> r.getValueLong("auditId")));
int lateDetails = 0;
for(QRecord audit : audits)
{
List<Long> ids = grouped.getOrDefault(audit.getValueLong("id"), List.of()).stream().map(r -> r.getValueLong("id")).sorted().toList();
for(int start = 0; start == 0 || start < ids.size(); start += 250)
{
List<Long> chunk = ids.subList(start, Math.min(start + 250, ids.size()));
QRecord link = link(store, nextId++, start == 0 ? "AUDIT" : "LATE_DETAIL", audit, chunk, details, previous, now);
appended.add(link);
previous = link.getValueString("link_hash");
if(start > 0)
{
lateDetails += chunk.size();
}
}
}
insertLinks(store, appended);
int linkCount = appended.size();
appended.clear();
details = AuditStore.index(store.lateDetails(cutoff, config.getLateDetailLookbackHours(), config.getBatchSize()));
grouped = details.values().stream().collect(Collectors.groupingBy(r -> r.getValueLong("auditId"), TreeMap::new, Collectors.toList()));
Map<Long, QRecord> headers = AuditStore.index(store.in("audit", "id", grouped.keySet()));
for(var entry : grouped.entrySet())
{
List<Long> ids = entry.getValue().stream().map(r -> r.getValueLong("id")).sorted().toList();
for(int start = 0; start < ids.size(); start += 250)
{
List<Long> chunk = ids.subList(start, Math.min(start + 250, ids.size()));
QRecord link = link(store, nextId++, "LATE_DETAIL", headers.get(entry.getKey()), chunk, details, previous, now);
appended.add(link);
previous = link.getValueString("link_hash");
lateDetails += chunk.size();
}
}
insertLinks(store, appended);
return (new Counts(audits.size(), lateDetails, linkCount + appended.size(), checkpoint(store, now), false));
}
/*******************************************************************************
** Persist bounded insert batches inside the writer transaction.
*******************************************************************************/
private void insertLinks(AuditStore store, List<QRecord> links) throws QException
{
for(int start = 0; start < links.size(); start += config.getBatchSize())
{
store.insert(WormAuditTables.CHAIN, links.subList(start, Math.min(start + config.getBatchSize(), links.size())));
}
}
/*******************************************************************************
** Build one immutable link from live stored content.
*******************************************************************************/
private QRecord link(AuditStore store, long id, String kind, QRecord audit, List<Long> details, Map<Long, QRecord> rows, String previous, Instant now) throws QException
{
return (new QRecord().withValue("id", id).withValue("kind", kind).withValue("audit_id", audit.getValueLong("id"))
.withValue("detail_ids", details.stream().map(Object::toString).collect(Collectors.joining(",")))
.withValue("prev_hash", previous).withValue("link_hash", Hashes.link(previous, store.canonical(audit, details, rows))).withValue("chained_at", now));
}
/*******************************************************************************
** Checkpoint the uncheckpointed suffix by count or elapsed time.
*******************************************************************************/
private int checkpoint(AuditStore store, Instant now) throws Exception
{
if(config.getSigningKey() == null)
{
return (0);
}
QRecord latest = store.last(WormAuditTables.CHECKPOINT);
long lastId = latest == null ? store.anchor().getValueLong("anchor_link_id") : latest.getValueLong("to_link_id");
int count = 0;
while(true)
{
List<QRecord> pending = store.page(WormAuditTables.CHAIN, lastId, config.getCheckpointEveryLinks());
if(pending.isEmpty())
{
return (count);
}
Instant since = latest == null ? pending.getFirst().getValueInstant("chained_at") : latest.getValueInstant("created_at");
if(pending.size() < config.getCheckpointEveryLinks() && now.isBefore(since.plusSeconds(config.getCheckpointEveryMinutes() * 60L)))
{
return (count);
}
latest = store.insert(WormAuditTables.CHECKPOINT, List.of(Checkpoints.sign(pending, config, now))).getFirst();
lastId = latest.getValueLong("to_link_id");
count++;
}
}
}
@@ -0,0 +1,120 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit.checkpoint;
import java.security.GeneralSecurityException;
import java.security.PublicKey;
import java.security.Signature;
import java.time.Instant;
import java.util.Base64;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
import com.kingsrook.qbits.wormaudit.canonical.Jcs;
import com.kingsrook.qbits.wormaudit.chain.Hashes;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
/*******************************************************************************
** Ed25519 checkpoint envelopes. Trust comes exclusively from host-pinned keys,
** never the public_key supplied by an untrusted checkpoint row.
*******************************************************************************/
public final class Checkpoints
{
private static final List<String> SIGNED_FIELDS = List.of("from_link_id", "to_link_id", "link_count", "head_hash", "merkle_root", "algorithm", "key_id");
/*******************************************************************************
** Utility class.
*******************************************************************************/
private Checkpoints()
{
}
/*******************************************************************************
** The exact signed payload; public_key and signature are envelope fields.
*******************************************************************************/
public static byte[] payload(QRecord checkpoint)
{
Map<String, Object> fields = new LinkedHashMap<>();
for(String field : SIGNED_FIELDS)
{
fields.put(field, field.endsWith("_id") && !field.equals("key_id") || field.equals("link_count")
? checkpoint.getValueLong(field) : checkpoint.getValueString(field));
}
return (Jcs.bytes(fields));
}
/*******************************************************************************
** Create a checkpoint only when the configured private key matches its pin.
*******************************************************************************/
public static QRecord sign(List<QRecord> links, WormAuditQBitConfig config, Instant now) throws GeneralSecurityException
{
WormAuditQBitConfig.SigningKey key = config.getSigningKey().get();
PublicKey publicKey = config.getTrustedPublicKeys().get(key.keyId());
if(publicKey == null)
{
throw (new GeneralSecurityException("Signing key id is not pinned"));
}
QRecord record = new QRecord().withValue("from_link_id", links.getFirst().getValueLong("id"))
.withValue("to_link_id", links.getLast().getValueLong("id")).withValue("link_count", (long) links.size())
.withValue("head_hash", links.getLast().getValueString("link_hash"))
.withValue("merkle_root", Hashes.merkle(links.stream().map(r -> r.getValueString("link_hash")).toList()))
.withValue("algorithm", "Ed25519").withValue("key_id", key.keyId())
.withValue("public_key", Base64.getEncoder().encodeToString(publicKey.getEncoded())).withValue("created_at", now);
Signature signature = Signature.getInstance("Ed25519");
signature.initSign(key.privateKey());
signature.update(payload(record));
record.setValue("signature", Base64.getEncoder().encodeToString(signature.sign()));
if(!verify(record, publicKey))
{
throw (new GeneralSecurityException("Signing private key does not match its pin"));
}
return (record);
}
/*******************************************************************************
** Verify with a pinned key, rejecting substituted algorithms and public keys.
*******************************************************************************/
public static boolean verify(QRecord record, PublicKey pin)
{
try
{
if(pin == null || !"Ed25519".equals(record.getValueString("algorithm"))
|| !Base64.getEncoder().encodeToString(pin.getEncoded()).equals(record.getValueString("public_key")))
{
return (false);
}
Signature signature = Signature.getInstance("Ed25519");
signature.initVerify(pin);
signature.update(payload(record));
return (signature.verify(Base64.getDecoder().decode(record.getValueString("signature"))));
}
catch(GeneralSecurityException | IllegalArgumentException e)
{
return (false);
}
}
}
@@ -0,0 +1,112 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit.ddl;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.LinkedHashMap;
import java.util.Map;
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails;
/*******************************************************************************
** Render a reviewed migration for the host's physical audit table names.
** Core physical columns must use snake_case: audit.id/timestamp,
** audit_detail.id/audit_id, audit_table.id/name/label/create_date/modify_date,
** and audit_user.id/name/create_date/modify_date. Only table names are rendered;
** QInstance field backend-name overrides do not change this DDL contract.
** Identifiers are restricted and quoted; configuration is never executable SQL.
*******************************************************************************/
public final class WormAuditDdl
{
public static final Map<String, String> DEFAULT_NAMES = Map.of("audit", "audit", "auditDetail", "audit_detail", "auditTable", "audit_table", "auditUser", "audit_user");
/*******************************************************************************
** Utility class.
*******************************************************************************/
private WormAuditDdl()
{
}
/*******************************************************************************
** A safe SQL identifier, optionally schema qualified.
*******************************************************************************/
public static String identifier(String value)
{
if(value == null || !value.matches("[A-Za-z_][A-Za-z0-9_]*(\\.[A-Za-z_][A-Za-z0-9_]*)?"))
{
throw (new IllegalArgumentException("Invalid SQL identifier"));
}
return ("\"" + value.replace(".", "\".\"") + "\"");
}
/*******************************************************************************
** Resolve backend names while preserving explicit DDL overrides.
*******************************************************************************/
public static void resolveNames(WormAuditQBitConfig config, QInstance instance)
{
Map<String, String> names = new LinkedHashMap<>();
for(String name : DEFAULT_NAMES.keySet())
{
QTableMetaData table = instance.getTable(name);
if(table == null)
{
throw (new IllegalArgumentException("Core audit table is required: " + name));
}
names.put(name, table.getBackendDetails() instanceof RDBMSTableBackendDetails details ? details.getTableName() : DEFAULT_NAMES.get(name));
}
names.putAll(config.getCoreAuditTableNames());
config.withCoreAuditTableNames(Map.copyOf(names));
}
/*******************************************************************************
** Render the template. Retention is compiled into the function, never a GUC.
*******************************************************************************/
public static String postgres(WormAuditQBitConfig config)
{
if(config.getRetentionYears() < 1 || config.getAppRole().contains("."))
{
throw (new IllegalArgumentException("Invalid retention or app role"));
}
try(var stream = WormAuditDdl.class.getResourceAsStream("/com/kingsrook/qbits/wormaudit/ddl/postgres/worm-audit.sql"))
{
String sql = new String(java.util.Objects.requireNonNull(stream).readAllBytes(), StandardCharsets.UTF_8);
Map<String, String> names = new LinkedHashMap<>(DEFAULT_NAMES);
names.putAll(config.getCoreAuditTableNames());
for(var entry : DEFAULT_NAMES.entrySet())
{
sql = sql.replace("${" + entry.getValue() + "}", identifier(names.get(entry.getKey())));
}
return (sql.replace("${app_role}", identifier(config.getAppRole())).replace("${retention}", config.getRetentionYears() + " years"));
}
catch(IOException e)
{
throw (new IllegalStateException("Cannot read WORM audit DDL", e));
}
}
}
@@ -0,0 +1,81 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit.guard;
import java.sql.SQLException;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
/*******************************************************************************
** Metadata defense and a value-free mapping of the database write-once error.
*******************************************************************************/
public class AppendOnlyGuard implements MetaDataCustomizerInterface<QTableMetaData>
{
public static final List<String> CORE_TABLES = List.of("audit", "auditDetail", "auditTable", "auditUser");
public static final String MESSAGE = "Audit records are write-once and cannot be changed or deleted.";
/*******************************************************************************
** Protect core and sidecar audit metadata after host customization.
*******************************************************************************/
@Override
public QTableMetaData customizeMetaData(QInstance instance, QTableMetaData table)
{
if(CORE_TABLES.contains(table.getName()) || table.getName().startsWith("worm_audit_"))
{
table.withoutCapabilities(Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
}
if(table.getName().startsWith("worm_audit_"))
{
table.withoutCapabilities(Capability.TABLE_INSERT);
}
return (table);
}
/*******************************************************************************
** Preserve unrelated failures, map only the documented SQLSTATE and prefix.
*******************************************************************************/
public static QException map(Exception error)
{
for(Throwable cause = error; cause != null; cause = cause.getCause())
{
if(cause instanceof SQLException sql && "P0001".equals(sql.getSQLState())
&& sql.getMessage() != null && sql.getMessage().startsWith("worm_audit:"))
{
return (new QUserFacingException(MESSAGE));
}
// The PostgreSQL driver prepends ERROR: to server error messages.
if(cause instanceof SQLException sql && "P0001".equals(sql.getSQLState())
&& sql.getMessage() != null && sql.getMessage().startsWith("ERROR: worm_audit:"))
{
return (new QUserFacingException(MESSAGE));
}
}
return (error instanceof QException q ? q : new QException("WORM audit operation failed", error));
}
}
@@ -0,0 +1,88 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit.model;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChildRecordListData;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
/*******************************************************************************
** A standard child-record list for a checkpoint/export/verification link range.
** This is a record-screen relation, not an app-home widget. Reads retain user
** permissions and use a bounded page with a filtered View All link.
*******************************************************************************/
public class LinkRangeWidget extends AbstractWidgetRenderer
{
/*******************************************************************************
** Render the inclusive range stored on the parent record.
*******************************************************************************/
@Override
public RenderWidgetOutput render(RenderWidgetInput input) throws QException
{
String parent = (String) input.getWidgetMetaData().getDefaultValues().get("parentTable");
GetInput get = new GetInput().withPrimaryKey(input.getQueryParams().get("id")).withInputSource(QInputSource.USER);
get.setTableName(parent);
get.withPermissionEnforcement(PermissionEnforcement.ENFORCE);
QRecord record = new GetAction().execute(get).getRecord();
if(record == null)
{
throw (new QNotFoundException("Audit range record was not found"));
}
Long from = record.getValueLong("from_link_id");
Long to = record.getValueLong("to_link_id");
QQueryFilter filter = new QQueryFilter()
.withCriteria(new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, from == null ? 1L : from))
.withCriteria(new QFilterCriteria("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, to == null ? 0L : to))
.withOrderBy(new QFilterOrderBy("id", true));
CountInput count = new CountInput(WormAuditTables.CHAIN).withFilter(filter).withInputSource(QInputSource.USER);
count.withPermissionEnforcement(PermissionEnforcement.ENFORCE);
int total = new CountAction().execute(count).getCount();
String path = QContext.getQInstance().getTablePath(WormAuditTables.CHAIN);
String viewAll = path + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), StandardCharsets.UTF_8);
QueryInput query = new QueryInput(WormAuditTables.CHAIN).withFilter(filter.clone().withLimit(100)).withInputSource(QInputSource.USER);
query.withPermissionEnforcement(PermissionEnforcement.ENFORCE);
return (new RenderWidgetOutput(new ChildRecordListData("Chain links", new QueryAction().execute(query),
QContext.getQInstance().getTable(WormAuditTables.CHAIN), path, viewAll, total)));
}
}
@@ -0,0 +1,124 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit.model;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails;
/*******************************************************************************
** Append-only sidecar tables. Snake-case fields map directly to DDL columns.
** Export metadata is the integration hook for phase (b); no export runs here.
*******************************************************************************/
public final class WormAuditTables
{
public static final String CHAIN = "worm_audit_chain";
public static final String CHECKPOINT = "worm_audit_checkpoint";
public static final String EXPORT = "worm_audit_export";
public static final String VERIFICATION = "worm_audit_verification";
public static final String PURGE = "worm_audit_purge";
public static final List<String> ALL = List.of(CHAIN, CHECKPOINT, EXPORT, VERIFICATION, PURGE);
public static final List<String> NAV = List.of(VERIFICATION, EXPORT, CHECKPOINT, CHAIN, PURGE);
/*******************************************************************************
** Utility class.
*******************************************************************************/
private WormAuditTables()
{
}
/*******************************************************************************
** Construct full record screens, with no recursive audit writes.
*******************************************************************************/
public static List<QTableMetaData> tables(WormAuditQBitConfig config)
{
List<QTableMetaData> tables = new ArrayList<>();
tables.add(table(config, CHAIN, "Audit Chain Link", "id:L kind:S audit_id:L detail_ids:T prev_hash:S link_hash:S chained_at:D"));
tables.add(table(config, CHECKPOINT, "Checkpoint", "id:L from_link_id:L to_link_id:L link_count:L head_hash:S merkle_root:S algorithm:S key_id:S public_key:T signature:T created_at:D"));
tables.add(table(config, EXPORT, "Sealed Export", "id:L from_link_id:L to_link_id:L from_checkpoint_id:L to_checkpoint_id:L bucket_alias:S object_prefix:T manifest_key:T manifest_sha256:S prev_manifest_sha256:S object_count:L byte_count:L lock_mode:S retain_until:D sealed_at:D"));
tables.add(table(config, VERIFICATION, "Verification", "id:L kind:S trigger:S started_at:D finished_at:D from_link_id:L to_link_id:L links_checked:L outcome:S break_count:L first_break_link_id:L first_break_audit_id:L first_break_reason:T export_id:L run_by:S"));
tables.add(table(config, PURGE, "Purge", "id:L requested_at:D requested_by:S cutoff:D retention_interval:S outcome:S reason:T audits_deleted:L details_deleted:L links_deleted:L anchor_link_id:L anchor_hash:S"));
return (tables);
}
/*******************************************************************************
** Common metadata for a process-written table.
*******************************************************************************/
private static QTableMetaData table(WormAuditQBitConfig config, String name, String label, String fields)
{
QTableMetaData table = new QTableMetaData().withName(name).withLabel(label)
.withBackendName(config.getBackendName()).withBackendDetails(new RDBMSTableBackendDetails().withTableName(name))
.withPrimaryKeyField("id").withRecordLabelFields("id").withRecordLabelFormat("%s")
.withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.NONE))
.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_INSERT_EDIT_DELETE_PERMISSIONS).withPermissionBaseName(name));
for(String spec : fields.split(" "))
{
String[] parts = spec.split(":");
QFieldType type = switch(parts[1])
{
case "L" -> QFieldType.LONG;
case "D" -> QFieldType.DATE_TIME;
case "T" -> QFieldType.TEXT;
default -> QFieldType.STRING;
};
table.withField(new QFieldMetaData(parts[0], type).withBackendName(parts[0]));
}
for(String reference : List.of("from_link_id", "to_link_id", "first_break_link_id", "anchor_link_id"))
{
if(table.getFields().containsKey(reference))
{
table.getField(reference).withPossibleValueSourceName(CHAIN);
}
}
for(String reference : List.of("from_checkpoint_id", "to_checkpoint_id"))
{
if(table.getFields().containsKey(reference))
{
table.getField(reference).withPossibleValueSourceName(CHECKPOINT);
}
}
if(table.getFields().containsKey("export_id"))
{
table.getField("export_id").withPossibleValueSourceName(EXPORT);
}
table.withSection(new QFieldSection("record", label, new QIcon("fact_check"), Tier.T1, new ArrayList<>(table.getFields().keySet())));
if(List.of(CHECKPOINT, EXPORT, VERIFICATION).contains(name))
{
table.withSection(new QFieldSection("links", new QIcon("link"), Tier.T2).withLabel("Chain links").withWidgetName(name + "_links"));
}
return (table);
}
}
@@ -0,0 +1,116 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit.processes;
import java.time.Instant;
import java.util.List;
import java.util.Set;
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
import com.kingsrook.qbits.wormaudit.verify.VerificationResult;
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
import com.kingsrook.qqq.backend.core.alerts.AlertRecordRef;
import com.kingsrook.qqq.backend.core.alerts.AlertSignal;
import com.kingsrook.qqq.backend.core.alerts.QAlerts;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
/*******************************************************************************
** Verification run persistence and the core alerts SPI. Both UI and schedule
** write exactly one immutable result row; only schedules change alert state.
*******************************************************************************/
public final class VerificationRuns
{
/*******************************************************************************
** Utility class.
*******************************************************************************/
private VerificationRuns()
{
}
/*******************************************************************************
** Resolve this instance's configuration, never a global cross-instance cache.
*******************************************************************************/
public static WormAuditQBitConfig config()
{
return (QContext.getQInstance().getQBits().values().stream().map(q -> q.getConfig()).filter(WormAuditQBitConfig.class::isInstance)
.map(WormAuditQBitConfig.class::cast).findFirst().orElseThrow(() -> new IllegalStateException("WORM audit qbit is not installed")));
}
/*******************************************************************************
** Verify and persist, sending a heartbeat even if a scheduled run errors.
*******************************************************************************/
public static VerificationResult run(Set<Long> selection, String trigger) throws QException
{
boolean scheduled = "SCHEDULE".equals(trigger);
try
{
VerificationResult result = new WormAuditVerifier(config()).verify(selection);
QRecord row = save(result, trigger);
if(scheduled)
{
if(result.passed())
{
QAlerts.resolve("worm_audit.chain.break", "chain", "Audit chain verification passed");
}
else
{
QAlerts.raise(new AlertSignal("worm_audit.chain.break").withDedupKey("chain")
.withTitle(result.getBreaks().getFirst().reason()).withRecordRef(new AlertRecordRef(WormAuditTables.VERIFICATION, row.getValue("id"))));
}
}
return (result);
}
finally
{
if(scheduled)
{
QAlerts.heartbeat("worm_audit.verify.daily");
}
}
}
/*******************************************************************************
** Persist identifiers and a value-free first reason, never audit content.
*******************************************************************************/
public static QRecord save(VerificationResult result, String trigger) throws QException
{
String user = QContext.getQSession() == null || QContext.getQSession().getUser() == null ? "system" : QContext.getQSession().getUser().getIdReference();
QRecord row = new QRecord().withValue("kind", "CHAIN").withValue("trigger", trigger)
.withValue("started_at", result.getStartedAt()).withValue("finished_at", Instant.now())
.withValue("from_link_id", result.getFromLinkId()).withValue("to_link_id", result.getToLinkId())
.withValue("links_checked", result.getLinksChecked()).withValue("outcome", result.passed() ? "PASS" : "FAIL")
.withValue("break_count", result.getBreakCount()).withValue("run_by", user);
if(!result.passed())
{
var first = result.getBreaks().getFirst();
row.withValue("first_break_link_id", first.linkId()).withValue("first_break_audit_id", first.auditId()).withValue("first_break_reason", first.reason());
}
return (new AuditStore(null).insert(WormAuditTables.VERIFICATION, List.of(row)).getFirst());
}
}
@@ -0,0 +1,312 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit.processes;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
import com.kingsrook.qbits.wormaudit.verify.VerificationResult;
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
import com.kingsrook.qqq.backend.core.actions.permissions.UseOtherPermissionNameCustomPermissionChecker;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunInput;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunOutput;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.CouldNotFindQueryFilterForExtractStepException;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
/*******************************************************************************
** Standard streamed ETL Review/Result action plus two scheduled-only processes.
** No process is an app navigation entry.
*******************************************************************************/
public final class WormAuditProcesses
{
public static final String VERIFY = "worm_audit_verify_chain";
public static final String CHAIN = "worm_audit_chain_tick";
public static final String DAILY = "worm_audit_verify_daily";
public static final String PERMISSION = "worm_audit.verify";
/*******************************************************************************
** Utility class.
*******************************************************************************/
private WormAuditProcesses()
{
}
/*******************************************************************************
** Metadata for table/record verification and scheduled background work.
*******************************************************************************/
public static List<QProcessMetaData> processes(WormAuditQBitConfig config)
{
Map<String, Serializable> values = new HashMap<>();
values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, WormAuditTables.CHAIN);
values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, WormAuditTables.CHAIN);
values.put(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION, true);
values.put(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true);
values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, "Verify live audit content, link continuity and signed checkpoints.");
QProcessMetaData verify = StreamedETLWithFrontendProcess.defineProcessMetaData(Extract.class, Review.class, Load.class, values)
.withName(VERIFY).withLabel("Verify audit chain").withTableName(WormAuditTables.CHAIN).withPermissionRules(permission());
QProcessMetaData chain = new QProcessMetaData().withName(CHAIN).withLabel("Chain settled audit records")
.withStep(new QBackendStepMetaData().withName("chain").withCode(new QCodeReference(ChainTick.class)))
.withPermissionRules(permission()).withSchedule(new QScheduleMetaData().withSchedulerName(config.getSchedulerName())
.withRepeatSeconds(config.getChainIntervalSeconds()).withInitialDelaySeconds(config.getChainIntervalSeconds()));
QProcessMetaData daily = new QProcessMetaData().withName(DAILY).withLabel("Daily audit chain verification")
.withStep(new QBackendStepMetaData().withName("verify").withCode(new QCodeReference(Daily.class)))
.withPermissionRules(permission()).withSchedule(new QScheduleMetaData().withSchedulerName(config.getSchedulerName())
.withCronExpression(config.getVerifyCron()).withCronTimeZoneId("UTC"));
return (List.of(verify, chain, daily));
}
/*******************************************************************************
** One explicit permission, not process-name access suffixes.
*******************************************************************************/
private static QPermissionRules permission()
{
return (new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION)
.withCustomPermissionChecker(UseOtherPermissionNameCustomPermissionChecker.build(PERMISSION)));
}
/*******************************************************************************
** Use core selection/filter interpretation, with a full-chain default. An
** empty frontend filter means a full run, so completeness is still checked.
*******************************************************************************/
public static class Extract extends ExtractViaQueryStep
{
/*******************************************************************************
** Extract links in id order and make the effective scope available to Load.
*******************************************************************************/
@Override
public QQueryFilter getQueryFilter(RunBackendStepInput input) throws QException
{
input.addValue(FIELD_SOURCE_TABLE, WormAuditTables.CHAIN);
QQueryFilter filter;
try
{
filter = super.getQueryFilter(input).clone();
}
catch(CouldNotFindQueryFilterForExtractStepException expected)
{
filter = new QQueryFilter();
}
filter.setOrderBys(List.of(new QFilterOrderBy("id", true)));
return (filter);
}
}
/*******************************************************************************
** Review counts the streamed selection without verifying or writing rows.
*******************************************************************************/
public static class Review extends AbstractTransformStep implements ProcessSummaryProviderInterface
{
private long count;
private long min = Long.MAX_VALUE;
private long max = Long.MIN_VALUE;
private Long checkpoints;
/*******************************************************************************
** Accumulate the scope and forward links to the loader.
*******************************************************************************/
@Override
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
{
for(QRecord record : input.getRecords())
{
count++;
min = Math.min(min, record.getValueLong("id"));
max = Math.max(max, record.getValueLong("id"));
}
checkpoints = null;
output.getRecords().addAll(input.getRecords());
}
/*******************************************************************************
** Standard Review line, including the link range and checkpoint count.
*******************************************************************************/
@Override
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
{
if(checkpoints == null)
{
try
{
CountInput input = new CountInput(WormAuditTables.CHECKPOINT).withFilter(new QQueryFilter(
new QFilterCriteria("from_link_id", QCriteriaOperator.LESS_THAN_OR_EQUALS, max),
new QFilterCriteria("to_link_id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, min)));
input.withPermissionBypass("WORM audit internal review");
checkpoints = count == 0 ? 0L : new CountAction().execute(input).getCount().longValue();
}
catch(QException e)
{
return (new ArrayList<>(List.of(new ProcessSummaryLine(Status.ERROR, "Cannot count overlapping audit checkpoints"))));
}
}
String range = count == 0 ? "none" : WormAuditVerifier.number(min) + "–" + WormAuditVerifier.number(max);
return (new ArrayList<>(List.of(new ProcessSummaryLine(Status.INFO, WormAuditVerifier.number(count)
+ " chain links will be verified (links " + range + ", " + WormAuditVerifier.number(checkpoints) + " checkpoints)"))));
}
}
/*******************************************************************************
** Load collects selected ids, then performs one snapshot verification in
** postRun and appends one result row, including for an empty chain.
*******************************************************************************/
public static class Load extends AbstractLoadStep implements ProcessSummaryProviderInterface
{
private Set<Long> ids;
private VerificationResult result;
/*******************************************************************************
** Retain ids only, not audit content, across streamed pages.
*******************************************************************************/
@Override
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
{
if(selected(new Extract().getQueryFilter(input)))
{
if(ids == null)
{
ids = new HashSet<>();
}
input.getRecords().forEach(r -> ids.add(r.getValueLong("id")));
}
output.getRecords().addAll(input.getRecords());
}
/*******************************************************************************
** Run once after all pages. Explicit filters are selection runs.
*******************************************************************************/
@Override
public void postRun(BackendStepPostRunInput input, BackendStepPostRunOutput output) throws QException
{
QQueryFilter filter = new Extract().getQueryFilter(input);
if(selected(filter) && ids == null)
{
// Some callers supply the effective selection only to postRun.
ids = new HashSet<>();
new AuditStore(null).query(WormAuditTables.CHAIN, filter).forEach(r -> ids.add(r.getValueLong("id")));
}
result = VerificationRuns.run(selected(filter) ? ids : null, "USER");
}
/*******************************************************************************
** Empty filters preserve full completeness verification and retain no ids.
*******************************************************************************/
private boolean selected(QQueryFilter filter)
{
return ((filter.getCriteria() != null && !filter.getCriteria().isEmpty())
|| (filter.getSubFilters() != null && !filter.getSubFilters().isEmpty())
|| filter.getLimit() != null || (filter.getSkip() != null && filter.getSkip() > 0));
}
/*******************************************************************************
** One ERROR per exact break; warnings remain visible on a passing run.
*******************************************************************************/
@Override
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
{
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
if(result == null)
{
return (lines);
}
if(result.passed())
{
lines.add(new ProcessSummaryLine(Status.OK, WormAuditVerifier.number(result.getLinksChecked()) + " links verified"));
}
result.getBreaks().forEach(b -> lines.add(new ProcessSummaryLine(Status.ERROR, b.reason())));
result.getWarnings().forEach(w -> lines.add(new ProcessSummaryLine(Status.WARNING, w)));
return (lines);
}
}
/*******************************************************************************
** Scheduled writer tick.
*******************************************************************************/
public static class ChainTick implements BackendStep
{
/*******************************************************************************
** Append settled work under the shared writer lock.
*******************************************************************************/
@Override
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
{
output.addValue("linksChained", new WormAuditChainer(VerificationRuns.config()).chainOnce().links());
}
}
/*******************************************************************************
** Daily verification, including alerts and heartbeat.
*******************************************************************************/
public static class Daily implements BackendStep
{
/*******************************************************************************
** Verify the full settled chain.
*******************************************************************************/
@Override
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
{
output.addValue("outcome", VerificationRuns.run(null, "SCHEDULE").passed() ? "PASS" : "FAIL");
}
}
}
@@ -0,0 +1,165 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit.verify;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
/*******************************************************************************
** Ordered, value-free findings. Break identifiers are separately available for
** record navigation and alert correlation without parsing human-readable text.
*******************************************************************************/
public class VerificationResult
{
/*******************************************************************************
** One break; null ids apply to coverage and checkpoint findings.
*******************************************************************************/
public record Break(Long linkId, Long auditId, String reason)
{
}
private final List<Break> breaks = new ArrayList<>();
private final List<String> warnings = new ArrayList<>();
private final Instant startedAt = Instant.now();
private long breakCount;
private long linksChecked;
private Long fromLinkId;
private Long toLinkId;
/*******************************************************************************
** Add a finding in deterministic verification order.
*******************************************************************************/
public void add(Long linkId, Long auditId, String reason)
{
breakCount++;
if(breaks.size() < 1000)
{
breaks.add(new Break(linkId, auditId, reason));
}
}
/*******************************************************************************
** Count additional SQL offenders without retaining their content.
*******************************************************************************/
public void omitted(long count)
{
breakCount += count;
}
/*******************************************************************************
** Total findings, including diagnostics omitted by the bounded display limit.
*******************************************************************************/
public long getBreakCount()
{
return (breakCount);
}
/*******************************************************************************
** Keep warnings bounded even when many checkpoint ranges have been purged.
*******************************************************************************/
public void warn(String warning)
{
if(warnings.size() < 100)
{
warnings.add(warning);
}
}
/*******************************************************************************
** Record the scope that was checked.
*******************************************************************************/
public void scope(long count, Long from, Long to)
{
linksChecked = count;
fromLinkId = from;
toLinkId = to;
}
/*******************************************************************************
** Ordered findings.
*******************************************************************************/
public List<Break> getBreaks()
{
return (List.copyOf(breaks));
}
/*******************************************************************************
** Warnings do not break the chain.
*******************************************************************************/
public List<String> getWarnings()
{
return (warnings);
}
/*******************************************************************************
** Whether all checked invariants hold.
*******************************************************************************/
public boolean passed()
{
return (breakCount == 0);
}
/*******************************************************************************
** Number of checked links.
*******************************************************************************/
public long getLinksChecked()
{
return (linksChecked);
}
/*******************************************************************************
** First checked link, null for an empty scope.
*******************************************************************************/
public Long getFromLinkId()
{
return (fromLinkId);
}
/*******************************************************************************
** Last checked link, null for an empty scope.
*******************************************************************************/
public Long getToLinkId()
{
return (toLinkId);
}
/*******************************************************************************
** Verification start time.
*******************************************************************************/
public Instant getStartedAt()
{
return (startedAt);
}
}
@@ -0,0 +1,477 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit.verify;
import java.sql.Connection;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
import com.kingsrook.qbits.wormaudit.chain.Hashes;
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
import com.kingsrook.qbits.wormaudit.checkpoint.Checkpoints;
import com.kingsrook.qbits.wormaudit.guard.AppendOnlyGuard;
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSTransaction;
/*******************************************************************************
** Snapshot verification of live content, continuity, exact coverage and signed
** checkpoints. Selection runs verify predecessor boundaries but deliberately
** omit global completeness; full runs begin at the latest retention anchor.
*******************************************************************************/
public class WormAuditVerifier
{
public static final int PAGE_SIZE = 1000;
private final WormAuditQBitConfig config;
/*******************************************************************************
** Bind trust pins and settle horizon.
*******************************************************************************/
public WormAuditVerifier(WormAuditQBitConfig config)
{
this.config = config;
}
/*******************************************************************************
** Verify all live links and settled coverage.
*******************************************************************************/
public VerificationResult verify() throws QException
{
return (verify(null));
}
/*******************************************************************************
** Null means full verification; an empty set is an empty selection.
*******************************************************************************/
public VerificationResult verify(Set<Long> selection) throws QException
{
WormAuditChainer.JVM_LOCK.lock();
try(QBackendTransaction transaction = QBackendTransaction.openFor(new QueryInput(WormAuditTables.CHAIN)))
{
try
{
if(transaction instanceof RDBMSTransaction rdbms)
{
rdbms.getConnection().setTransactionIsolation(Connection.TRANSACTION_REPEATABLE_READ);
}
VerificationResult result = verify(new AuditStore(transaction), selection, Instant.now());
transaction.commit();
return (result);
}
catch(Exception e)
{
transaction.rollback();
throw (AppendOnlyGuard.map(e));
}
}
finally
{
WormAuditChainer.JVM_LOCK.unlock();
}
}
/*******************************************************************************
** Verify one consistent storage snapshot.
*******************************************************************************/
private VerificationResult verify(AuditStore store, Set<Long> selection, Instant now) throws QException
{
VerificationResult result = new VerificationResult();
if(config.getSigningKey() == null)
{
result.warn("no signing key configured");
}
QRecord anchor = store.anchor();
long anchorId = anchor.getValueLong("anchor_link_id");
long lastId = anchorId;
String previous = anchor.getValueString("anchor_hash");
long checked = 0;
Long first = null;
Long last = null;
Set<Long> missing = selection == null ? null : new HashSet<>(selection);
CheckpointCursor checkpoints = new CheckpointCursor(store, selection, anchor, result);
while(true)
{
List<QRecord> page = store.page(WormAuditTables.CHAIN, lastId, PAGE_SIZE);
if(page.isEmpty())
{
break;
}
List<QRecord> selected = page.stream().filter(r -> selection == null || selection.contains(r.getValueLong("id"))).toList();
Map<Long, QRecord> headers = AuditStore.index(store.in("audit", "id", selected.stream().map(r -> r.getValueLong("audit_id")).toList()));
Set<Long> detailIds = new HashSet<>();
for(QRecord link : selected)
{
try
{
List<Long> ids = AuditStore.detailIds(link);
if(ids.size() <= 250)
{
detailIds.addAll(ids);
}
}
catch(IllegalArgumentException e)
{
// Content verification reports malformed coverage with its link id.
}
}
Map<Long, QRecord> details = new HashMap<>();
List<Long> ids = new ArrayList<>(detailIds);
for(int start = 0; start < ids.size(); start += PAGE_SIZE)
{
details.putAll(AuditStore.index(store.in("auditDetail", "id", ids.subList(start, Math.min(start + PAGE_SIZE, ids.size())))));
}
for(QRecord link : page)
{
long id = link.getValueLong("id");
long auditId = link.getValueLong("audit_id");
if(selection == null || selection.contains(id))
{
if(missing != null)
{
missing.remove(id);
}
checked++;
first = first == null ? id : first;
last = id;
if(selection == null && id != lastId + 1)
{
result.add(id, auditId, "Missing link ids " + number(lastId + 1) + "–" + number(id - 1) + " (gap)");
}
if(id != lastId + 1 || previous == null || !previous.equals(link.getValueString("prev_hash")))
{
result.add(id, auditId, "Break at link " + number(id) + " (audit " + number(auditId) + "): prev-hash discontinuity");
}
verifyContent(store, link, headers.get(auditId), details, result);
}
previous = link.getValueString("link_hash");
lastId = id;
}
checkpoints.through(lastId);
}
checkpoints.through(Long.MAX_VALUE);
result.scope(checked, first, last);
if(missing != null)
{
missing.stream().sorted().forEach(id -> result.add(id, null, "Missing selected link " + number(id)));
}
if(selection == null)
{
Instant cutoff = now.minusSeconds(config.getSettleHorizonSeconds());
if(store.isRdbms())
{
store.completeness(cutoff, result);
}
else
{
completeness(store, store.all(WormAuditTables.CHAIN), cutoff, result);
}
}
return (result);
}
/*******************************************************************************
** Recompute exact stored content. Diagnostics name covered detail ids because
** v1 stores a document hash, not per-field hashes or prior values.
*******************************************************************************/
private void verifyContent(AuditStore store, QRecord link, QRecord audit, Map<Long, QRecord> rows, VerificationResult result) throws QException
{
long id = link.getValueLong("id");
long auditId = link.getValueLong("audit_id");
if(List.of("kind", "detail_ids", "prev_hash", "link_hash", "chained_at").stream().anyMatch(field -> link.getValue(field) == null))
{
result.add(id, auditId, "Break at link " + number(id) + " (audit " + number(auditId) + "): malformed link");
return;
}
if(audit == null)
{
result.add(id, auditId, "Break at link " + number(id) + " (audit " + number(auditId) + "): audit header is missing");
return;
}
if(audit.getValueInstant("timestamp") == null)
{
result.add(id, auditId, "Break at link " + number(id) + " (audit " + number(auditId) + "): missing audit timestamp");
return;
}
try
{
List<Long> details = AuditStore.detailIds(link);
String kind = link.getValueString("kind");
if(!List.of("AUDIT", "LATE_DETAIL").contains(kind) || details.size() > 250 || ("LATE_DETAIL".equals(kind) && details.isEmpty()))
{
result.add(id, auditId, "Break at link " + number(id) + ": invalid kind or detail coverage");
}
if(details.size() > 250)
{
return;
}
Set<Long> unique = new HashSet<>();
for(long detail : details)
{
if(!unique.add(detail))
{
result.add(id, auditId, "audit_detail " + number(detail) + " is covered twice or more by the chain");
}
}
String hash = Hashes.link(link.getValueString("prev_hash"), store.canonical(audit, details, rows));
if(!hash.equals(link.getValueString("link_hash")))
{
String table = store.name("auditTable", audit.getValueLong("auditTableId"));
result.add(id, auditId, "Break at link " + number(id) + " (audit " + number(auditId) + ", table " + table + ", record "
+ audit.getValueString("recordId") + "): content differs from the chained hash"
+ (details.isEmpty() ? "" : " (audit_detail " + details.stream().map(WormAuditVerifier::number).collect(java.util.stream.Collectors.joining(", ")) + ")"));
}
}
catch(QException | IllegalArgumentException e)
{
result.add(id, auditId, "Break at link " + number(id) + " (audit " + number(auditId) + "): " + e.getMessage());
}
}
/*******************************************************************************
** Every settled header and detail must occur once; orphan details also fail.
*******************************************************************************/
private void completeness(AuditStore store, List<QRecord> links, Instant cutoff, VerificationResult result) throws QException
{
Map<Long, Integer> headers = new HashMap<>();
Map<Long, Integer> details = new HashMap<>();
for(QRecord link : links)
{
if("AUDIT".equals(link.getValueString("kind")))
{
headers.merge(link.getValueLong("audit_id"), 1, Integer::sum);
}
try
{
AuditStore.detailIds(link).forEach(id -> details.merge(id, 1, Integer::sum));
}
catch(IllegalArgumentException e)
{
// Invalid coverage was already reported while checking link content.
}
}
Map<Long, QRecord> audits = new HashMap<>();
for(QRecord audit : store.all("audit"))
{
long id = audit.getValueLong("id");
audits.put(id, audit);
if(audit.getValueInstant("timestamp") == null)
{
result.add(null, id, "audit " + number(id) + " has no timestamp");
}
else if(audit.getValueInstant("timestamp").isBefore(cutoff))
{
coverage("audit", id, headers.getOrDefault(id, 0), result);
}
}
for(QRecord detail : store.all("auditDetail"))
{
QRecord audit = audits.get(detail.getValueLong("auditId"));
if(audit == null || audit.getValueInstant("timestamp") == null || audit.getValueInstant("timestamp").isBefore(cutoff))
{
coverage("audit_detail", detail.getValueLong("id"), details.getOrDefault(detail.getValueLong("id"), 0), result);
}
}
}
/*******************************************************************************
** Report coverage without printing any audit content.
*******************************************************************************/
private void coverage(String table, long id, int count, VerificationResult result)
{
if(count != 1)
{
result.add(null, table.equals("audit") ? id : null, table + " " + number(id)
+ (count == 0 ? " is not covered by the chain" : " is covered twice or more by the chain"));
}
}
/*******************************************************************************
** Check pinned signatures even for purged ranges. A partially purged range's
** Merkle tree requires its sealed export; never claim to recompute it live.
*******************************************************************************/
private class CheckpointCursor
{
private final AuditStore store;
private final Set<Long> selection;
private final QRecord anchor;
private final VerificationResult result;
private long after;
private QRecord next;
/*******************************************************************************
** Keep only one checkpoint pending as pages complete.
*******************************************************************************/
CheckpointCursor(AuditStore store, Set<Long> selection, QRecord anchor, VerificationResult result)
{
this.store = store;
this.selection = selection;
this.anchor = anchor;
this.result = result;
}
/*******************************************************************************
** Verify completed ranges, including missing tails at end of the scan.
*******************************************************************************/
void through(long lastId) throws QException
{
while(true)
{
if(next == null)
{
List<QRecord> rows = store.page(WormAuditTables.CHECKPOINT, after, 1);
if(rows.isEmpty())
{
return;
}
next = rows.getFirst();
}
Long to = next.getValueLong("to_link_id");
if(to != null && to > lastId)
{
return;
}
check(next);
after = next.getValueLong("id");
next = null;
}
}
/*******************************************************************************
** Re-read at most one capped range of hashes in the same snapshot.
*******************************************************************************/
private void check(QRecord checkpoint) throws QException
{
String prefix = "checkpoint " + number(checkpoint.getValueLong("id")) + ": ";
if(List.of("from_link_id", "to_link_id", "link_count", "head_hash", "merkle_root", "algorithm", "key_id", "public_key", "signature")
.stream().anyMatch(field -> checkpoint.getValue(field) == null))
{
result.add(null, null, prefix + "malformed checkpoint");
return;
}
long from = checkpoint.getValueLong("from_link_id");
long to = checkpoint.getValueLong("to_link_id");
long anchorId = anchor.getValueLong("anchor_link_id");
if(selection != null && selection.stream().noneMatch(id -> id >= from && id <= to))
{
return;
}
var pin = config.getTrustedPublicKeys().get(checkpoint.getValueString("key_id"));
if(pin == null)
{
result.add(null, null, prefix + "untrusted key");
}
else if(!Checkpoints.verify(checkpoint, pin))
{
result.add(null, null, prefix + "signature does not verify");
}
if(from < 1 || to < from || checkpoint.getValueLong("link_count") != to - from + 1
|| to - from + 1 > config.getCheckpointEveryLinks())
{
result.add(null, null, prefix + "invalid link range or count");
return;
}
if(from <= anchorId)
{
QRecord head = to > anchorId ? store.get(WormAuditTables.CHAIN, to) : null;
if(to > anchorId && (head == null || !Objects.equals(head.getValueString("link_hash"), checkpoint.getValueString("head_hash"))))
{
result.add(to, null, prefix + "head mismatch");
}
if(to == anchorId && !Objects.equals(anchor.getValueString("anchor_hash"), checkpoint.getValueString("head_hash")))
{
result.add(to, null, prefix + "head mismatch with purge anchor");
}
result.warn(prefix + "purged range; Merkle proof requires the sealed export");
return;
}
List<String> hashes = new ArrayList<>();
long cursor = from - 1;
String head = null;
while(cursor < to)
{
List<QRecord> page = store.page(WormAuditTables.CHAIN, cursor, (int) Math.min(PAGE_SIZE, to - cursor));
if(page.isEmpty())
{
break;
}
for(QRecord link : page)
{
cursor = link.getValueLong("id");
if(cursor <= to)
{
hashes.add(link.getValueString("link_hash"));
}
if(cursor == to)
{
head = link.getValueString("link_hash");
}
}
}
if(!Objects.equals(head, checkpoint.getValueString("head_hash")))
{
result.add(to, null, prefix + "head mismatch");
}
try
{
if(hashes.size() != checkpoint.getValueLong("link_count") || hashes.isEmpty()
|| hashes.stream().anyMatch(Objects::isNull) || !Hashes.merkle(hashes).equals(checkpoint.getValueString("merkle_root")))
{
result.add(null, null, prefix + "merkle mismatch");
}
}
catch(IllegalArgumentException e)
{
result.add(null, null, prefix + "merkle mismatch");
}
}
}
/*******************************************************************************
** Stable human-readable ids matching the process summary contract.
*******************************************************************************/
public static String number(long value)
{
return (String.format(Locale.US, "%,d", value));
}
}
@@ -0,0 +1,325 @@
-- liquibase formatted sql
-- changeset worm-audit:1
CREATE TABLE IF NOT EXISTS public.worm_audit_chain (
id bigint PRIMARY KEY,
kind text NOT NULL,
audit_id bigint NOT NULL,
detail_ids text NOT NULL,
prev_hash text NOT NULL,
link_hash text NOT NULL,
chained_at timestamptz NOT NULL,
CHECK (id > 0),
CHECK (kind IN ('AUDIT', 'LATE_DETAIL')),
CHECK (prev_hash ~ '^[0-9a-f]{64}$' AND link_hash ~ '^[0-9a-f]{64}$'),
CHECK (detail_ids = '' OR (detail_ids ~ '^(0|[1-9][0-9]*)(,(0|[1-9][0-9]*))*$' AND cardinality(string_to_array(detail_ids, ',')) <= 250))
);
CREATE TABLE IF NOT EXISTS public.worm_audit_checkpoint (
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
from_link_id bigint,
to_link_id bigint,
link_count bigint,
head_hash text,
merkle_root text,
algorithm text,
key_id text,
public_key text,
signature text,
created_at timestamptz
);
CREATE TABLE IF NOT EXISTS public.worm_audit_export (
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
from_link_id bigint,
to_link_id bigint,
from_checkpoint_id bigint,
to_checkpoint_id bigint,
bucket_alias text,
object_prefix text,
manifest_key text,
manifest_sha256 text,
prev_manifest_sha256 text,
object_count bigint,
byte_count bigint,
lock_mode text,
retain_until timestamptz,
sealed_at timestamptz
);
CREATE TABLE IF NOT EXISTS public.worm_audit_verification (
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
kind text,
"trigger" text,
started_at timestamptz,
finished_at timestamptz,
from_link_id bigint,
to_link_id bigint,
links_checked bigint,
outcome text,
break_count bigint,
first_break_link_id bigint,
first_break_audit_id bigint,
first_break_reason text,
export_id bigint,
run_by text
);
CREATE TABLE IF NOT EXISTS public.worm_audit_purge (
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
requested_at timestamptz,
requested_by text,
cutoff timestamptz,
retention_interval text,
outcome text,
reason text,
audits_deleted bigint,
details_deleted bigint,
links_deleted bigint,
anchor_link_id bigint,
anchor_hash text
);
CREATE UNIQUE INDEX IF NOT EXISTS worm_audit_chain_audit ON public.worm_audit_chain(audit_id) WHERE kind = 'AUDIT';
CREATE INDEX IF NOT EXISTS worm_audit_chain_audit_all ON public.worm_audit_chain(audit_id);
-- changeset worm-audit:2 splitStatements:false
CREATE OR REPLACE FUNCTION public.worm_audit_reject() RETURNS trigger
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp SET timezone = 'UTC' AS $$
DECLARE
old_time timestamptz;
function_owner name;
stack text;
row_identity text := 'all';
BEGIN
IF TG_OP = 'DELETE' AND current_setting('worm_audit.purging', true) = 'on' THEN
SELECT pg_get_userbyid(proowner) INTO function_owner FROM pg_proc
WHERE oid = 'public.worm_audit_purge(timestamptz)'::regprocedure;
GET DIAGNOSTICS stack = PG_CONTEXT;
IF current_user = function_owner AND position('function worm_audit_purge(timestamp with time zone)' in stack) > 0 THEN
IF TG_RELID = '${audit}'::regclass THEN
old_time := OLD.timestamp;
ELSIF TG_RELID = '${audit_detail}'::regclass THEN
SELECT timestamp INTO old_time FROM ${audit} WHERE id = OLD.audit_id;
ELSIF TG_RELID = 'public.worm_audit_chain'::regclass THEN
SELECT timestamp INTO old_time FROM ${audit} WHERE id = OLD.audit_id;
END IF;
IF old_time < now() - interval '${retention}' THEN
RETURN OLD;
END IF;
END IF;
END IF;
IF TG_LEVEL = 'ROW' THEN
row_identity := OLD.id::text;
END IF;
RAISE EXCEPTION USING ERRCODE = 'P0001',
MESSAGE = format('worm_audit: audit rows are write-once; %s of "%s" is not allowed (id=%s). Expired rows are removed only by worm_audit_purge().',
TG_OP, TG_TABLE_NAME, row_identity),
HINT = 'see qbit-worm-audit docs/DESIGN.md section 4';
END;
$$;
-- changeset worm-audit:3 splitStatements:false
CREATE OR REPLACE FUNCTION public.worm_audit_lookup_reject() RETURNS trigger
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $$
BEGIN
IF TG_OP = 'UPDATE' AND NEW.id IS NOT DISTINCT FROM OLD.id AND NEW.name IS NOT DISTINCT FROM OLD.name THEN
RETURN NEW;
END IF;
RAISE EXCEPTION USING ERRCODE = 'P0001',
MESSAGE = format('worm_audit: audit rows are write-once; %s of "%s" is not allowed (id=%s). Expired rows are removed only by worm_audit_purge().',
TG_OP, TG_TABLE_NAME, OLD.id),
HINT = 'see qbit-worm-audit docs/DESIGN.md section 4';
END;
$$;
-- changeset worm-audit:4 splitStatements:false
DO $$
BEGIN
IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'worm_audit_purger') THEN
CREATE ROLE worm_audit_purger NOLOGIN;
END IF;
END;
$$;
-- changeset worm-audit:5 splitStatements:false
CREATE OR REPLACE FUNCTION public.worm_audit_purge(p_cutoff timestamptz) RETURNS void
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, public, pg_temp SET timezone = 'UTC' AS $$
DECLARE
boundary bigint;
smaller bigint;
anchor text;
audits_count bigint;
details_count bigint;
links_count bigint;
BEGIN
IF p_cutoff IS NULL OR p_cutoff > now() - interval '${retention}' THEN
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome, reason)
VALUES(now(), session_user, p_cutoff, '${retention}', 'REFUSED', 'Cutoff is within retention or null');
RETURN;
END IF;
-- Same lock as the Java chainer; table locks also exclude concurrent audit inserts.
PERFORM pg_advisory_xact_lock(86753090421);
LOCK TABLE ${audit}, ${audit_detail}, public.worm_audit_chain IN SHARE ROW EXCLUSIVE MODE;
SELECT coalesce(min(c.id) - 1, (SELECT max(id) FROM public.worm_audit_chain)) INTO boundary
FROM public.worm_audit_chain c LEFT JOIN ${audit} a ON a.id = c.audit_id
WHERE a.id IS NULL OR a.timestamp IS NULL OR a.timestamp >= p_cutoff;
-- Shrink to a prefix closed over every audit's links AND all its stored details.
LOOP
SELECT min(c.id) - 1 INTO smaller FROM public.worm_audit_chain c
WHERE c.id <= boundary AND (
EXISTS (SELECT FROM public.worm_audit_chain later WHERE later.audit_id = c.audit_id AND later.id > boundary)
OR EXISTS (SELECT FROM ${audit_detail} d WHERE d.audit_id = c.audit_id AND NOT EXISTS (
SELECT FROM public.worm_audit_chain covered WHERE covered.audit_id = c.audit_id
AND d.id::text = ANY(string_to_array(covered.detail_ids, ',')))));
EXIT WHEN smaller IS NULL;
boundary := smaller;
END LOOP;
SELECT link_hash INTO anchor FROM public.worm_audit_chain WHERE id = boundary;
IF anchor IS NULL THEN
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome, reason)
VALUES(now(), session_user, p_cutoff, '${retention}', 'REFUSED', 'No complete expired chain prefix');
RETURN;
END IF;
PERFORM set_config('worm_audit.purging', 'on', true);
DELETE FROM ${audit_detail} d WHERE EXISTS
(SELECT FROM public.worm_audit_chain c WHERE c.audit_id = d.audit_id AND c.id <= boundary);
GET DIAGNOSTICS details_count = ROW_COUNT;
-- Keep ids locally because links must be deleted before their audit headers.
WITH removed AS (DELETE FROM public.worm_audit_chain WHERE id <= boundary RETURNING audit_id),
removed_audits AS (DELETE FROM ${audit} WHERE id IN (SELECT audit_id FROM removed) RETURNING id)
SELECT (SELECT count(*) FROM removed), (SELECT count(*) FROM removed_audits) INTO links_count, audits_count;
PERFORM set_config('worm_audit.purging', 'off', true);
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome,
audits_deleted, details_deleted, links_deleted, anchor_link_id, anchor_hash)
VALUES(now(), session_user, p_cutoff, '${retention}', 'PURGED', audits_count, details_count, links_count, boundary, anchor);
END;
$$;
-- changeset worm-audit:6
DROP TRIGGER IF EXISTS worm_audit_row_guard ON ${audit};
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON ${audit}
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON ${audit};
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON ${audit}
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
REVOKE UPDATE, DELETE, TRUNCATE ON ${audit} FROM ${app_role};
GRANT SELECT, INSERT ON ${audit} TO ${app_role};
DROP TRIGGER IF EXISTS worm_audit_row_guard ON ${audit_detail};
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON ${audit_detail}
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON ${audit_detail};
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON ${audit_detail}
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
REVOKE UPDATE, DELETE, TRUNCATE ON ${audit_detail} FROM ${app_role};
GRANT SELECT, INSERT ON ${audit_detail} TO ${app_role};
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_chain;
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_chain
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_chain;
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_chain
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_chain FROM ${app_role};
GRANT SELECT, INSERT ON public.worm_audit_chain TO ${app_role};
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_checkpoint;
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_checkpoint
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_checkpoint;
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_checkpoint
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_checkpoint FROM ${app_role};
GRANT SELECT, INSERT ON public.worm_audit_checkpoint TO ${app_role};
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_export;
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_export
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_export;
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_export
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_export FROM ${app_role};
GRANT SELECT, INSERT ON public.worm_audit_export TO ${app_role};
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_verification;
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_verification
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_verification;
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_verification
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_verification FROM ${app_role};
GRANT SELECT, INSERT ON public.worm_audit_verification TO ${app_role};
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_purge;
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_purge
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_purge;
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_purge
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_purge FROM ${app_role};
GRANT SELECT, INSERT ON public.worm_audit_purge TO ${app_role};
DROP TRIGGER IF EXISTS worm_audit_row_guard ON ${audit_table};
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON ${audit_table}
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_lookup_reject();
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON ${audit_table};
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON ${audit_table}
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
REVOKE UPDATE, DELETE, TRUNCATE ON ${audit_table} FROM ${app_role};
GRANT SELECT, INSERT ON ${audit_table} TO ${app_role};
DROP TRIGGER IF EXISTS worm_audit_row_guard ON ${audit_user};
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON ${audit_user}
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_lookup_reject();
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON ${audit_user};
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON ${audit_user}
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
REVOKE UPDATE, DELETE, TRUNCATE ON ${audit_user} FROM ${app_role};
GRANT SELECT, INSERT ON ${audit_user} TO ${app_role};
-- Lookups may refresh display metadata, but never identities or names.
GRANT UPDATE (label, create_date, modify_date) ON ${audit_table} TO ${app_role};
GRANT UPDATE (create_date, modify_date) ON ${audit_user} TO ${app_role};
REVOKE ALL ON FUNCTION public.worm_audit_purge(timestamptz) FROM PUBLIC;
REVOKE ALL ON FUNCTION public.worm_audit_purge(timestamptz) FROM ${app_role};
GRANT EXECUTE ON FUNCTION public.worm_audit_purge(timestamptz) TO worm_audit_purger;
-- changeset worm-audit:7 splitStatements:false
DO $$
DECLARE
table_name text;
sequence_name text;
BEGIN
FOREACH table_name IN ARRAY ARRAY['${audit}', '${audit_detail}', '${audit_table}', '${audit_user}',
'public.worm_audit_checkpoint', 'public.worm_audit_export', 'public.worm_audit_verification', 'public.worm_audit_purge'] LOOP
sequence_name := pg_get_serial_sequence(table_name, 'id');
IF sequence_name IS NOT NULL THEN
EXECUTE format('GRANT USAGE ON SEQUENCE %s TO ${app_role}', sequence_name);
END IF;
END LOOP;
END;
$$;
-- changeset worm-audit:8 splitStatements:false
CREATE OR REPLACE FUNCTION public.worm_audit_detail_coverage() RETURNS trigger
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $$
BEGIN
-- Serialize even direct SQL appends; the GIN index makes overlap checks indexed.
PERFORM pg_advisory_xact_lock(86753090421);
IF NEW.detail_ids <> '' AND (
EXISTS (SELECT FROM unnest(string_to_array(NEW.detail_ids, ',')) AS d(id) GROUP BY d.id HAVING count(*) > 1)
OR EXISTS (SELECT FROM public.worm_audit_chain c
WHERE string_to_array(c.detail_ids, ',') && string_to_array(NEW.detail_ids, ','))) THEN
RAISE EXCEPTION USING ERRCODE = '23505', MESSAGE = 'worm_audit: audit detail is already covered by the chain';
END IF;
RETURN NEW;
END;
$$;
-- changeset worm-audit:9
CREATE INDEX IF NOT EXISTS worm_audit_chain_detail_coverage ON public.worm_audit_chain USING gin (string_to_array(detail_ids, ','));
DROP TRIGGER IF EXISTS worm_audit_detail_coverage_guard ON public.worm_audit_chain;
CREATE TRIGGER worm_audit_detail_coverage_guard BEFORE INSERT ON public.worm_audit_chain
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_detail_coverage();
@@ -0,0 +1,204 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit;
import java.nio.charset.StandardCharsets;
import java.security.KeyPairGenerator;
import java.time.Instant;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qbits.wormaudit.canonical.AuditCanonical;
import com.kingsrook.qbits.wormaudit.canonical.Jcs;
import com.kingsrook.qbits.wormaudit.chain.Hashes;
import com.kingsrook.qbits.wormaudit.checkpoint.Checkpoints;
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
import com.kingsrook.qbits.wormaudit.guard.AppendOnlyGuard;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** Tests and fixtures for canonical and crypto test.
*******************************************************************************/
class CanonicalAndCryptoTest
{
/*******************************************************************************
** Verify jcs ordering escaping and invalid types.
*******************************************************************************/
@Test
void jcsOrderingEscapingAndInvalidTypes()
{
var value = new LinkedHashMap<String, Object>();
value.put("\u20ac", "Euro");
value.put("\ud83d\ude00", "Emoji");
value.put("\ufb33", "Hebrew");
value.put("a", null);
String result = Jcs.string(value);
assertThat(result)
.isEqualTo(
"{\"a\":null,\"\u20ac\":\"Euro\",\"\ud83d\ude00\":\"Emoji\",\"\ufb33\":\"Hebrew\"}");
assertThat(Jcs.string("\"\\\b\f\n\r\t\u0001/é"))
.isEqualTo("\"\\\"\\\\\\b\\f\\n\\r\\t\\u0001/é\"");
assertThat(Jcs.string(List.of(Long.MIN_VALUE, Long.MAX_VALUE)))
.isEqualTo("[-9223372036854775808,9223372036854775807]");
assertThatThrownBy(() -> Jcs.string(1.5)).isInstanceOf(IllegalArgumentException.class);
assertThatThrownBy(() -> Jcs.string("\ud800")).isInstanceOf(IllegalArgumentException.class);
assertThatThrownBy(() -> Jcs.string("\udc00")).isInstanceOf(IllegalArgumentException.class);
assertThatThrownBy(() -> Jcs.string("\ud800x"))
.isInstanceOf(IllegalArgumentException.class);
}
/*******************************************************************************
** Verify exact canonical document.
*******************************************************************************/
@Test
void exactCanonicalDocument()
{
var audit =
new QRecord()
.withValue("id", 4211L)
.withValue("recordId", 17)
.withValue("message", "Record was Edited")
.withValue("timestamp", Instant.parse("2026-10-13T21:04:05.123456Z"));
var detail =
new QRecord()
.withValue("id", 9001L)
.withValue("fieldName", "title")
.withValue("message", "Changed Title")
.withValue("oldValue", "hmac:v1:test:old")
.withValue("newValue", "hmac:v1:test:new");
assertThat(
new String(
AuditCanonical.bytes(audit, "cms_page", "james", List.of(detail)),
StandardCharsets.UTF_8))
.isEqualTo(
"{\"audit\":{\"clientChannel\":null,\"id\":4211,\"message\":\"Record was"
+ " Edited\",\"recordId\":\"17\",\"table\":\"cms_page\",\"timestamp\":\"2026-10-13T21:04:05.123456Z\",\"user\":\"james\"},\"details\":[{\"fieldName\":\"title\",\"id\":9001,\"message\":\"Changed"
+ " Title\",\"newValue\":\"hmac:v1:test:new\",\"oldValue\":\"hmac:v1:test:old\"}],\"v\":1}");
assertThat(
new String(
AuditCanonical.bytes(
audit,
null,
null,
List.of(
new QRecord().withValue("id", 2L),
new QRecord().withValue("id", 1L))),
StandardCharsets.UTF_8))
.containsSubsequence("\"id\":1,", "\"id\":2,");
}
/*******************************************************************************
** Verify independent hash and merkle vectors.
*******************************************************************************/
@Test
void independentHashAndMerkleVectors() throws Exception
{
assertThat(Hashes.link(Hashes.GENESIS, "{}".getBytes(StandardCharsets.UTF_8)))
.isEqualTo("72f1d7c888d85ce3159bd8bbf38f3046725d3d4d6cef2b9449d9fb4b58cf7a9c");
String a = "11".repeat(32), b = "22".repeat(32), c = "33".repeat(32);
assertThat(Hashes.merkle(List.of(a))).isEqualTo(a);
assertThat(Hashes.merkle(List.of(a, b, c)))
.isEqualTo("2470b93c7954a79f41c0106caeeeae350559c2af06bd9a9ab203b163951f64f7");
assertThatThrownBy(() -> Hashes.merkle(List.of()))
.isInstanceOf(IllegalArgumentException.class);
}
/*******************************************************************************
** Verify checkpoint pinning and signing.
*******************************************************************************/
@Test
void checkpointPinningAndSigning() throws Exception
{
var keys = KeyPairGenerator.getInstance("Ed25519").generateKeyPair();
var other = KeyPairGenerator.getInstance("Ed25519").generateKeyPair();
var config =
new WormAuditQBitConfig()
.withSigningKey(
() -> new WormAuditQBitConfig.SigningKey("a", keys.getPrivate()))
.withTrustedPublicKeys(Map.of("a", keys.getPublic()));
var links =
List.of(new QRecord().withValue("id", 1L).withValue("link_hash", "11".repeat(32)));
var checkpoint = Checkpoints.sign(links, config, Instant.now());
assertThat(Checkpoints.verify(checkpoint, keys.getPublic())).isTrue();
assertThat(Checkpoints.verify(checkpoint, other.getPublic())).isFalse();
assertThat(Checkpoints.verify(checkpoint, null)).isFalse();
checkpoint.setValue("signature", "bad");
assertThat(Checkpoints.verify(checkpoint, keys.getPublic())).isFalse();
checkpoint.setValue("algorithm", "RSA");
assertThat(Checkpoints.verify(checkpoint, keys.getPublic())).isFalse();
config.withTrustedPublicKeys(Map.of());
assertThatThrownBy(() -> Checkpoints.sign(links, config, Instant.now()))
.hasMessageContaining("not pinned");
config.withTrustedPublicKeys(Map.of("a", other.getPublic()));
assertThatThrownBy(() -> Checkpoints.sign(links, config, Instant.now()))
.hasMessageContaining("does not match");
}
/*******************************************************************************
** Verify ddl is reviewed rendering and rejects injection.
*******************************************************************************/
@Test
void ddlIsReviewedRenderingAndRejectsInjection() throws Exception
{
String sql = WormAuditDdl.postgres(new WormAuditQBitConfig());
assertThat(sql)
.isEqualTo(
java.nio.file.Files.readString(
java.nio.file.Path.of("docs/ddl/worm-audit-postgres.sql")));
assertThat(sql).doesNotContain("${");
assertThat(sql).contains("GET DIAGNOSTICS stack = PG_CONTEXT", "function worm_audit_purge(timestamp with time zone)", "SET timezone = 'UTC'");
// The documented migration contract deliberately uses physical snake_case columns.
assertThat(sql).contains("OLD.audit_id", "OLD.timestamp", "NEW.name", "GRANT UPDATE (label, create_date, modify_date)");
assertThat(java.nio.file.Files.readString(java.nio.file.Path.of("README.md")))
.contains("renders table names only", "audit_detail.audit_id", "audit_user.id/name/create_date/modify_date");
assertThat(
WormAuditDdl.postgres(
new WormAuditQBitConfig()
.withRetentionYears(7)
.withAppRole("runtime")
.withCoreAuditTableNames(Map.of("audit", "history.audit"))))
.contains("\"history\".\"audit\"", "7 years", "TO \"runtime\"");
assertThatThrownBy(() -> WormAuditDdl.identifier("a; DROP TABLE audit"))
.isInstanceOf(IllegalArgumentException.class);
assertThatThrownBy(
() ->
WormAuditDdl.postgres(
new WormAuditQBitConfig().withRetentionYears(0)))
.isInstanceOf(IllegalArgumentException.class);
assertThatThrownBy(
() -> WormAuditDdl.postgres(new WormAuditQBitConfig().withAppRole("a.b")))
.isInstanceOf(IllegalArgumentException.class);
var sqlError = new java.sql.SQLException("worm_audit: rejected", "P0001");
assertThat(AppendOnlyGuard.map(new Exception(sqlError)))
.hasMessage(AppendOnlyGuard.MESSAGE);
assertThat(
AppendOnlyGuard.map(
new java.sql.SQLException("ERROR: worm_audit: rejected", "P0001")))
.hasMessage(AppendOnlyGuard.MESSAGE);
assertThat(AppendOnlyGuard.map(new java.sql.SQLException("other", "P0001")))
.hasMessage("WORM audit operation failed");
var original = new com.kingsrook.qqq.backend.core.exceptions.QException("original");
assertThat(AppendOnlyGuard.map(original)).isSameAs(original);
}
}
@@ -0,0 +1,347 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit;
import java.security.KeyPairGenerator;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
import com.kingsrook.qbits.wormaudit.chain.Hashes;
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.OLD;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.STORE;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.core;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.detail;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.remove;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.tamper;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** Tests and fixtures for chain verification test.
*******************************************************************************/
class ChainVerificationTest
{
/*******************************************************************************
** Verify signed.
*******************************************************************************/
static WormAuditQBitConfig signed() throws Exception
{
var pair = KeyPairGenerator.getInstance("Ed25519").generateKeyPair();
return (new WormAuditQBitConfig()
.withCheckpointEveryLinks(1)
.withSigningKey(() -> new WormAuditQBitConfig.SigningKey("test", pair.getPrivate()))
.withTrustedPublicKeys(Map.of("test", pair.getPublic())));
}
/*******************************************************************************
** Verify late headers details and chunk boundaries.
*******************************************************************************/
@Test
void lateHeadersDetailsAndChunkBoundaries() throws Exception
{
var config = signed().withBatchSize(1);
init(config);
audit(20);
detail(1, 20);
var writer = new WormAuditChainer(config);
assertThat(writer.chainOnce().audits()).isEqualTo(1);
audit(1);
detail(2, 1);
detail(3, 20);
assertThat(writer.chainOnce().lateDetails()).isEqualTo(1);
for(long i = 4; i <= 260; i++)
{
detail(i, 1);
}
assertThat(writer.chainOnce().lateDetails()).isEqualTo(257);
var links = STORE.all(WormAuditTables.CHAIN);
assertThat(links).extracting(r -> r.getValueLong("id")).containsExactly(1L, 2L, 3L, 4L, 5L);
assertThat(links).allMatch(r -> AuditStore.detailIds(r).size() <= 250);
var result = new WormAuditVerifier(config).verify();
assertThat(result.getBreaks()).isEmpty();
assertThat(result.getLinksChecked()).isEqualTo(5);
assertThat(result.getFromLinkId()).isEqualTo(1);
assertThat(result.getToLinkId()).isEqualTo(5);
assertThat(writer.chainOnce().links()).isZero();
}
/*******************************************************************************
** Verify overflow at initial chain and settling and elapsed checkpoint.
*******************************************************************************/
@Test
void overflowAtInitialChainAndSettlingAndElapsedCheckpoint() throws Exception
{
var config = signed().withCheckpointEveryLinks(10000).withCheckpointEveryMinutes(1);
init(config);
audit(1);
for(long i = 1; i <= 251; i++)
{
detail(i, 1);
}
var writer = new WormAuditChainer(config);
assertThat(writer.chainOnce(OLD.plusSeconds(30)).links()).isZero();
Instant now = Instant.now();
assertThat(writer.chainOnce(now).links()).isEqualTo(2);
assertThat(STORE.all(WormAuditTables.CHECKPOINT)).isEmpty();
assertThat(writer.chainOnce(now.plusSeconds(61)).checkpoints()).isEqualTo(1);
assertThat(new WormAuditVerifier(config).verify().passed()).isTrue();
config.withSigningKey(null);
assertThat(new WormAuditVerifier(config).verify().getWarnings())
.contains("no signing key configured");
}
/*******************************************************************************
** Verify detects breaks.
*******************************************************************************/
@ParameterizedTest
@ValueSource(
strings =
{
"content",
"headerDeleted",
"detailDeleted",
"inserted",
"detailInserted",
"gap",
"previous",
"reorder",
"forged",
"untrusted",
"merkle",
"head",
"duplicate",
"kind",
"lookup",
"range",
"nullCheckpoint",
"nullPrevious",
"nullTimestamp"
})
void detectsBreaks(String scenario) throws Exception
{
var config = signed();
init(config);
audit(1);
detail(1, 1);
audit(2);
detail(2, 2);
new WormAuditChainer(config).chainOnce();
String expected;
switch(scenario)
{
case "content" ->
{
tamper("auditDetail", 1, "newValue", "tampered");
expected =
"Break at link 1 (audit 1, table cms_page, record 17): content differs from"
+ " the chained hash (audit_detail 1)";
}
case "headerDeleted" ->
{
remove("audit", 1);
expected = "audit header is missing";
}
case "detailDeleted" ->
{
remove("auditDetail", 1);
expected = "audit_detail 1 is missing";
}
case "inserted" ->
{
audit(3);
expected = "audit 3 is not covered";
}
case "detailInserted" ->
{
detail(3, 1);
expected = "audit_detail 3 is not covered";
}
case "gap" ->
{
remove(WormAuditTables.CHAIN, 1);
expected = "gap";
}
case "previous" ->
{
tamper(WormAuditTables.CHAIN, 2, "prev_hash", Hashes.GENESIS);
expected = "prev-hash discontinuity";
}
case "reorder" ->
{
tamper(WormAuditTables.CHAIN, 1, "audit_id", 2L);
expected = "another audit";
}
case "forged" ->
{
tamper(WormAuditTables.CHECKPOINT, 1, "signature", "AA==");
expected = "checkpoint 1: signature does not verify";
}
case "untrusted" ->
{
config.withTrustedPublicKeys(Map.of());
expected = "untrusted key";
}
case "merkle" ->
{
tamper(WormAuditTables.CHECKPOINT, 1, "merkle_root", Hashes.GENESIS);
expected = "merkle mismatch";
}
case "head" ->
{
tamper(WormAuditTables.CHECKPOINT, 1, "head_hash", Hashes.GENESIS);
expected = "head mismatch";
}
case "duplicate" ->
{
tamper(WormAuditTables.CHAIN, 1, "detail_ids", "1,1");
expected = "audit_detail 1 is covered twice";
}
case "kind" ->
{
tamper(WormAuditTables.CHAIN, 1, "kind", "INVALID");
expected = "invalid kind";
}
case "lookup" ->
{
tamper("auditTable", 1, "name", "renamed");
expected = "content differs";
}
case "nullCheckpoint" ->
{
tamper(WormAuditTables.CHECKPOINT, 1, "signature", null);
expected = "malformed checkpoint";
}
case "nullPrevious" ->
{
tamper(WormAuditTables.CHAIN, 1, "prev_hash", null);
expected = "malformed link";
}
case "nullTimestamp" ->
{
tamper("audit", 1, "timestamp", null);
expected = "missing audit timestamp";
}
default ->
{
tamper(WormAuditTables.CHECKPOINT, 1, "link_count", 22L);
expected = "invalid link range";
}
}
var result = new WormAuditVerifier(config).verify();
assertThat(result.passed()).isFalse();
assertThat(result.getBreaks()).anyMatch(b -> b.reason().contains(expected));
}
/*******************************************************************************
** Verify selection does not run completeness and checks boundary.
*******************************************************************************/
@Test
void selectionDoesNotRunCompletenessAndChecksBoundary() throws Exception
{
var config = signed();
init(config);
audit(1);
audit(2);
new WormAuditChainer(config).chainOnce();
audit(3);
assertThat(new WormAuditVerifier(config).verify(Set.of(2L)).passed()).isTrue();
assertThat(new WormAuditVerifier(config).verify(Set.of()).getLinksChecked()).isZero();
assertThat(new WormAuditVerifier(config).verify(Set.of(99L)).getBreaks())
.anyMatch(b -> b.reason().contains("Missing selected link 99"));
tamper(WormAuditTables.CHAIN, 2, "prev_hash", Hashes.GENESIS);
assertThat(new WormAuditVerifier(config).verify(Set.of(2L)).getBreaks())
.anyMatch(b -> b.reason().contains("prev-hash"));
}
/*******************************************************************************
** Verify continues after purge anchor including empty live chain.
*******************************************************************************/
@Test
void continuesAfterPurgeAnchorIncludingEmptyLiveChain() throws Exception
{
var config = signed();
init(config);
audit(1);
detail(1, 1);
audit(2);
new WormAuditChainer(config).chainOnce();
var first = STORE.get(WormAuditTables.CHAIN, 1);
STORE.insert(
WormAuditTables.PURGE,
List.of(
new com.kingsrook.qqq.backend.core.model.data.QRecord()
.withValue("outcome", "PURGED")
.withValue("anchor_link_id", 1L)
.withValue("anchor_hash", first.getValueString("link_hash"))));
remove("auditDetail", 1);
remove("audit", 1);
remove(WormAuditTables.CHAIN, 1);
assertThat(new WormAuditVerifier(config).verify().passed()).isTrue();
var last = STORE.get(WormAuditTables.CHAIN, 2);
STORE.insert(
WormAuditTables.PURGE,
List.of(
new com.kingsrook.qqq.backend.core.model.data.QRecord()
.withValue("outcome", "PURGED")
.withValue("anchor_link_id", 2L)
.withValue("anchor_hash", last.getValueString("link_hash"))));
remove("audit", 2);
remove(WormAuditTables.CHAIN, 2);
assertThat(new WormAuditVerifier(config).verify().passed()).isTrue();
audit(3);
new WormAuditChainer(config).chainOnce();
assertThat(STORE.all(WormAuditTables.CHAIN))
.extracting(r -> r.getValueLong("id"))
.containsExactly(3L);
assertThat(new WormAuditVerifier(config).verify().passed()).isTrue();
}
/*******************************************************************************
** Findings retain bounded diagnostics while recording the complete failure count.
*******************************************************************************/
@Test
void boundedDiagnosticsPreserveTotalCount() throws Exception
{
init(new WormAuditQBitConfig());
var result = new com.kingsrook.qbits.wormaudit.verify.VerificationResult();
for(long id = 1; id <= 1200; id++)
{
result.add(id, id, "gap");
result.warn("warning " + id);
}
result.omitted(20);
assertThat(result.getBreaks()).hasSize(1000);
assertThat(result.getWarnings()).hasSize(100);
assertThat(result.getBreakCount()).isEqualTo(1220);
assertThat(result.passed()).isFalse();
assertThat(com.kingsrook.qbits.wormaudit.processes.VerificationRuns.save(result, "USER").getValueLong("break_count")).isEqualTo(1220);
}
}
@@ -0,0 +1,43 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** Tests and fixtures for docker condition test.
*******************************************************************************/
class DockerConditionTest
{
/*******************************************************************************
** Verify never skips in ci.
*******************************************************************************/
@Test
void neverSkipsInCi()
{
assertThat(RequiresDockerCondition.evaluate(false, null).isDisabled()).isTrue();
assertThat(RequiresDockerCondition.evaluate(true, "true").isDisabled()).isFalse();
assertThatThrownBy(() -> RequiresDockerCondition.evaluate(false, "false"))
.hasMessageContaining("CI is set");
}
}
@@ -0,0 +1,206 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
import java.util.Objects;
import java.util.UUID;
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
import com.kingsrook.qqq.backend.core.actions.audits.AuditAction;
import com.kingsrook.qqq.backend.core.actions.audits.AuditActionCustomizerInterface;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QSupplementalFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import org.junit.jupiter.api.Test;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.OLD;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.STORE;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.core;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.detail;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** Tests and fixtures for phi canary test.
*******************************************************************************/
class PhiCanaryTest
{
// RC.5 has AiFieldMetaData but not DataClassification; use a test-local marker.
/*******************************************************************************
** Tests and fixtures for phi.
*******************************************************************************/
public static class Phi extends QSupplementalFieldMetaData
{
/*******************************************************************************
** Verify get type.
*******************************************************************************/
public String getType()
{
return ("test-phi");
}
}
// Real RC.5 AuditAction customization point: rewrite details BEFORE insertion.
/*******************************************************************************
** Tests and fixtures for token hook.
*******************************************************************************/
public static class TokenHook implements AuditActionCustomizerInterface
{
/*******************************************************************************
** Verify customize input.
*******************************************************************************/
public void customizeInput(AuditSingleInput input)
{
for(QRecord detail : input.getDetails())
{
var field =
QContext.getQInstance()
.getTable(input.getAuditTableName())
.getField(detail.getValueString("fieldName"));
if(field.getSupplementalMetaData("test-phi") != null)
{
for(String column : List.of("oldValue", "newValue"))
{
try
{
Mac mac = Mac.getInstance("HmacSHA256");
mac.init(
new SecretKeySpec(
"fixture-key-not-production"
.getBytes(StandardCharsets.UTF_8),
"HmacSHA256"));
detail.setValue(
column,
"hmac:v1:test:"
+ Base64.getUrlEncoder()
.withoutPadding()
.encodeToString(
mac.doFinal(
detail.getValueString(column)
.getBytes(
StandardCharsets
.UTF_8))));
}
catch(Exception e)
{
throw new IllegalStateException(e);
}
}
detail.setValue("message", "Changed Diagnosis");
}
}
}
}
/*******************************************************************************
** Verify token hook keeps every audit and chain value free of canary.
*******************************************************************************/
@Test
void tokenHookKeepsEveryAuditAndChainValueFreeOfCanary() throws Exception
{
var config = ChainVerificationTest.signed();
var instance = init(config);
instance.addTable(
new QTableMetaData()
.withName("patient")
.withBackendName("memory")
.withPrimaryKeyField("id")
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withField(
new QFieldMetaData("diagnosis", QFieldType.STRING)
.withSupplementalMetaData(new Phi())));
instance.addSupplementalCustomizer(
AuditActionCustomizerInterface.CUSTOMIZER_TYPE,
new QCodeReference(TokenHook.class));
String canary = "CANARY-PHI-" + UUID.randomUUID();
var detail =
new QRecord()
.withValue("fieldName", "diagnosis")
.withValue("oldValue", canary)
.withValue("newValue", canary)
.withValue("message", "Changed Diagnosis from " + canary + " to " + canary);
new AuditAction()
.execute(
new AuditInput()
.withAuditSingleInput(
new AuditSingleInput()
.withAuditTableName("patient")
.withAuditUserName("james")
.withRecordId(17)
.withTimestamp(OLD)
.withMessage("Record was Edited")
.withDetails(List.of(detail))));
assertThat(STORE.all("audit")).hasSize(1);
assertThat(STORE.all("auditDetail")).hasSize(1);
assertThat(STORE.all("auditDetail").getFirst().getValueString("newValue"))
.startsWith("hmac:v1:test:");
new WormAuditChainer(config).chainOnce();
List<String> values = new ArrayList<>();
for(String table :
List.of(
"audit",
"auditDetail",
WormAuditTables.CHAIN,
WormAuditTables.CHECKPOINT))
{
STORE.all(table)
.forEach(
r ->
r.getValues().values().stream()
.filter(Objects::nonNull)
.map(Object::toString)
.forEach(values::add));
}
for(var link : STORE.all(WormAuditTables.CHAIN))
{
values.add(
new String(
STORE.canonical(
STORE.get("audit", link.getValueLong("audit_id")),
AuditStore.detailIds(link)),
StandardCharsets.UTF_8));
}
for(String value : values)
{
assertThat(value).doesNotContain(canary);
// Every longer substring contains a 5-character substring.
for(int start = 0; start <= canary.length() - 5; start++)
{
assertThat(value).doesNotContain(canary.substring(start, start + 5));
}
}
assertThat(new WormAuditVerifier(config).verify().passed()).isTrue();
}
}
@@ -0,0 +1,552 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.testcontainers.containers.PostgreSQLContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.STORE;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.core;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.detail;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.lookups;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** Tests and fixtures for postgres acceptance test.
*******************************************************************************/
@ExtendWith(RequiresDockerCondition.class)
@Testcontainers
class PostgresAcceptanceTest
{
@Container
static final PostgreSQLContainer<?> POSTGRES = new PostgreSQLContainer<>("postgres:16-alpine");
WormAuditQBitConfig config;
QInstance instance;
/*******************************************************************************
** Verify owner.
*******************************************************************************/
Connection owner() throws SQLException
{
return (DriverManager.getConnection(
POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword()));
}
/*******************************************************************************
** Verify app.
*******************************************************************************/
Connection app() throws SQLException
{
return (DriverManager.getConnection(POSTGRES.getJdbcUrl(), "app", "fixture"));
}
/*******************************************************************************
** Verify sql.
*******************************************************************************/
void sql(Connection c, String sql) throws SQLException
{
try(var s = c.createStatement())
{
s.execute(sql);
}
}
/*******************************************************************************
** Verify count.
*******************************************************************************/
long count(Connection c, String sql) throws SQLException
{
try(var s = c.createStatement();
var r = s.executeQuery(sql))
{
r.next();
return (r.getLong(1));
}
}
/*******************************************************************************
** Verify database.
*******************************************************************************/
@BeforeEach
void database() throws Exception
{
config = ChainVerificationTest.signed().withSettleHorizonSeconds(0);
instance = RdbmsTestSupport.instance(POSTGRES.getJdbcUrl(), "app", "fixture", config, true);
try(var c = owner())
{
sql(
c,
"DROP SCHEMA public CASCADE; CREATE SCHEMA public; GRANT USAGE ON SCHEMA public"
+ " TO PUBLIC");
sql(
c,
"DO $$ BEGIN IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='app') THEN"
+ " CREATE ROLE app LOGIN PASSWORD 'fixture'; END IF; END $$");
RdbmsTestSupport.createTables(c, instance, true);
String ddl = WormAuditDdl.postgres(config);
sql(c, ddl);
sql(c, ddl);
}
QContext.init(instance, new QSession());
lookups();
}
/*******************************************************************************
** Verify grants triggers lookup rules and spoofed flag.
*******************************************************************************/
@Test
void grantsTriggersLookupRulesAndSpoofedFlag() throws Exception
{
audit(1);
detail(1, 1);
new WormAuditChainer(config).chainOnce();
try(var c = owner();
var runtime = app())
{
assertThatThrownBy(() -> sql(runtime,
"INSERT INTO worm_audit_chain(id,kind,audit_id,detail_ids,prev_hash,link_hash,chained_at) "
+ "SELECT 99,'LATE_DETAIL',audit_id,detail_ids,prev_hash,link_hash,chained_at FROM worm_audit_chain WHERE id=1"))
.isInstanceOf(SQLException.class).satisfies(e -> assertThat(((SQLException) e).getSQLState()).isEqualTo("23505"));
for(String table :
List.of("worm_audit_export", "worm_audit_verification", "worm_audit_purge"))
{
sql(c, "INSERT INTO " + table + " DEFAULT VALUES");
}
List<String> tables =
List.of(
"audit",
"audit_detail",
"worm_audit_chain",
"worm_audit_checkpoint",
"worm_audit_export",
"worm_audit_verification",
"worm_audit_purge");
for(String table : tables)
{
assertThat(
count(
c,
"SELECT has_table_privilege('app','"
+ table
+ "','UPDATE')::int"))
.isZero();
assertThat(
count(
c,
"SELECT has_table_privilege('app','"
+ table
+ "','DELETE')::int"))
.isZero();
assertThat(
count(
c,
"SELECT has_table_privilege('app','"
+ table
+ "','TRUNCATE')::int"))
.isZero();
// PostgreSQL checks grants before firing triggers: an unprivileged app
// receives 42501. Temporarily grant DML to prove the trigger defense too.
assertThatThrownBy(() -> sql(runtime, "UPDATE " + table + " SET id=id"))
.isInstanceOf(SQLException.class)
.satisfies(
e ->
assertThat(((SQLException) e).getSQLState())
.isEqualTo("42501"));
sql(c, "GRANT UPDATE, DELETE, TRUNCATE ON " + table + " TO app");
for(String operation :
List.of(
"UPDATE " + table + " SET id=id",
"DELETE FROM " + table,
"TRUNCATE " + table))
{
assertThatThrownBy(() -> sql(runtime, operation))
.isInstanceOf(SQLException.class)
.hasMessageContaining("worm_audit:")
.satisfies(
e ->
assertThat(((SQLException) e).getSQLState())
.isEqualTo("P0001"));
}
}
sql(runtime, "SELECT set_config('worm_audit.purging','on',false)");
assertThatThrownBy(() -> sql(runtime, "DELETE FROM audit"))
.hasMessageContaining("worm_audit:");
sql(runtime, "UPDATE audit_table SET label='Allowed label'");
sql(runtime, "UPDATE audit_user SET modify_date=now()");
for(String table : List.of("audit_table", "audit_user"))
{
sql(c, "GRANT UPDATE, DELETE ON " + table + " TO app");
assertThatThrownBy(() -> sql(runtime, "UPDATE " + table + " SET name='renamed'"))
.hasMessageContaining("worm_audit:");
assertThatThrownBy(() -> sql(runtime, "UPDATE " + table + " SET id=id+1"))
.hasMessageContaining("worm_audit:");
assertThatThrownBy(() -> sql(runtime, "DELETE FROM " + table))
.hasMessageContaining("worm_audit:");
assertThatThrownBy(() -> sql(c, "TRUNCATE " + table))
.hasMessageContaining("worm_audit:");
sql(runtime, "UPDATE " + table + " SET modify_date=now()");
}
sql(runtime, "UPDATE audit_table SET label='New label'");
sql(c, WormAuditDdl.postgres(config));
assertThat(
count(
c,
"SELECT"
+ " has_function_privilege('app','worm_audit_purge(timestamptz)','EXECUTE')::int"))
.isZero();
}
}
/*******************************************************************************
** Verify retention refusal purge anchor and real backend tamper.
*******************************************************************************/
@Test
void retentionRefusalPurgeAnchorAndRealBackendTamper() throws Exception
{
audit(1);
detail(1, 1);
audit(2);
detail(2, 2);
// A live suffix remains after the expired prefix is removed.
try(var c = owner())
{
sql(
c,
"INSERT INTO audit(id,audit_table_id,audit_user_id,record_id,message,timestamp)"
+ " VALUES(3,1,1,17,'Record was Edited',now()-interval '2 minutes')");
}
new WormAuditChainer(config).chainOnce();
assertThat(new WormAuditVerifier(config).verify().getBreaks()).isEmpty();
String anchor = STORE.get(WormAuditTables.CHAIN, 2).getValueString("link_hash");
try(var c = owner())
{
sql(
c,
"DO $$ BEGIN IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='purge_login')"
+ " THEN CREATE ROLE purge_login LOGIN PASSWORD 'purger-fixture'; END IF;"
+ " END $$; GRANT worm_audit_purger TO purge_login");
}
try(var purger =
DriverManager.getConnection(
POSTGRES.getJdbcUrl(), "purge_login", "purger-fixture"))
{
sql(purger, "SELECT worm_audit_purge(now())");
assertThat(STORE.all(WormAuditTables.PURGE).getFirst().getValueString("outcome"))
.isEqualTo("REFUSED");
sql(purger, "SELECT worm_audit_purge(now()-interval '6 years')");
}
var purge = STORE.all(WormAuditTables.PURGE).getLast();
assertThat(purge.getValueString("outcome")).isEqualTo("PURGED");
assertThat(purge.getValueString("requested_by")).isEqualTo("purge_login");
assertThat(purge.getValueLong("audits_deleted")).isEqualTo(2);
assertThat(purge.getValueLong("details_deleted")).isEqualTo(2);
assertThat(purge.getValueLong("links_deleted")).isEqualTo(2);
assertThat(purge.getValueString("anchor_hash")).isEqualTo(anchor);
assertThat(new WormAuditVerifier(config).verify().getBreaks()).isEmpty();
detail(3, 3);
new WormAuditChainer(config).chainOnce();
try(var c = owner())
{
sql(
c,
"ALTER TABLE audit_detail DISABLE TRIGGER ALL; UPDATE audit_detail SET"
+ " new_value='tampered' WHERE id=3; ALTER TABLE audit_detail ENABLE"
+ " TRIGGER ALL");
}
assertThat(new WormAuditVerifier(config).verify().getBreaks())
.anyMatch(
b ->
b.reason()
.contains(
"Break at link 4 (audit 3, table cms_page,"
+ " record 17)")
&& b.reason().contains("audit_detail 3"));
}
/*******************************************************************************
** Verify prefix cannot split an audit and advisory lock skips tick.
*******************************************************************************/
@Test
void prefixCannotSplitAnAuditAndAdvisoryLockSkipsTick() throws Exception
{
// This retention fixture deliberately polls a detail for a six-year-old header.
config.withLateDetailLookbackHours(24 * 365 * 10);
audit(1);
new WormAuditChainer(config).chainOnce();
try(var c = owner())
{
sql(
c,
"INSERT INTO audit(id,audit_table_id,audit_user_id,record_id,message,timestamp)"
+ " VALUES(2,1,1,17,'Record was Edited',now()-interval '2 minutes')");
}
new WormAuditChainer(config).chainOnce();
detail(1, 1);
new WormAuditChainer(config).chainOnce();
try(var c = owner())
{
sql(c, "SELECT worm_audit_purge(now()-interval '6 years')");
assertThat(STORE.all(WormAuditTables.PURGE).getLast().getValueString("outcome"))
.isEqualTo("REFUSED");
assertThat(STORE.all(WormAuditTables.CHAIN)).hasSize(3);
c.setAutoCommit(false);
sql(c, "SELECT pg_advisory_xact_lock(" + WormAuditChainer.ADVISORY_LOCK + ")");
assertThat(new WormAuditChainer(config).chainOnce().skipped()).isTrue();
c.rollback();
}
}
/*******************************************************************************
** Verify eight writers and three chainers with late commits.
*******************************************************************************/
@Test
void eightWritersAndThreeChainersWithLateCommits() throws Exception
{
var done = new AtomicBoolean(false);
var executor = Executors.newFixedThreadPool(11);
List<Future<?>> writers = new ArrayList<>();
List<Future<?>> chainers = new ArrayList<>();
try
{
for(int thread = 0; thread < 8; thread++)
{
final int writer = thread;
writers.add(
executor.submit(
() ->
{
try(var c = app())
{
for(int i = 0; i < 8; i++)
{
long id = 1000L + writer * 100 + i;
c.setAutoCommit(false);
sql(
c,
"INSERT INTO"
+ " audit(id,audit_table_id,audit_user_id,record_id,message,timestamp)"
+ " VALUES("
+ id
+ ",1,1,17,'Record was"
+ " Edited',now()-interval '2"
+ " minutes')");
if(i % 2 == 0)
{
c.commit();
}
Thread.sleep(5L * (writer + 1));
sql(
c,
"INSERT INTO"
+ " audit_detail(id,audit_id,field_name,message,old_value,new_value)"
+ " VALUES("
+ id
+ ","
+ id
+ ",'title','Changed"
+ " Title','old','new')");
c.commit();
}
}
catch(Exception e)
{
throw new RuntimeException(e);
}
}));
}
for(int i = 0; i < 3; i++)
{
chainers.add(
executor.submit(
() ->
{
QContext.init(instance, new QSession());
try
{
do
{
new WormAuditChainer(config).chainOnce();
Thread.sleep(10);
} while(!done.get());
}
catch(Exception e)
{
throw new RuntimeException(e);
}
}));
}
for(var writer : writers)
{
writer.get(90, TimeUnit.SECONDS);
}
done.set(true);
for(var chainer : chainers)
{
chainer.get(90, TimeUnit.SECONDS);
}
}
finally
{
done.set(true);
executor.shutdownNow();
}
new WormAuditChainer(config).chainOnce();
var links = STORE.all(WormAuditTables.CHAIN);
for(int i = 0; i < links.size(); i++)
{
assertThat(links.get(i).getValueLong("id")).isEqualTo(i + 1L);
}
assertThat(links.stream().filter(r -> "AUDIT".equals(r.getValueString("kind"))).count())
.isEqualTo(64);
assertThat(links.stream().flatMap(r -> AuditStore.detailIds(r).stream()).toList())
.hasSize(64)
.doesNotHaveDuplicates();
assertThat(new WormAuditVerifier(config).verify().getBreaks()).isEmpty();
}
/*******************************************************************************
** Even the function owner cannot authorize direct deletes by setting a GUC.
*******************************************************************************/
@Test
void ownerCannotSpoofPurgeStack() throws Exception
{
audit(1);
detail(1, 1);
try(var connection = owner())
{
sql(connection, "ALTER TABLE audit ALTER COLUMN timestamp TYPE timestamp without time zone USING timestamp AT TIME ZONE 'UTC'");
sql(connection, "INSERT INTO audit(id,audit_table_id,audit_user_id,timestamp) VALUES"
+ "(2,1,1,(now() AT TIME ZONE 'UTC')-interval '6 years'-interval '1 hour')");
}
new WormAuditChainer(config).chainOnce();
try(var connection = owner())
{
sql(connection, "SELECT set_config('worm_audit.purging','on',false)");
for(String table : List.of("audit", "audit_detail", "worm_audit_chain"))
{
assertThatThrownBy(() -> sql(connection, "DELETE FROM " + table + " WHERE id=1"))
.isInstanceOf(SQLException.class).hasMessageContaining("worm_audit:");
}
assertThat(count(connection, "SELECT count(*) FROM worm_audit_purge")).isZero();
// This succeeds only if the PG16 context text matches the stack guard.
sql(connection, "SET timezone='America/Chicago'");
sql(connection, "SELECT worm_audit_purge(now()-interval '6 years')");
assertThat(count(connection, "SELECT count(*) FROM audit")).isZero();
assertThat(count(connection, "SELECT count(*) FROM worm_audit_purge WHERE outcome='PURGED'")).isEqualTo(1);
}
}
/*******************************************************************************
** A UTC without-tz audit column settles identically in a Chicago JVM/session.
*******************************************************************************/
@Test
void chainerUsesUtcCutoffInChicago() throws Exception
{
var original = java.util.TimeZone.getDefault();
config.withSettleHorizonSeconds(60);
try(var connection = owner())
{
sql(connection, "ALTER TABLE audit ALTER COLUMN timestamp TYPE timestamp without time zone USING timestamp AT TIME ZONE 'UTC'");
sql(connection, "INSERT INTO audit(id,audit_table_id,audit_user_id,timestamp) VALUES"
+ "(1,1,1,TIMESTAMP '2026-10-13 12:00:00'),(2,1,1,TIMESTAMP '2026-10-13 12:01:30')");
// New runtime connections inherit this non-UTC session setting.
sql(connection, "ALTER ROLE app SET timezone='America/Chicago'");
java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("America/Chicago"));
assertThat(new WormAuditChainer(config).chainOnce(java.time.Instant.parse("2026-10-13T12:02:00Z")).audits()).isEqualTo(1);
assertThat(STORE.all(WormAuditTables.CHAIN)).extracting(r -> r.getValueLong("audit_id")).containsExactly(1L);
java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("UTC"));
assertThat(new WormAuditVerifier(config).verify(java.util.Set.of(1L)).getBreaks()).isEmpty();
}
finally
{
java.util.TimeZone.setDefault(original);
try(var connection = owner())
{
sql(connection, "ALTER ROLE app RESET timezone");
}
}
}
/*******************************************************************************
** PostgreSQL containment polling includes the lower window boundary only.
*******************************************************************************/
@Test
void postgresLateDetailWindowAndCompleteness() throws Exception
{
config.withSettleHorizonSeconds(60);
var now = java.time.Instant.now().truncatedTo(java.time.temporal.ChronoUnit.SECONDS);
var cutoff = now.minusSeconds(60);
try(var connection = owner(); var statement = connection.prepareStatement(
"INSERT INTO audit(id,audit_table_id,audit_user_id,timestamp) VALUES(?,1,1,?)"))
{
for(int id = 1; id <= 4; id++)
{
long seconds = switch(id)
{
case 1 -> 86401;
case 2 -> 86400;
case 3 -> 1;
default -> 0;
};
statement.setInt(1, id);
statement.setObject(2, cutoff.minusSeconds(seconds).atOffset(java.time.ZoneOffset.UTC));
statement.executeUpdate();
}
}
var writer = new WormAuditChainer(config);
assertThat(writer.chainOnce(now).audits()).isEqualTo(3);
for(long id = 1; id <= 4; id++)
{
detail(id, id);
}
assertThat(writer.chainOnce(now).lateDetails()).isEqualTo(2);
assertThat(STORE.all(WormAuditTables.CHAIN).stream().flatMap(r -> AuditStore.detailIds(r).stream()).toList()).containsExactly(2L, 3L);
assertThat(new WormAuditVerifier(config).verify().getBreaks()).anyMatch(b -> b.reason().contains("audit_detail 1 is not covered"));
}
}
@@ -0,0 +1,234 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
import com.kingsrook.qbits.wormaudit.processes.VerificationRuns;
import com.kingsrook.qbits.wormaudit.processes.WormAuditProcesses;
import com.kingsrook.qqq.backend.core.alerts.AlertResolution;
import com.kingsrook.qqq.backend.core.alerts.AlertSignal;
import com.kingsrook.qqq.backend.core.alerts.AlertSinkInterface;
import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunInput;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunOutput;
import org.junit.jupiter.api.Test;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.STORE;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.core;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.detail;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.tamper;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** Tests and fixtures for processes and alerts test.
*******************************************************************************/
class ProcessesAndAlertsTest
{
/*******************************************************************************
** Tests and fixtures for sink.
*******************************************************************************/
public static class Sink implements AlertSinkInterface
{
static final List<AlertSignal> raised = new ArrayList<>();
static final List<AlertResolution> resolved = new ArrayList<>();
static final List<String> beats = new ArrayList<>();
/*******************************************************************************
** Verify raise.
*******************************************************************************/
public void raise(AlertSignal signal)
{
raised.add(signal);
}
/*******************************************************************************
** Verify resolve.
*******************************************************************************/
public void resolve(AlertResolution resolution)
{
resolved.add(resolution);
}
/*******************************************************************************
** Verify heartbeat.
*******************************************************************************/
public void heartbeat(String key, Instant at)
{
beats.add(key);
}
}
/*******************************************************************************
** Verify daily raise resolve heartbeat and user result.
*******************************************************************************/
@Test
void dailyRaiseResolveHeartbeatAndUserResult() throws Exception
{
var config = ChainVerificationTest.signed();
var instance = init(config);
Sink.raised.clear();
Sink.resolved.clear();
Sink.beats.clear();
AlertsInstanceMetaData.ofOrWithNew(instance).withSink(new QCodeReference(Sink.class));
audit(1);
detail(1, 1);
new WormAuditProcesses.ChainTick()
.run(new RunBackendStepInput(), new RunBackendStepOutput());
tamper("auditDetail", 1, "newValue", "broken");
var daily = new WormAuditProcesses.Daily();
var output = new RunBackendStepOutput();
daily.run(new RunBackendStepInput(), output);
assertThat(output.getValueString("outcome")).isEqualTo("FAIL");
assertThat(Sink.raised).hasSize(1);
assertThat(Sink.raised.getFirst().getAlertTypeName()).isEqualTo("worm_audit.chain.break");
assertThat(Sink.raised.getFirst().getDedupKey()).isEqualTo("chain");
assertThat(Sink.raised.getFirst().getRecordRefs().getFirst().getTableName())
.isEqualTo(WormAuditTables.VERIFICATION);
assertThat(Sink.raised.getFirst().getTitle()).contains("Break at link 1");
var first = STORE.all(WormAuditTables.VERIFICATION).getFirst();
assertThat(first.getValueLong("first_break_link_id")).isEqualTo(1);
assertThat(first.getValueLong("first_break_audit_id")).isEqualTo(1);
assertThat(first.getValueString("trigger")).isEqualTo("SCHEDULE");
tamper("auditDetail", 1, "newValue", "new");
daily.run(new RunBackendStepInput(), new RunBackendStepOutput());
assertThat(Sink.resolved).hasSize(1);
assertThat(Sink.beats)
.containsExactly("worm_audit.verify.daily", "worm_audit.verify.daily");
assertThat(STORE.all(WormAuditTables.VERIFICATION)).hasSize(2);
assertThat(VerificationRuns.run(null, "USER").passed()).isTrue();
assertThat(Sink.beats).hasSize(2);
}
/*******************************************************************************
** Verify review and load across pages including failures and empty scope.
*******************************************************************************/
@Test
void reviewAndLoadAcrossPagesIncludingFailuresAndEmptyScope() throws Exception
{
var config = new WormAuditQBitConfig();
init(config);
audit(1);
audit(2);
new WormAuditChainer(config).chainOnce();
var links = STORE.all(WormAuditTables.CHAIN);
var review = new WormAuditProcesses.Review();
var load = new WormAuditProcesses.Load();
var output = new RunBackendStepOutput();
assertThat(load.getProcessSummary(output, true)).isEmpty();
for(var link : links)
{
var input = new RunBackendStepInput().withRecords(List.of(link));
review.runOnePage(input, output);
load.runOnePage(input, output);
}
assertThat(review.getProcessSummary(output, false).toString())
.contains("2 chain links will be verified", "0 checkpoints");
load.postRun(
new BackendStepPostRunInput(new RunBackendStepInput()),
new BackendStepPostRunOutput(output));
assertThat(STORE.all(WormAuditTables.VERIFICATION)).hasSize(1);
assertThat(load.getProcessSummary(output, true).toString())
.contains("2 links verified", "no signing key configured");
audit(3);
var fullWithEmptyFilter = new WormAuditProcesses.Load();
fullWithEmptyFilter.runOnePage(new RunBackendStepInput().withRecords(links), output);
fullWithEmptyFilter.postRun(new BackendStepPostRunInput(new RunBackendStepInput().addValue("queryFilterJson", "{}")),
new BackendStepPostRunOutput(output));
assertThat(fullWithEmptyFilter.getProcessSummary(output, true).toString()).contains("audit 3 is not covered by the chain");
tamper("audit", 1, "message", "changed");
var selected = new WormAuditProcesses.Load();
selected.runOnePage(
new RunBackendStepInput().withRecords(List.of(links.getFirst())), output);
selected.postRun(
new BackendStepPostRunInput(new RunBackendStepInput().addValue("recordIds", "1")),
new BackendStepPostRunOutput(output));
assertThat(selected.getProcessSummary(output, true).toString())
.contains("ERROR", "content differs");
init(config);
var empty = new WormAuditProcesses.Load();
empty.postRun(
new BackendStepPostRunInput(new RunBackendStepInput()),
new BackendStepPostRunOutput(output));
assertThat(empty.getProcessSummary(output, true).toString()).contains("0 links verified");
assertThat(new WormAuditProcesses.Review().getProcessSummary(output, false).toString())
.contains("links none");
}
/*******************************************************************************
** Full loads retain no id collection; review summarizes disjoint streamed pages once.
*******************************************************************************/
@Test
void reviewRangesAndSelectionOnlyIds() throws Exception
{
var config = ChainVerificationTest.signed().withCheckpointEveryLinks(2);
init(config);
for(long id = 1; id <= 4; id++)
{
audit(id);
}
new WormAuditChainer(config).chainOnce();
var links = STORE.all(WormAuditTables.CHAIN);
var review = new WormAuditProcesses.Review();
var full = new WormAuditProcesses.Load();
var selected = new WormAuditProcesses.Load();
var output = new RunBackendStepOutput();
var ids = WormAuditProcesses.Load.class.getDeclaredField("ids");
ids.setAccessible(true);
for(var link : links)
{
var input = new RunBackendStepInput().withRecords(List.of(link));
review.runOnePage(input, output);
full.runOnePage(input, output);
assertThat(ids.get(full)).isNull();
}
String summary = review.getProcessSummary(output, false).toString();
assertThat(summary).contains("4 chain links", "links 1–4", "2 checkpoints");
// A second call reuses the count; later inserts cannot silently change Review.
STORE.insert(WormAuditTables.CHECKPOINT, List.of(new com.kingsrook.qqq.backend.core.model.data.QRecord()
.withValue("from_link_id", 1L).withValue("to_link_id", 4L)));
assertThat(review.getProcessSummary(output, false).toString()).isEqualTo(summary);
selected.runOnePage(new RunBackendStepInput().addValue("recordIds", "2").withRecords(List.of(links.get(1))), output);
assertThat(ids.get(selected)).isEqualTo(java.util.Set.of(2L));
}
/*******************************************************************************
** Docker dependencies must not enable debug wire logs during tests.
*******************************************************************************/
@Test
void dockerLoggersAreQuiet()
{
for(String name : List.of("org.testcontainers", "com.github.dockerjava", "org.apache.hc", "tc"))
{
assertThat(org.apache.logging.log4j.LogManager.getLogger(name).isDebugEnabled()).isFalse();
assertThat(org.apache.logging.log4j.LogManager.getLogger(name).isWarnEnabled()).isTrue();
}
}
}
@@ -0,0 +1,212 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit;
import java.sql.DriverManager;
import java.util.Map;
import java.util.UUID;
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import org.junit.jupiter.api.Test;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.STORE;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.core;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.detail;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.lookups;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** Tests and fixtures for rdbms chain test.
*******************************************************************************/
class RdbmsChainTest
{
/*******************************************************************************
** Verify real qqq rdbms actions anti join rollback and tamper.
*******************************************************************************/
@Test
void realQqqRdbmsActionsAntiJoinRollbackAndTamper() throws Exception
{
String url =
"jdbc:h2:mem:worm_"
+ UUID.randomUUID()
+ ";DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE";
var config = ChainVerificationTest.signed();
var instance = RdbmsTestSupport.instance(url, "sa", "", config, false);
try(var connection = DriverManager.getConnection(url, "sa", ""))
{
RdbmsTestSupport.createTables(connection, instance, false);
QContext.init(instance, new QSession());
lookups();
audit(1);
detail(1, 1);
var writer = new WormAuditChainer(config);
assertThat(writer.chainOnce().links()).isEqualTo(1);
assertThat(writer.chainOnce().links()).isZero();
assertThat(new WormAuditVerifier(config).verify().getBreaks()).isEmpty();
// Fail signing after append, verifying the real RDBMS transaction rolls back.
var pins = config.getTrustedPublicKeys();
config.withTrustedPublicKeys(Map.of());
audit(2);
assertThatThrownBy(writer::chainOnce)
.hasMessageContaining("WORM audit operation failed");
assertThat(STORE.all(WormAuditTables.CHAIN)).hasSize(1);
config.withTrustedPublicKeys(pins);
assertThat(writer.chainOnce().links()).isEqualTo(1);
assertThat(STORE.all(WormAuditTables.CHAIN))
.extracting(r -> r.getValueLong("id"))
.containsExactly(1L, 2L);
connection
.createStatement()
.execute("UPDATE audit_detail SET new_value='changed' WHERE id=1");
assertThat(new WormAuditVerifier(config).verify().getBreaks())
.anyMatch(b -> b.reason().contains("audit_detail 1"));
}
}
/*******************************************************************************
** Exercise multiple keyset pages, capped checkpoints and bounded SQL query counts.
*******************************************************************************/
@Test
void batchesPagesAndCheckpointRanges() throws Exception
{
String url = "jdbc:h2:mem:pages_" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE";
var config = ChainVerificationTest.signed().withBatchSize(1500).withCheckpointEveryLinks(400).withCheckpointEveryMinutes(1);
var instance = RdbmsTestSupport.instance(url, "sa", "", config, false);
try(var connection = DriverManager.getConnection(url, "sa", ""); var statement = connection.createStatement())
{
RdbmsTestSupport.createTables(connection, instance, false);
QContext.init(instance, new QSession());
lookups();
statement.execute("INSERT INTO audit(id,audit_table_id,audit_user_id,record_id,timestamp) SELECT x,1,1,17,CURRENT_TIMESTAMP - INTERVAL '2' MINUTE FROM SYSTEM_RANGE(1,1005) series(x)");
statement.execute("INSERT INTO audit_detail(id,audit_id,field_name,new_value) SELECT x,x,'title','new' FROM SYSTEM_RANGE(1,1005) series(x)");
statement.execute("SET QUERY_STATISTICS TRUE");
var writer = new WormAuditChainer(config);
var now = java.time.Instant.now();
var counts = writer.chainOnce(now);
assertThat(counts.links()).isEqualTo(1005);
assertThat(counts.checkpoints()).isEqualTo(2);
assertSelectCount(connection, 100);
assertThat(STORE.last(WormAuditTables.CHECKPOINT).getValueInstant("created_at")).isCloseTo(now, org.assertj.core.api.Assertions.within(1, java.time.temporal.ChronoUnit.SECONDS));
statement.execute("SET QUERY_STATISTICS FALSE");
statement.execute("SET QUERY_STATISTICS TRUE");
assertThat(writer.chainOnce(now.plusSeconds(61)).checkpoints()).isEqualTo(1);
assertThat(new WormAuditVerifier(config).verify().getBreaks()).isEmpty();
assertSelectCount(connection, 100);
assertThat(STORE.all(WormAuditTables.CHECKPOINT)).extracting(r -> r.getValueLong("link_count")).containsExactly(400L, 400L, 205L);
statement.execute("UPDATE audit_detail SET new_value='changed' WHERE id=1001");
assertThat(new WormAuditVerifier(config).verify().getBreaks()).anyMatch(b -> b.reason().contains("audit_detail 1,001"));
statement.execute("DELETE FROM worm_audit_chain WHERE id=1000");
assertThat(new WormAuditVerifier(config).verify().getBreaks()).anyMatch(b -> b.reason().contains("gap"));
}
}
/*******************************************************************************
** Assert actual query executions stay proportional to batches, not row count.
*******************************************************************************/
private void assertSelectCount(java.sql.Connection connection, int limit) throws Exception
{
try(var statement = connection.createStatement(); var rows = statement.executeQuery(
"SELECT COALESCE(SUM(EXECUTION_COUNT),0) FROM INFORMATION_SCHEMA.QUERY_STATISTICS WHERE UPPER(SQL_STATEMENT) LIKE 'SELECT%'"))
{
rows.next();
assertThat(rows.getLong(1)).isLessThan(limit);
}
}
/*******************************************************************************
** Poll only the lookback window and report older omissions with exact bounded ids.
*******************************************************************************/
@Test
void lateWindowAndCompletenessOffenderLimit() throws Exception
{
String url = "jdbc:h2:mem:window_" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE";
var config = new WormAuditQBitConfig().withBatchSize(500);
var instance = RdbmsTestSupport.instance(url, "sa", "", config, false);
try(var connection = DriverManager.getConnection(url, "sa", ""); var statement = connection.createStatement())
{
RdbmsTestSupport.createTables(connection, instance, false);
QContext.init(instance, new QSession());
lookups();
audit(1);
statement.execute("INSERT INTO audit(id,audit_table_id,audit_user_id,timestamp) VALUES(2,1,1,CURRENT_TIMESTAMP - INTERVAL '2' MINUTE)");
var writer = new WormAuditChainer(config);
writer.chainOnce();
detail(1, 1);
detail(2, 2);
assertThat(writer.chainOnce().lateDetails()).isEqualTo(1);
assertThat(new WormAuditVerifier(config).verify().getBreaks()).anyMatch(b -> b.reason().contains("audit_detail 1 is not covered"));
statement.execute("INSERT INTO worm_audit_chain SELECT 99,kind,audit_id,detail_ids,prev_hash,link_hash,chained_at FROM worm_audit_chain WHERE id=3");
assertThat(new WormAuditVerifier(config).verify().getBreaks()).anyMatch(b -> b.reason().contains("audit_detail 2 is covered twice"));
statement.execute("DELETE FROM worm_audit_chain WHERE id=99");
statement.execute("INSERT INTO audit_detail(id,audit_id) SELECT x,1 FROM SYSTEM_RANGE(3,107) series(x)");
var result = new WormAuditVerifier(config).verify();
assertThat(result.getBreakCount()).isEqualTo(106);
assertThat(result.getBreaks()).hasSize(100);
assertThat(result.getBreaks().getFirst().reason()).contains("audit_detail 1 is not covered");
assertThat(result.getBreaks().getLast().reason()).contains("audit_detail 101 is not covered");
assertThat(result.getWarnings()).anyMatch(w -> w.contains("106 coverage offenders"));
statement.execute("INSERT INTO audit(id,timestamp) SELECT x,CURRENT_TIMESTAMP - INTERVAL '2' MINUTE FROM SYSTEM_RANGE(2000,2104) series(x)");
result = new WormAuditVerifier(config).verify();
assertThat(result.getBreakCount()).isEqualTo(211);
assertThat(result.getBreaks()).hasSize(200);
assertThat(result.getBreaks().getFirst().reason()).contains("audit 2,000 is not covered");
assertThat(result.getBreaks().get(99).reason()).contains("audit 2,099 is not covered");
}
}
/*******************************************************************************
** Cutoffs are UTC local values for timestamp columns, independent of JVM zone.
*******************************************************************************/
@Test
void withoutTimezoneCutoffInChicago() throws Exception
{
var original = java.util.TimeZone.getDefault();
String url = "jdbc:h2:mem:timezone_" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE";
var config = new WormAuditQBitConfig();
var instance = RdbmsTestSupport.instance(url, "sa", "", config, false);
try(var connection = DriverManager.getConnection(url, "sa", ""); var statement = connection.createStatement())
{
RdbmsTestSupport.createTables(connection, instance, false);
statement.execute("ALTER TABLE audit ALTER COLUMN timestamp timestamp without time zone");
QContext.init(instance, new QSession());
lookups();
statement.execute("INSERT INTO audit(id,audit_table_id,audit_user_id,timestamp) VALUES(1,1,1,TIMESTAMP '2026-10-13 12:00:00'),(2,1,1,TIMESTAMP '2026-10-13 12:01:30')");
java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("America/Chicago"));
assertThat(new WormAuditChainer(config).chainOnce(java.time.Instant.parse("2026-10-13T12:02:00Z")).audits()).isEqualTo(1);
assertThat(STORE.all(WormAuditTables.CHAIN)).extracting(r -> r.getValueLong("audit_id")).containsExactly(1L);
java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("UTC"));
assertThat(new WormAuditVerifier(config).verify(java.util.Set.of(1L)).getBreaks()).isEmpty();
}
finally
{
java.util.TimeZone.setDefault(original);
}
}
}
@@ -0,0 +1,157 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit;
import java.sql.Connection;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.module.rdbms.jdbc.SimpleConnectionProvider;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails;
import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy;
/*******************************************************************************
** Tests and fixtures for rdbms test support.
*******************************************************************************/
class RdbmsTestSupport
{
// RC.5 supplies the RDBMS actions and a pluggable strategy. Hosts supply their
// PostgreSQL quoting strategy; this fixture uses the core actions unchanged.
/*******************************************************************************
** Tests and fixtures for postgres strategy.
*******************************************************************************/
public static class PostgresStrategy extends BaseRDBMSActionStrategy
{
/*******************************************************************************
** Verify get identifier quote string.
*******************************************************************************/
@Override
public String getIdentifierQuoteString()
{
return ("\"");
}
/*******************************************************************************
** Verify get insert default values clause.
*******************************************************************************/
@Override
public String getInsertDefaultValuesClause()
{
return ("DEFAULT VALUES");
}
}
/*******************************************************************************
** Verify instance.
*******************************************************************************/
static QInstance instance(
String url, String user, String password, WormAuditQBitConfig config, boolean postgres)
throws Exception
{
com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager.resetConnectionProviders();
var backend =
new RDBMSBackendMetaData()
.withName("db")
.withVendor(postgres ? "postgresql" : "h2")
.withJdbcUrl(url)
.withJdbcDriverClassName(
postgres ? "org.postgresql.Driver" : "org.h2.Driver")
.withUsername(user)
.withPassword(password)
.withConnectionProvider(new QCodeReference(SimpleConnectionProvider.class));
if(postgres)
{
backend.withActionStrategyCodeReference(new QCodeReference(PostgresStrategy.class));
}
QInstance instance = WormAuditTestSupport.core(backend);
for(var table : instance.getTables().values())
{
table.withBackendDetails(
new RDBMSTableBackendDetails().withTableName(snake(table.getName())));
table.getFields().values().forEach(f -> f.withBackendName(snake(f.getName())));
}
new WormAuditQBitProducer().withQBitConfig(config.withBackendName("db")).produce(instance);
return (instance);
}
/*******************************************************************************
** Verify snake.
*******************************************************************************/
static String snake(String value)
{
return (value.replaceAll("([a-z])([A-Z])", "$1_$2").toLowerCase(Locale.ROOT));
}
/*******************************************************************************
** Verify create tables.
*******************************************************************************/
static void createTables(Connection connection, QInstance instance, boolean coreOnly)
throws Exception
{
for(var table : instance.getTables().values())
{
if(coreOnly && table.getName().startsWith("worm_audit_"))
{
continue;
}
List<String> columns = new ArrayList<>();
for(var field : table.getFields().values())
{
String type =
switch(field.getType())
{
case INTEGER -> "integer";
case LONG -> "bigint";
case DATE_TIME -> "timestamp(6) with time zone";
default -> "text";
};
if(field.getName().equals("id"))
{
type +=
table.getName().equals("worm_audit_chain")
? " PRIMARY KEY"
: " GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY";
}
columns.add(
WormAuditDdl.identifier(
field.getBackendName() == null
? field.getName()
: field.getBackendName())
+ " "
+ type);
}
String name = ((RDBMSTableBackendDetails) table.getBackendDetails()).getTableName();
try(var statement = connection.createStatement())
{
statement.execute(
"CREATE TABLE "
+ WormAuditDdl.identifier(name)
+ " ("
+ String.join(",", columns)
+ ")");
}
}
}
}
@@ -0,0 +1,125 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
import com.kingsrook.qbits.wormaudit.model.LinkRangeWidget;
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
import com.kingsrook.qbits.wormaudit.processes.VerificationRuns;
import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeEvent;
import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeListenerInterface;
import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeType;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChildRecordListData;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import org.junit.jupiter.api.Test;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** Tests and fixtures for record integration test.
*******************************************************************************/
class RecordIntegrationTest
{
/*******************************************************************************
** Tests and fixtures for event sink.
*******************************************************************************/
public static class EventSink implements RecordChangeListenerInterface
{
static List<String> tables = new ArrayList<>();
/*******************************************************************************
** Verify applies to.
*******************************************************************************/
public boolean appliesTo(String table, RecordChangeType type)
{
return (List.of(WormAuditTables.VERIFICATION, WormAuditTables.EXPORT).contains(table)
&& type == RecordChangeType.INSERT);
}
/*******************************************************************************
** Verify on records changed.
*******************************************************************************/
public void onRecordsChanged(RecordChangeEvent event)
{
tables.add(event.getTableName());
}
}
/*******************************************************************************
** Verify checkpoint child list uses range and read permissions.
*******************************************************************************/
@Test
void checkpointChildListUsesRangeAndReadPermissions() throws Exception
{
var config = ChainVerificationTest.signed().withCheckpointEveryLinks(2);
var instance = init(config);
audit(1);
audit(2);
new WormAuditChainer(config).chainOnce();
var input =
new RenderWidgetInput()
.withWidgetMetaData(
instance.getWidget(WormAuditTables.CHECKPOINT + "_links"));
input.setQueryParams(Map.of("id", "1"));
assertThatThrownBy(() -> new LinkRangeWidget().render(input)).isInstanceOf(Exception.class);
QContext.getQSession()
.withPermissions("worm_audit_checkpoint.read", "worm_audit_chain.read");
var data = (ChildRecordListData) new LinkRangeWidget().render(input).getWidgetData();
assertThat(data.getTotalRows()).isEqualTo(2);
assertThat(data.getQueryOutput().getRecords()).hasSize(2);
assertThat(data.getViewAllLink()).contains("filter=");
input.setQueryParams(Map.of("id", "999"));
assertThatThrownBy(() -> new LinkRangeWidget().render(input))
.hasMessageContaining("was not found");
assertThat(
instance.getTable(WormAuditTables.EXPORT)
.getField("from_checkpoint_id")
.getPossibleValueSourceName())
.isEqualTo(WormAuditTables.CHECKPOINT);
assertThat(
instance.getTable(WormAuditTables.VERIFICATION)
.getField("export_id")
.getPossibleValueSourceName())
.isEqualTo(WormAuditTables.EXPORT);
}
/*******************************************************************************
** Verify verification inserts reach core event spi for host esb publications.
*******************************************************************************/
@Test
void verificationInsertsReachCoreEventSpiForHostEsbPublications() throws Exception
{
var instance = init(new WormAuditQBitConfig());
EventSink.tables.clear();
instance.withRecordChangeListener(new QCodeReference(EventSink.class));
audit(1);
new WormAuditChainer(VerificationRuns.config()).chainOnce();
VerificationRuns.run(null, "USER");
assertThat(EventSink.tables).containsExactly(WormAuditTables.VERIFICATION);
}
}
@@ -0,0 +1,67 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit;
import org.junit.jupiter.api.extension.ConditionEvaluationResult;
import org.junit.jupiter.api.extension.ExecutionCondition;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.testcontainers.DockerClientFactory;
/*******************************************************************************
** Tests and fixtures for requires docker condition.
*******************************************************************************/
public class RequiresDockerCondition implements ExecutionCondition
{
/*******************************************************************************
** Verify evaluate execution condition.
*******************************************************************************/
@Override
public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context)
{
boolean available;
try
{
available = DockerClientFactory.instance().isDockerAvailable();
}
catch(Exception | LinkageError e)
{
available = false;
}
return (evaluate(available, System.getenv("CI")));
}
/*******************************************************************************
** Verify evaluate.
*******************************************************************************/
static ConditionEvaluationResult evaluate(boolean available, String ci)
{
if(available)
{
return (ConditionEvaluationResult.enabled("Docker is available"));
}
if(ci != null)
{
throw new IllegalStateException("Docker is required when CI is set");
}
return (ConditionEvaluationResult.disabled(
"Docker unavailable; coordinator must run PostgreSQL acceptance tests"));
}
}
@@ -0,0 +1,163 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit;
import java.util.List;
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
import com.kingsrook.qbits.wormaudit.processes.WormAuditProcesses;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** Tests and fixtures for worm audit qbit producer test.
*******************************************************************************/
class WormAuditQBitProducerTest
{
/*******************************************************************************
** Verify metadata and defaults.
*******************************************************************************/
@Test
void metadataAndDefaults() throws Exception
{
var config = new WormAuditQBitConfig();
QInstance instance = WormAuditTestSupport.init(config);
assertThat(config.getRetentionYears()).isEqualTo(6);
assertThat(config.getTableMetaDataCustomizer()).isNull();
assertThat(config.getSigningKey()).isNull();
assertThat(config.getTrustedPublicKeys()).isEmpty();
assertThat(config.getChainIntervalSeconds()).isEqualTo(30);
assertThat(config.getSettleHorizonSeconds()).isEqualTo(60);
assertThat(config.getCheckpointEveryLinks()).isEqualTo(10000);
assertThat(config.getCheckpointEveryMinutes()).isEqualTo(60);
assertThat(config.getBatchSize()).isEqualTo(1000);
assertThat(config.getLateDetailLookbackHours()).isEqualTo(24);
assertThat(config.getVerifyCron()).isEqualTo("0 0 3 * * ?");
assertThat(config.getAppRole()).isEqualTo("app");
assertThat(instance.getQBits().get("com.kingsrook.qbits:qbit-worm-audit").getConfig())
.isSameAs(config);
assertThat(instance.getApp("worm_audit").getSections())
.extracting(s -> s.getName())
.containsExactlyElementsOf(WormAuditTables.NAV);
assertThat(instance.getApp("worm_audit").getChildren())
.allMatch(
c ->
c.getName().startsWith("worm_audit_")
&& !c.getName().contains("verify"));
assertThat(instance.getProcess(WormAuditProcesses.VERIFY).getTableName())
.isEqualTo(WormAuditTables.CHAIN);
assertThat(instance.getProcess(WormAuditProcesses.CHAIN).getSchedule().getRepeatSeconds())
.isEqualTo(30);
assertThat(instance.getProcess(WormAuditProcesses.DAILY).getSchedule().getCronExpression())
.isEqualTo(config.getVerifyCron());
for(String name :
List.of(
"audit",
"auditDetail",
"auditUser",
"auditTable",
WormAuditTables.CHAIN,
WormAuditTables.CHECKPOINT,
WormAuditTables.EXPORT,
WormAuditTables.VERIFICATION,
WormAuditTables.PURGE))
{
var table = instance.getTable(name);
assertThat(
table.isCapabilityEnabled(
instance.getBackendForTable(name), Capability.TABLE_UPDATE))
.isFalse();
assertThat(
table.isCapabilityEnabled(
instance.getBackendForTable(name), Capability.TABLE_DELETE))
.isFalse();
if(name.startsWith("worm_audit_"))
{
assertThat(table.getPermissionRules().getPermissionBaseName()).isEqualTo(name);
assertThat(
table.isCapabilityEnabled(
instance.getBackendForTable(name), Capability.TABLE_INSERT))
.isFalse();
assertThat(table.getSections()).isNotEmpty();
}
}
}
/*******************************************************************************
** Verify customizations and validation.
*******************************************************************************/
@Test
void customizationsAndValidation() throws Exception
{
assertThat(new WormAuditQBitProducer().isEnabled()).isFalse();
var config =
new WormAuditQBitConfig()
.withRetentionYears(7)
.withChainIntervalSeconds(45)
.withSettleHorizonSeconds(0)
.withBatchSize(13)
.withCheckpointEveryLinks(5)
.withCheckpointEveryMinutes(2)
.withVerifyCron("0 1 3 * * ?")
.withAppRole("runtime")
.withSchedulerName("jobs")
.withCoreAuditTableNames(java.util.Map.of("audit", "custom_audit"))
.withTableMetaDataCustomizer(
(instance, table) ->
table.withLabel("custom")
.withCapabilities(Capability.TABLE_UPDATE));
var instance =
WormAuditTestSupport.core(
new com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData()
.withName("memory")
.withBackendType(
com.kingsrook.qqq.backend.core.modules.backend
.implementations.memory.MemoryBackendModule.class));
var producer =
new WormAuditQBitProducer()
.withNamespace("test")
.withQBitConfig(config.withBackendName("memory"));
producer.produce(instance);
assertThat(producer.getNamespace()).isEqualTo("test");
assertThat(producer.getQBitConfig()).isSameAs(config);
assertThat(producer.isEnabled()).isTrue();
assertThat(config.getCoreAuditTableNames()).containsEntry("audit", "custom_audit");
assertThat(instance.getTable(WormAuditTables.CHAIN).getLabel()).isEqualTo("custom");
assertThat(
instance.getTable(WormAuditTables.CHAIN)
.isCapabilityEnabled(
instance.getBackend("memory"), Capability.TABLE_UPDATE))
.isFalse();
assertThatThrownBy(() -> new WormAuditQBitConfig().validate())
.isInstanceOf(IllegalArgumentException.class);
assertThat(config.withLateDetailLookbackHours(48).getLateDetailLookbackHours()).isEqualTo(48);
assertThatThrownBy(() -> config.withLateDetailLookbackHours(0).validate()).isInstanceOf(IllegalArgumentException.class);
config.withLateDetailLookbackHours(24);
assertThatThrownBy(() -> config.withRetentionYears(0).validate())
.isInstanceOf(IllegalArgumentException.class);
assertThatThrownBy(
() -> WormAuditDdl.resolveNames(new WormAuditQBitConfig(), new QInstance()))
.isInstanceOf(IllegalArgumentException.class);
}
}
@@ -0,0 +1,136 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.wormaudit;
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.audits.AuditsMetaDataProvider;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import java.time.Instant;
import java.util.List;
class WormAuditTestSupport {
static final Instant OLD = Instant.parse("2020-01-02T03:04:05.123456Z");
static final AuditStore STORE = new AuditStore(null);
static QInstance init(WormAuditQBitConfig config) throws Exception {
MemoryRecordStore.fullReset();
QInstance instance =
core(
new QBackendMetaData()
.withName("memory")
.withBackendType(MemoryBackendModule.class));
new WormAuditQBitProducer()
.withQBitConfig(config.withBackendName("memory"))
.produce(instance);
QContext.init(instance, new QSession());
lookups();
return (instance);
}
static QInstance core(QBackendMetaData backend) throws Exception {
QInstance instance = new QInstance();
instance.withInstanceDefaultAuthentication(
new com.kingsrook.qqq.backend.core.model.metadata.authentication
.QAuthenticationMetaData()
.withType(
com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType
.FULLY_ANONYMOUS)
.withName("anon"));
instance.addBackend(backend);
new AuditsMetaDataProvider().defineAll(instance, backend.getName(), null);
instance.getTable("audit")
.withField(new QFieldMetaData("clientChannel", QFieldType.STRING));
return (instance);
}
static void lookups() throws Exception {
STORE.insert(
"auditTable",
List.of(
new QRecord()
.withValue("id", 1)
.withValue("name", "cms_page")
.withValue("label", "Page")));
STORE.insert(
"auditUser", List.of(new QRecord().withValue("id", 1).withValue("name", "james")));
}
static QRecord audit(long id) throws Exception {
QRecord row =
new QRecord()
.withValue("id", id)
.withValue("auditTableId", 1)
.withValue("auditUserId", 1)
.withValue("recordId", 17)
.withValue("message", "Record was Edited")
.withValue("timestamp", OLD.plusSeconds(id));
return (STORE.insert("audit", List.of(row)).getFirst());
}
static QRecord detail(long id, long auditId) throws Exception {
return (STORE.insert(
"auditDetail",
List.of(
new QRecord()
.withValue("id", id)
.withValue("auditId", auditId)
.withValue("fieldName", "title")
.withValue("message", "Changed Title")
.withValue("oldValue", "old")
.withValue("newValue", "new")))
.getFirst());
}
static void tamper(String table, long id, String field, java.io.Serializable value)
throws Exception {
MemoryRecordStore.getInstance()
.update(
new UpdateInput(table)
.withRecords(
List.of(
new QRecord()
.withValue(
"id",
table.equals("auditTable")
|| table.equals(
"auditUser")
? (java.io.Serializable)
(int) id
: id)
.withValue(field, value))),
false);
}
static void remove(String table, long id) throws Exception {
MemoryRecordStore.getInstance().delete(new DeleteInput(table).withPrimaryKeys(List.of(id)));
}
}
+17
View File
@@ -0,0 +1,17 @@
<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="WARN">
<Appenders>
<Console name="Console" target="SYSTEM_OUT">
<!-- Core can emit diagnostic events at OFF; exclude those as well. -->
<LevelRangeFilter minLevel="FATAL" maxLevel="WARN" onMatch="NEUTRAL" onMismatch="DENY"/>
<PatternLayout pattern="%d{HH:mm:ss} %-5level %logger{36} - %msg%n"/>
</Console>
</Appenders>
<Loggers>
<Logger name="org.testcontainers" level="WARN"/>
<Logger name="com.github.dockerjava" level="WARN"/>
<Logger name="org.apache.hc" level="WARN"/>
<Logger name="tc" level="WARN"/>
<Root level="WARN"><AppenderRef ref="Console"/></Root>
</Loggers>
</Configuration>