Compare commits
9
Commits
main
...
feature/chain
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
19ac187886
|
||
|
|
f380b9b2a1
|
||
|
|
906df343b0 | ||
|
|
d25df1de0a
|
||
|
|
bbac29e4d0
|
||
|
|
a6f4eb79f6 | ||
|
|
8ab4fb686d
|
||
|
|
1212aedca6 | ||
|
|
87c171387a |
No files matched your search
+60
-4
@@ -1,9 +1,65 @@
|
||||
version: 2.1
|
||||
|
||||
orbs:
|
||||
qqq-orb: kingsrook/qqq-orb@0.5.0
|
||||
qqq-orb: kingsrook/qqq-orb@0.6.8
|
||||
|
||||
commands:
|
||||
configure-package-install:
|
||||
steps:
|
||||
- run:
|
||||
name: Keep package installation unattended
|
||||
command: |
|
||||
# These build machines are discarded after the job.
|
||||
sudo install -d /etc/needrestart/conf.d
|
||||
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
|
||||
|
||||
workflows:
|
||||
build:
|
||||
jobs:
|
||||
- qqq-orb/mvn_test
|
||||
|
||||
test_only:
|
||||
jobs:
|
||||
- qqq-orb/mvn_test_only:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [ qqq-maven-registry-credentials ]
|
||||
filters:
|
||||
branches:
|
||||
ignore: /(develop|release/.*|integration.*)/
|
||||
tags:
|
||||
only: []
|
||||
|
||||
publish_snapshot:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [ qqq-maven-registry-credentials ]
|
||||
branch_type: snapshot
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- develop
|
||||
|
||||
publish_release_candidate:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [ qqq-maven-registry-credentials ]
|
||||
branch_type: release_candidate
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- /release\/.*/
|
||||
|
||||
publish_release:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [ qqq-maven-registry-credentials ]
|
||||
branch_type: release
|
||||
filters:
|
||||
branches:
|
||||
ignore: /.*/
|
||||
tags:
|
||||
only: [ /^v\d+\.\d+\.\d+$/ ]
|
||||
@@ -0,0 +1,20 @@
|
||||
# QRun-IO qbit .editorconfig: the Kingsrook layout (3-space indentation, never tabs).
|
||||
root = true
|
||||
|
||||
[*]
|
||||
charset = utf-8
|
||||
end_of_line = lf
|
||||
insert_final_newline = true
|
||||
trim_trailing_whitespace = true
|
||||
indent_style = space
|
||||
indent_size = 3
|
||||
tab_width = 3
|
||||
|
||||
[*.md]
|
||||
trim_trailing_whitespace = false
|
||||
|
||||
[*.{yml,yaml}]
|
||||
indent_size = 2
|
||||
|
||||
[*.{cmd,bat}]
|
||||
end_of_line = crlf
|
||||
@@ -0,0 +1,3 @@
|
||||
* text=auto eol=lf
|
||||
*.cmd text eol=crlf
|
||||
mvnw text eol=lf
|
||||
@@ -0,0 +1,25 @@
|
||||
# Weekly, grouped dependency updates. Minor and patch updates arrive as one PR per ecosystem;
|
||||
# major updates stay separate so each gets its own review. At most three version-update PRs per
|
||||
# ecosystem are open at once. Security updates are enabled in the repository settings and are
|
||||
# not limited by this schedule.
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: maven
|
||||
directory: /
|
||||
target-branch: develop
|
||||
schedule:
|
||||
interval: weekly
|
||||
open-pull-requests-limit: 3
|
||||
groups:
|
||||
maven-minor-and-patch:
|
||||
update-types: [minor, patch]
|
||||
|
||||
- package-ecosystem: github-actions
|
||||
directory: /
|
||||
target-branch: develop
|
||||
schedule:
|
||||
interval: weekly
|
||||
open-pull-requests-limit: 3
|
||||
groups:
|
||||
github-actions:
|
||||
update-types: [minor, patch]
|
||||
@@ -0,0 +1,31 @@
|
||||
# GitHub Release Configuration
|
||||
# Auto-generates release notes from PR labels
|
||||
|
||||
changelog:
|
||||
exclude:
|
||||
labels:
|
||||
- ignore-for-release
|
||||
- duplicate
|
||||
- invalid
|
||||
authors:
|
||||
- dependabot
|
||||
- dependabot[bot]
|
||||
categories:
|
||||
- title: Breaking Changes
|
||||
labels:
|
||||
- breaking-change
|
||||
- breaking
|
||||
- title: New Features
|
||||
labels:
|
||||
- enhancement
|
||||
- feature
|
||||
- title: Bug Fixes
|
||||
labels:
|
||||
- bug
|
||||
- fix
|
||||
- title: Dependencies
|
||||
labels:
|
||||
- dependencies
|
||||
- title: Other Changes
|
||||
labels:
|
||||
- "*"
|
||||
@@ -0,0 +1,25 @@
|
||||
# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
|
||||
# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
|
||||
# the release line and the release/* push runs CircleCI publish_release_candidate.
|
||||
name: Release candidate
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: Print the plan without merging or creating branches
|
||||
type: boolean
|
||||
default: false
|
||||
pull_request:
|
||||
types: [closed]
|
||||
branches: [develop]
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
release-rc:
|
||||
if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
|
||||
uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
|
||||
with:
|
||||
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
|
||||
+16
-15
@@ -1,20 +1,21 @@
|
||||
# Maven
|
||||
target/
|
||||
.env
|
||||
.idea
|
||||
*.iml
|
||||
*.class
|
||||
*.log
|
||||
*.jar
|
||||
*.war
|
||||
*.nar
|
||||
*.ear
|
||||
|
||||
# IDE
|
||||
.idea/
|
||||
*.iml
|
||||
.vscode/
|
||||
.project
|
||||
.classpath
|
||||
.settings/
|
||||
|
||||
# OS
|
||||
*.zip
|
||||
*.tar.gz
|
||||
*.rar
|
||||
hs_err_pid*
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
*.sw*
|
||||
.flattened-pom.xml
|
||||
dependency-reduced-pom.xml
|
||||
/.env.local
|
||||
/.cache/
|
||||
.claude/settings.local.json
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
wrapperVersion=3.3.4
|
||||
distributionType=only-script
|
||||
distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.11/apache-maven-3.9.11-bin.zip
|
||||
@@ -0,0 +1,51 @@
|
||||
# AGENTS.md
|
||||
|
||||
Guidance for coding agents working in this repository.
|
||||
|
||||
## Overview
|
||||
|
||||
QBit WORM Audit makes QQQ audit records write-once: database-enforced append-only audit tables (Postgres triggers plus an INSERT/SELECT-only app role), a hash chain over every audit row with signed checkpoints, a verification action and daily job, and sealed JSONL exports to an S3 Object Lock (COMPLIANCE) bucket through qbit-object-storage. Design: `docs/DESIGN.md`.
|
||||
|
||||
## Build commands
|
||||
|
||||
```bash
|
||||
./mvnw compile
|
||||
./mvnw test
|
||||
./mvnw verify # Checkstyle; JaCoCo: 70% instructions, 90% classes
|
||||
```
|
||||
|
||||
Use the Maven 3.9.11 wrapper (Java 21); Maven 3.10 cannot read the parent POM. On Windows use `./mvnw.cmd`.
|
||||
|
||||
## Architecture
|
||||
|
||||
Package: `com.kingsrook.qbits.wormaudit`. Entry point: `WormAuditQBitProducer implements QBitMetaDataProducer<WormAuditQBitConfig>`. Config requires the host `backendName`; core audit tables must exist before production.
|
||||
|
||||
- `ddl`: reviewed Liquibase PostgreSQL template and safe renderer; public sidecar tables and owner-only retention purge.
|
||||
- `guard`: immutable QQQ capabilities and value-free SQLSTATE mapping.
|
||||
- `canonical`, `chain`, `checkpoint`: strict JCS v1, transaction/advisory-locked writer, explicit detail coverage, pinned Ed25519 ranges.
|
||||
- `verify`: snapshot checks of content, continuity, coverage and checkpoints, starting at the purge anchor.
|
||||
- `processes`: standard streamed ETL action, scheduled chainer and daily QAlerts producer; never app navigation entries.
|
||||
- `model`: read-only tables, range record references and standard record child lists. No app-home widgets or export execution in phase (a).
|
||||
|
||||
PostgreSQL uses a transaction advisory lock; other backends require a single writer JVM.
|
||||
Use `./mvnw.cmd -o verify` offline. PostgreSQL tests skip without Docker locally, but fail whenever `CI` is set.
|
||||
Core RC.5 has no assistant deny API (reflective registration) and disallows underscores in declared alert-type metadata;
|
||||
signals retain the required `worm_audit.chain.break` name and dispatch through QAlerts without that declaration.
|
||||
|
||||
## Invariants and naming
|
||||
|
||||
- Audit rows are never updated or deleted by the application; the only removal path is the audited retention purge through a separate database role, after the retention date.
|
||||
- The chain and the sealed exports hash exactly what core wrote. Never add raw PHI to chain rows, checkpoints, exports, logs or alerts.
|
||||
- Tables use lowercase snake_case with the `worm_audit_` prefix; backend names equal table names. Permissions use the full `worm_audit` prefix. Processes are table or record actions, never navigation entries.
|
||||
|
||||
## Code style
|
||||
|
||||
3-space indent, next-line braces, `return (x);`, flower-box Javadoc on classes and methods, fluent `withX()` setters. Follow `checkstyle/` and `CODE_STYLE.md`.
|
||||
|
||||
## Dependencies
|
||||
|
||||
Parent: `com.kingsrook:qbit-build-parent:2.0.0`. QQQ BOM: `4.1.0-RC.5` (default-active `qqq-snapshot` profile).
|
||||
|
||||
## Release flow
|
||||
|
||||
CircleCI uses `kingsrook/qqq-orb@0.6.8`: develop publishes SNAPSHOTs, release/* publishes RCs, and only `vX.Y.Z` GA tags publish releases. Hotfix branches run tests only; never push `hotfix/*` or `release/*` by hand. RC automation uses workflow_dispatch or a merged develop PR labelled `release:rc`. Feature branches start from develop and return by PR. Never publish manually.
|
||||
@@ -0,0 +1,14 @@
|
||||
# Changelog
|
||||
|
||||
All notable changes are documented here, following [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
|
||||
- Phase (a): PostgreSQL append-only tables/triggers, safe DDL renderer, retention purge with immutable refusal/anchor records, and runtime grants.
|
||||
- Canonical audit v1 documents, contiguous hash chain with late-detail coverage, pinned Ed25519 checkpoints and binary Merkle roots.
|
||||
- Snapshot verification, streamed ETL Review/Result action, scheduled chaining/daily verification, and core QAlerts raise/resolve/heartbeat integration.
|
||||
- Read-only metadata, WORM Audit table navigation, record range references and child lists; reflective RC.6 assistant deny registration.
|
||||
- Offline memory/H2 acceptance, real pre-insert PHI canary hook, and PostgreSQL Testcontainers acceptance with CI-strict Docker availability.
|
||||
- Adoption and operations documentation. Export execution and app-home widgets remain later phases.
|
||||
@@ -1,15 +1,51 @@
|
||||
# qbit-worm-audit
|
||||
# CLAUDE.md
|
||||
|
||||
## Knowledge base
|
||||
Guidance for coding agents working in this repository.
|
||||
|
||||
Reviewed knowledge for this repo and the wider QQQ platform lives in the second-brain vault:
|
||||
## Overview
|
||||
|
||||
- Platform hub: `R:/Git.Local/KofTwentyTwo/second-brain/knowledge/qqq/qqq-hub.md`
|
||||
- This repo's dossier: `R:/Git.Local/KofTwentyTwo/second-brain/knowledge/qqq/repos/qbit-worm-audit.md`
|
||||
(reviewed at commit `294cd30f2d91`, branch `main`, 2026-07-04)
|
||||
QBit WORM Audit makes QQQ audit records write-once: database-enforced append-only audit tables (Postgres triggers plus an INSERT/SELECT-only app role), a hash chain over every audit row with signed checkpoints, a verification action and daily job, and sealed JSONL exports to an S3 Object Lock (COMPLIANCE) bucket through qbit-object-storage. Design: `docs/DESIGN.md`.
|
||||
|
||||
Key facts from the review: this repo is an **un-customized instantiation** of the
|
||||
`qbit-template-extension` GitHub template — no WORM-audit code exists yet; Maven coordinates are
|
||||
still `com.kingsrook.qbits:qbit-example-extension`; qqq pin is pre-4.0 (`qqq-backend-core 0.35.0`
|
||||
on main, `0.40.0-SNAPSHOT` on develop); README's "AGPL-3.0" line contradicts the Apache-2.0
|
||||
LICENSE. See the dossier's "v4.0 impact" section before starting implementation work here.
|
||||
## Build commands
|
||||
|
||||
```bash
|
||||
./mvnw compile
|
||||
./mvnw test
|
||||
./mvnw verify # Checkstyle; JaCoCo: 70% instructions, 90% classes
|
||||
```
|
||||
|
||||
Use the Maven 3.9.11 wrapper (Java 21); Maven 3.10 cannot read the parent POM. On Windows use `./mvnw.cmd`.
|
||||
|
||||
## Architecture
|
||||
|
||||
Package: `com.kingsrook.qbits.wormaudit`. Entry point: `WormAuditQBitProducer implements QBitMetaDataProducer<WormAuditQBitConfig>`. Config requires the host `backendName`; core audit tables must exist before production.
|
||||
|
||||
- `ddl`: reviewed Liquibase PostgreSQL template and safe renderer; public sidecar tables and owner-only retention purge.
|
||||
- `guard`: immutable QQQ capabilities and value-free SQLSTATE mapping.
|
||||
- `canonical`, `chain`, `checkpoint`: strict JCS v1, transaction/advisory-locked writer, explicit detail coverage, pinned Ed25519 ranges.
|
||||
- `verify`: snapshot checks of content, continuity, coverage and checkpoints, starting at the purge anchor.
|
||||
- `processes`: standard streamed ETL action, scheduled chainer and daily QAlerts producer; never app navigation entries.
|
||||
- `model`: read-only tables, range record references and standard record child lists. No app-home widgets or export execution in phase (a).
|
||||
|
||||
PostgreSQL uses a transaction advisory lock; other backends require a single writer JVM.
|
||||
Use `./mvnw.cmd -o verify` offline. PostgreSQL tests skip without Docker locally, but fail whenever `CI` is set.
|
||||
Core RC.5 has no assistant deny API (reflective registration) and disallows underscores in declared alert-type metadata;
|
||||
signals retain the required `worm_audit.chain.break` name and dispatch through QAlerts without that declaration.
|
||||
|
||||
## Invariants and naming
|
||||
|
||||
- Audit rows are never updated or deleted by the application; the only removal path is the audited retention purge through a separate database role, after the retention date.
|
||||
- The chain and the sealed exports hash exactly what core wrote. Never add raw PHI to chain rows, checkpoints, exports, logs or alerts.
|
||||
- Tables use lowercase snake_case with the `worm_audit_` prefix; backend names equal table names. Permissions use the full `worm_audit` prefix. Processes are table or record actions, never navigation entries.
|
||||
|
||||
## Code style
|
||||
|
||||
3-space indent, next-line braces, `return (x);`, flower-box Javadoc on classes and methods, fluent `withX()` setters. Follow `checkstyle/` and `CODE_STYLE.md`.
|
||||
|
||||
## Dependencies
|
||||
|
||||
Parent: `com.kingsrook:qbit-build-parent:2.0.0`. QQQ BOM: `4.1.0-RC.5` (default-active `qqq-snapshot` profile).
|
||||
|
||||
## Release flow
|
||||
|
||||
CircleCI uses `kingsrook/qqq-orb@0.6.8`: develop publishes SNAPSHOTs, release/* publishes RCs, and only `vX.Y.Z` GA tags publish releases. Hotfix branches run tests only; never push `hotfix/*` or `release/*` by hand. RC automation uses workflow_dispatch or a merged develop PR labelled `release:rc`. Feature branches start from develop and return by PR. Never publish manually.
|
||||
@@ -0,0 +1,3 @@
|
||||
# Code of Conduct
|
||||
|
||||
See Project level [Code of Conduct](https://github.com/QRun-IO/qqq/blob/develop/CODE_OF_CONDUCT.md)
|
||||
@@ -0,0 +1,3 @@
|
||||
# Code Style Guide
|
||||
|
||||
* See Project [Code Style Guide](https://github.com/QRun-IO/qqq/blob/develop/CODE_STYLE.md)
|
||||
@@ -0,0 +1,6 @@
|
||||
# Contributing to QBit Secured Data
|
||||
|
||||
* See Project [Contributing](https://github.com/QRun-IO/qqq/blob/develop/CONTRIBUTING.md)
|
||||
* Branches follow GitFlow: `feature/*` from `develop`, pull requests to `develop`, `release/*` for release candidates, `vX.Y.Z` tags for releases.
|
||||
* Commits are signed and follow Conventional Commits.
|
||||
* Build and test locally with `./mvnw verify` (Java 21).
|
||||
@@ -176,7 +176,7 @@
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
Copyright 2022-2025 QRunIO, LLC
|
||||
Copyright 2024 Kingsrook, LLC
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
qBit Template - Extension
|
||||
Copyright 2022-2025 QRunIO, LLC
|
||||
QBit WORM Audit
|
||||
Copyright 2026 Kingsrook, LLC
|
||||
|
||||
This product includes software developed at QRunIO, LLC (https://qrun.io/).
|
||||
This product includes software developed at Kingsrook, LLC (https://kingsrook.com/).
|
||||
|
||||
Licensed under the Apache License, Version 2.0.
|
||||
@@ -1,48 +1,200 @@
|
||||
# QBit Template: Extension
|
||||
# QBit WORM Audit
|
||||
|
||||
Template repository for creating QQQ Extension QBits - infrastructure extensions, customizers, and action handlers.
|
||||
Phase (a) provides PostgreSQL append-only enforcement, an asynchronous audit hash chain,
|
||||
Ed25519 checkpoints, a verification action, and daily verification with core QAlerts signals.
|
||||
It hashes exactly the audit values core stored; it does not read application records or
|
||||
secured-data keys. Sealed exports and the app home are later phases. The export table and
|
||||
range references are present as integration hooks; no export job or home widgets are installed.
|
||||
|
||||
## What is an Extension QBit?
|
||||
Requires Java 21, Maven wrapper 3.9.11 and QQQ 4.1.0-RC.5. PostgreSQL provides enforcement;
|
||||
memory/H2 support tamper-evidence tests without database enforcement.
|
||||
|
||||
Extension QBits extend or enhance how QQQ operates at the infrastructure/framework level. They provide:
|
||||
- New backend implementations
|
||||
- Authentication mechanisms
|
||||
- Audit capabilities
|
||||
- Table customizers
|
||||
- Action handlers
|
||||
## Tables and permissions
|
||||
|
||||
## Quick Start
|
||||
All sidecar tables use the configured host backend, snake_case fields, BIGINT identifiers,
|
||||
record sections, and full table-name permissions. They do not audit themselves. The app lists
|
||||
Verifications, Sealed Exports, Checkpoints, Audit Chain, and Purges in that order.
|
||||
|
||||
1. Click "Use this template" to create your QBit
|
||||
2. Run `scripts/customize_template.py` to rename the example
|
||||
3. Implement your extension logic
|
||||
4. Add tests
|
||||
5. Publish to Maven repository
|
||||
| Table | Purpose | Permission |
|
||||
|---|---|---|
|
||||
| `worm_audit_verification` | Immutable USER/SCHEDULE results | `worm_audit_verification.read` |
|
||||
| `worm_audit_export` | Reserved sealed-export catalog | `worm_audit_export.read` |
|
||||
| `worm_audit_checkpoint` | Signed ranges and Merkle roots | `worm_audit_checkpoint.read` |
|
||||
| `worm_audit_chain` | Contiguous links and explicit detail coverage | `worm_audit_chain.read` |
|
||||
| `worm_audit_purge` | Refusals, removal counts and retention anchors | `worm_audit_purge.read` |
|
||||
|
||||
## Structure
|
||||
`worm_audit.verify` authorizes the standard streamed ETL **Verify audit chain** action.
|
||||
Processes are attached to the chain table or scheduled; none is a navigation entry.
|
||||
Insert/update/delete capabilities are absent on sidecar tables; update/delete capabilities
|
||||
are removed from all four core audit tables. Internal appends use an explicit permission bypass.
|
||||
Checkpoint, export and verification record screens have a bounded chain-link child list;
|
||||
range endpoints and export/checkpoint references link to their records.
|
||||
|
||||
```
|
||||
src/main/java/com/kingsrook/qbits/example/
|
||||
├── ExampleExtensionQBitConfig.java # Configuration
|
||||
├── ExampleExtensionQBitProducer.java # Entry point
|
||||
├── customizers/
|
||||
│ └── ExampleTableCustomizer.java # Table customization
|
||||
└── actions/
|
||||
└── ExampleActionCustomizer.java # Action handling
|
||||
The table metadata customizer can configure host ESB publications. Quick Search registration
|
||||
is host-owned through `QuickSearchQBitConfig.searchableTables`; add all five tables.
|
||||
Verification inserts use QQQ's record-change listener SPI (as will phase-(b) export inserts),
|
||||
so an installed ESB publisher receives those events when the host configures its destinations
|
||||
on these two tables. The qbit defines no destination and configures no chain-link publication.
|
||||
|
||||
## Configuration
|
||||
|
||||
| Field | Default / contract |
|
||||
|---|---|
|
||||
| `backendName` | Required; core and sidecar audit tables must use this backend |
|
||||
| `retentionYears` | 6; compiled into the migration, not changed by a runtime flag |
|
||||
| `signingKey` | Optional `Supplier<SigningKey>`; `SigningKey(keyId, PrivateKey)` |
|
||||
| `trustedPublicKeys` | `Map<String, PublicKey>`; must contain the signing key's public key |
|
||||
| `chainIntervalSeconds` | 30 |
|
||||
| `settleHorizonSeconds` | 60 (zero allowed) |
|
||||
| `batchSize` | 1,000 headers per tick / links per insert batch |
|
||||
| `lateDetailLookbackHours` | 24; positive hours, older omissions checked by full verification |
|
||||
| `checkpointEveryLinks` | 10,000; maximum links in each checkpoint |
|
||||
| `checkpointEveryMinutes` | 60; requires at least one uncheckpointed link |
|
||||
| `verifyCron` | `0 0 3 * * ?` (03:00 UTC, Quartz syntax) |
|
||||
| `schedulerName` | `worm_audit`; uses the host scheduler if present, otherwise registers Quartz |
|
||||
| `coreAuditTableNames` | Overrides keyed by `audit`, `auditDetail`, `auditTable`, `auditUser` |
|
||||
| `appRole` | `app` |
|
||||
| `tableMetaDataCustomizer` | Optional; append-only capability guard runs afterward |
|
||||
|
||||
A missing signing key produces no checkpoints and adds the warning **no signing key configured**
|
||||
to verification. It does not itself break the chain. Signing requires a matching public-key pin.
|
||||
Rotate by adding a new pin and changing the supplier; retain old pins for old checkpoints.
|
||||
Never store the private key in a database row or checked-in configuration.
|
||||
|
||||
## Adopting in another app
|
||||
|
||||
Add `com.kingsrook.qbits:qbit-worm-audit:0.1.0-SNAPSHOT`. Define the host's core audit tables
|
||||
with `AuditsMetaDataProvider` first, then install the qbit:
|
||||
|
||||
```java
|
||||
WormAuditQBitConfig config = new WormAuditQBitConfig()
|
||||
.withBackendName("main")
|
||||
.withAppRole("app")
|
||||
.withSigningKey(() -> new WormAuditQBitConfig.SigningKey("2026-10", privateKey))
|
||||
.withTrustedPublicKeys(Map.of("2026-10", publicKey));
|
||||
new WormAuditQBitProducer().withQBitConfig(config).produce(qInstance);
|
||||
String migration = WormAuditDdl.postgres(config);
|
||||
```
|
||||
|
||||
## Key Characteristics
|
||||
1. Render and review the migration, then copy it into the host Liquibase changelog.
|
||||
The [default rendering](docs/ddl/worm-audit-postgres.sql) uses `audit`, `audit_detail`,
|
||||
`audit_table`, `audit_user`, runtime role `app`, and six years. The renderer resolves table
|
||||
backend names during production; explicit overrides win. Standalone rendering uses those
|
||||
defaults. Identifiers are validated and quoted; schema-qualified core table names are supported.
|
||||
2. Apply the DDL as the schema owner. It creates the five sidecar tables in `public`, functions,
|
||||
triggers, the NOLOGIN `worm_audit_purger` role, and runtime grants. Core column names must use
|
||||
the usual snake_case mappings (`audit_id`, `audit_table_id`, etc.). The migration is idempotent. Provision the purger role beforehand or give the migration role permission to create it.
|
||||
3. Run as a separate runtime role with SELECT/INSERT and sequence USAGE. Do not make it a member
|
||||
of the purger role or a schema/table owner. Use the host's PostgreSQL RDBMS strategy (RC.5's
|
||||
base strategy uses MySQL quoting; see the test host strategy in `RdbmsTestSupport`).
|
||||
4. Start QQQ's standard scheduler with its system session and grant users only the required read
|
||||
and verification permissions. Point the existing alerts implementation at core `QAlerts`.
|
||||
5. Configure ESB publications on verification/export in the table customizer. Register all five
|
||||
tables in the host Quick Search config with `SearchableTableConfig` / `SearchableFieldConfig`.
|
||||
Index identifiers and status fields; set basepull timestamps to `chained_at`, `created_at`,
|
||||
`sealed_at`, `finished_at`, and `requested_at` respectively (these tables have no `modifyDate`).
|
||||
|
||||
- **No QAppSection** - Pure infrastructure
|
||||
- **No user-facing tables** (or minimal operational)
|
||||
- **Wraps existing tables/processes** with new behavior
|
||||
- **Validates dependencies** on other QQQ components
|
||||
The RC.6 assistant deny API is registered reflectively when present; absence in RC.5 is a no-op.
|
||||
RC.5's alert-type metadata validator disallows underscores, so the qbit sends the specified
|
||||
`worm_audit.chain.break` signal without declaring incompatible alert-type metadata. `QAlerts`
|
||||
still dispatches it to the sink (or its logging fallback). Every scheduled verification sends
|
||||
`worm_audit.verify.daily`; FAIL raises with the first break and verification record reference;
|
||||
PASS resolves dedup key `chain`.
|
||||
|
||||
## Documentation
|
||||
## Chain and verification format
|
||||
|
||||
- [Getting Started](docs/00-getting-started.md)
|
||||
- [Extension Patterns](docs/01-extension-patterns.md)
|
||||
- JCS v1 uses UTF-16 key ordering, strict surrogate validation and UTC microsecond timestamps.
|
||||
IDs are integer longs except canonical `recordId`, which is a string. No floating-point values.
|
||||
- Link hashes concatenate the predecessor's **64 lowercase ASCII hex bytes**, one NUL byte,
|
||||
and canonical UTF-8 bytes. Genesis is 64 ASCII zeroes.
|
||||
- Every link names the detail IDs it covered, at most 250. A PostgreSQL insert guard and GIN
|
||||
overlap index reject duplicate coverage under the same advisory lock. Overflow and later details use
|
||||
`LATE_DETAIL` links with the same canonical header-and-detail-subset document. The canonical
|
||||
JSON is never persisted. A v1 content mismatch can identify the link, audit and covered detail
|
||||
IDs, but cannot identify one changed field among multiple details without original content.
|
||||
- PostgreSQL chaining uses transaction-scoped advisory lock `86753090421` shared with purge.
|
||||
Other backends use a JVM-wide lock and require one writer JVM. SQL backends select settled
|
||||
unchained headers with an anti-join; late commits are picked up on a later tick.
|
||||
- Merkle leaves are decoded 32-byte link hashes; parents hash the concatenated binary children;
|
||||
an odd node is promoted unchanged. Ed25519 signs JCS of the seven specified range/trust fields.
|
||||
- SQL verification uses a repeatable-read snapshot. Full runs check settled header/detail
|
||||
completeness; selected or filtered runs check their links, predecessor boundaries and
|
||||
intersecting checkpoints. Full verification currently materializes link metadata in memory.
|
||||
|
||||
## Retention operations
|
||||
|
||||
Only the ops login in `worm_audit_purger` may execute:
|
||||
|
||||
```sql
|
||||
SELECT public.worm_audit_purge(now() - interval '6 years');
|
||||
```
|
||||
|
||||
A recent/null cutoff or an incomplete expired prefix records REFUSED and returns normally.
|
||||
Purge locks the audit tables against concurrent inserts, removes only a prefix closed over
|
||||
all links and details of its audits, and records its last link id/hash. Detail and chain deletion
|
||||
is allowed only under the function-owner identity, transaction-local purge flag, and compiled
|
||||
retention cutoff. An app session setting that flag gets no deletion authority.
|
||||
|
||||
Lookups cannot change id/name or be deleted/truncated. The runtime role may update only the
|
||||
lookup label/dates through column grants; table-level UPDATE stays revoked. With normal grants,
|
||||
PostgreSQL rejects prohibited runtime writes as SQLSTATE 42501 before executing a trigger.
|
||||
The second layer (triggers) returns `P0001` / `worm_audit:` even if someone grants wider DML
|
||||
rights. The qbit maps those errors to a value-free user-facing message.
|
||||
|
||||
Verification starts from the latest successful purge anchor. Checkpoint signatures still verify
|
||||
after purge, but Merkle recomputation of a purged/partly purged checkpoint requires its sealed
|
||||
export and is reported as a warning. Sealed exports are phase (b); preserve external evidence
|
||||
before exercising retention removal in production. Superusers/owners can disable triggers;
|
||||
protect signing keys and keep public-key pins outside the database.
|
||||
|
||||
## Build and acceptance
|
||||
|
||||
```powershell
|
||||
./mvnw.cmd -o verify
|
||||
```
|
||||
|
||||
Checkstyle and JaCoCo gates (70% instructions, 90% classes) run without Docker. Memory and H2
|
||||
tests cover canonical vectors, signatures, rollback, exact coverage, late arrivals/details,
|
||||
verification findings, PHI tokenization, process summaries, permissions, and the alerts SPI.
|
||||
The PHI stand-in uses RC.5 `AuditActionCustomizerInterface.customizeInput` before insertion;
|
||||
classification is test-local because RC.5 lacks `DataClassification`.
|
||||
|
||||
PostgreSQL acceptance uses `postgres:16-alpine`, applies the DDL twice and tests grants,
|
||||
triggers, purge, real QQQ RDBMS chaining, superuser tamper, and 8 writers / 3 chainers.
|
||||
It skips automatically without Docker locally and fails if **any** `CI` environment value is set.
|
||||
On a Docker-enabled coordinator (image already available if running without network):
|
||||
|
||||
```powershell
|
||||
$env:CI = 'true'
|
||||
./mvnw.cmd -o '-Dtest=PostgresAcceptanceTest' test
|
||||
./mvnw.cmd -o verify
|
||||
```
|
||||
|
||||
POSIX equivalents: `CI=true ./mvnw -o -Dtest=PostgresAcceptanceTest test` and
|
||||
`CI=true ./mvnw -o verify`. No commits, publishing or deployment are part of these commands.
|
||||
|
||||
## License
|
||||
|
||||
AGPL-3.0 - See [LICENSE](LICENSE)
|
||||
[Apache License 2.0](LICENSE). Copyright Kingsrook, LLC.
|
||||
|
||||
### Physical columns and bounded polling
|
||||
|
||||
`WormAuditDdl.postgres(config)` renders table names only. Core physical columns
|
||||
must be snake_case: `audit.id`, `audit.timestamp`, `audit_detail.id`,
|
||||
`audit_detail.audit_id`, `audit_table.id/name/label/create_date/modify_date`, and
|
||||
`audit_user.id/name/create_date/modify_date`. QInstance field backend-name
|
||||
customizations do not change this migration; adapt the reviewed SQL before
|
||||
applying it if your schema differs. UTC values in `timestamp without time zone`
|
||||
and `timestamptz` are both supported.
|
||||
|
||||
`lateDetailLookbackHours` defaults to **24** (positive hours). Each PostgreSQL
|
||||
poll checks uncovered details whose chained header timestamp lies in
|
||||
`[cutoff - lookback, cutoff)`. The daily full verifier reports omissions outside
|
||||
that window. Initial header details are always chained, including overflow
|
||||
chunks. Checkpoints contain at most `checkpointEveryLinks` links; multiple full
|
||||
ranges can be signed per tick. Verification uses 1,000-link keyset pages.
|
||||
|
||||
Full verification retains at most 1,000 exact break diagnostics and 100 warnings,
|
||||
while persisting the total break count. SQL completeness counts all offenders
|
||||
and reports the first 100 header ids and first 100 detail ids. This keeps a badly
|
||||
corrupted chain from exhausting verifier memory.
|
||||
@@ -0,0 +1,4 @@
|
||||
# Security Policy
|
||||
|
||||
* See Project [Security Policy](https://github.com/QRun-IO/qqq/blob/develop/SECURITY.md)
|
||||
* Report suspected vulnerabilities privately through GitHub private vulnerability reporting on this repository, never in a public issue.
|
||||
@@ -0,0 +1,246 @@
|
||||
<?xml version="1.0"?>
|
||||
<!--
|
||||
~ QQQ - Low-code Application Framework for Engineers.
|
||||
~ Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
~ 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
~ contact@kingsrook.com
|
||||
~ https://github.com/Kingsrook/
|
||||
~
|
||||
~ Licensed under the Apache License, Version 2.0 (the "License");
|
||||
~ you may not use this file except in compliance with the License.
|
||||
~ You may obtain a copy of the License at
|
||||
~
|
||||
~ https://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
|
||||
<!DOCTYPE module PUBLIC
|
||||
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
||||
"http://checkstyle.sourceforge.net/dtds/configuration_1_3.dtd">
|
||||
<!--
|
||||
See reference at https://checkstyle.sourceforge.io/checks.html
|
||||
-->
|
||||
<module name = "Checker">
|
||||
<property name="charset" value="UTF-8"/>
|
||||
|
||||
<property name="severity" value="warning"/>
|
||||
|
||||
<property name="fileExtensions" value="java, properties, xml"/>
|
||||
<!-- Checks for whitespace -->
|
||||
<!-- See http://checkstyle.sf.net/config_whitespace.html -->
|
||||
<module name="FileTabCharacter">
|
||||
<property name="eachLine" value="true"/>
|
||||
</module>
|
||||
|
||||
<!--
|
||||
<module name="LineLength">
|
||||
<property name="max" value="100"/>
|
||||
<property name="ignorePattern" value="^package.*|^import.*|a href|href|http://|https://|ftp://"/>
|
||||
</module>
|
||||
-->
|
||||
|
||||
<module name="TreeWalker">
|
||||
<module name="SuppressWarningsHolder"/>
|
||||
<module name="OuterTypeFilename"/>
|
||||
<module name="IllegalTokenText">
|
||||
<property name="tokens" value="STRING_LITERAL, CHAR_LITERAL"/>
|
||||
<property name="format" value="\\u00(09|0(a|A)|0(c|C)|0(d|D)|22|27|5(C|c))|\\(0(10|11|12|14|15|42|47)|134)"/>
|
||||
<property name="message" value="Consider using special escape sequence instead of octal value or Unicode escaped value."/>
|
||||
</module>
|
||||
<module name="AvoidEscapedUnicodeCharacters">
|
||||
<property name="allowEscapesForControlCharacters" value="true"/>
|
||||
<property name="allowByTailComment" value="true"/>
|
||||
<property name="allowNonPrintableEscapes" value="true"/>
|
||||
</module>
|
||||
<module name="AvoidStarImport"/>
|
||||
<module name="OneTopLevelClass"/>
|
||||
<module name="NoLineWrap"/>
|
||||
<module name="EmptyBlock">
|
||||
<property name="option" value="TEXT"/>
|
||||
<property name="tokens" value="LITERAL_TRY, LITERAL_FINALLY, LITERAL_IF, LITERAL_ELSE, LITERAL_SWITCH"/>
|
||||
</module>
|
||||
<module name="NeedBraces"/>
|
||||
<module name="LeftCurly">
|
||||
<property name="option" value="nl"/>
|
||||
</module>
|
||||
<module name="RightCurly">
|
||||
<property name="option" value="alone"/>
|
||||
</module>
|
||||
<!--
|
||||
<module name="WhitespaceAround">
|
||||
<property name="allowEmptyConstructors" value="true"/>
|
||||
<property name="allowEmptyMethods" value="true"/>
|
||||
<property name="allowEmptyTypes" value="true"/>
|
||||
<property name="allowEmptyLoops" value="true"/>
|
||||
<message key="ws.notFollowed"
|
||||
value="WhitespaceAround: ''{0}'' is not followed by whitespace. Empty blocks may only be represented as '{}' when not part of a multi-block statement (4.1.3)"/>
|
||||
<message key="ws.notPreceded"
|
||||
value="WhitespaceAround: ''{0}'' is not preceded with whitespace."/>
|
||||
</module>
|
||||
-->
|
||||
<module name="OneStatementPerLine"/>
|
||||
<module name="MultipleVariableDeclarations"/>
|
||||
<module name="ArrayTypeStyle"/>
|
||||
<module name="MissingSwitchDefault"/>
|
||||
<module name="FallThrough"/>
|
||||
<module name="UpperEll"/>
|
||||
<module name="ModifierOrder"/>
|
||||
<module name="EmptyLineSeparator">
|
||||
<property name="allowNoEmptyLineBetweenFields" value="true"/>
|
||||
</module>
|
||||
<module name="SeparatorWrap">
|
||||
<property name="id" value="SeparatorWrapDot"/>
|
||||
<property name="tokens" value="DOT"/>
|
||||
<property name="option" value="nl"/>
|
||||
</module>
|
||||
<module name="SeparatorWrap">
|
||||
<property name="id" value="SeparatorWrapComma"/>
|
||||
<property name="tokens" value="COMMA"/>
|
||||
<property name="option" value="EOL"/>
|
||||
</module>
|
||||
<module name="SeparatorWrap">
|
||||
<!-- ELLIPSIS is EOL until https://github.com/google/styleguide/issues/258 -->
|
||||
<property name="id" value="SeparatorWrapEllipsis"/>
|
||||
<property name="tokens" value="ELLIPSIS"/>
|
||||
<property name="option" value="EOL"/>
|
||||
</module>
|
||||
<module name="SeparatorWrap">
|
||||
<!-- ARRAY_DECLARATOR is EOL until https://github.com/google/styleguide/issues/259 -->
|
||||
<property name="id" value="SeparatorWrapArrayDeclarator"/>
|
||||
<property name="tokens" value="ARRAY_DECLARATOR"/>
|
||||
<property name="option" value="EOL"/>
|
||||
</module>
|
||||
<module name="SeparatorWrap">
|
||||
<property name="id" value="SeparatorWrapMethodRef"/>
|
||||
<property name="tokens" value="METHOD_REF"/>
|
||||
<property name="option" value="nl"/>
|
||||
</module>
|
||||
<module name="PackageName">
|
||||
<property name="format" value="^[a-z]+(\.[a-z][a-z0-9]*)*$"/>
|
||||
<message key="name.invalidPattern"
|
||||
value="Package name ''{0}'' must match pattern ''{1}''."/>
|
||||
</module>
|
||||
<module name="TypeName">
|
||||
<message key="name.invalidPattern"
|
||||
value="Type name ''{0}'' must match pattern ''{1}''."/>
|
||||
</module>
|
||||
<module name="MemberName">
|
||||
<property name="format" value="^[_a-zA-Z]([a-z0-9A-Z][a-zA-Z0-9]*)?$"/>
|
||||
<message key="name.invalidPattern" value="Member name ''{0}'' must match pattern ''{1}''."/>
|
||||
</module>
|
||||
<module name="ParameterName">
|
||||
<property name="format" value="^[a-zA-Z]([a-z0-9A-Z][a-zA-Z0-9]*)?$"/>
|
||||
<message key="name.invalidPattern" value="Parameter name ''{0}'' must match pattern ''{1}''."/>
|
||||
</module>
|
||||
<module name="CatchParameterName">
|
||||
<property name="format" value="^[a-zA-Z]([a-z0-9A-Z][a-zA-Z0-9]*)?$"/>
|
||||
<message key="name.invalidPattern" value="Catch parameter name ''{0}'' must match pattern ''{1}''."/>
|
||||
</module>
|
||||
<module name="LocalVariableName">
|
||||
<property name="tokens" value="VARIABLE_DEF"/>
|
||||
<property name="format" value="^[a-zA-Z]([a-z0-9A-Z][a-zA-Z0-9]*)?$"/>
|
||||
<message key="name.invalidPattern" value="Local variable name ''{0}'' must match pattern ''{1}''."/>
|
||||
</module>
|
||||
<module name="ClassTypeParameterName">
|
||||
<property name="format" value="(^[A-Z][0-9]?)$|([A-Z][_A-Z0-9]*$)"/>
|
||||
<message key="name.invalidPattern" value="Class type name ''{0}'' must match pattern ''{1}''."/>
|
||||
</module>
|
||||
<module name="MethodTypeParameterName">
|
||||
<property name="format" value="(^[A-Z][0-9]?)$|([A-Z][_A-Z0-9]*$)"/>
|
||||
<message key="name.invalidPattern" value="Method type name ''{0}'' must match pattern ''{1}''."/>
|
||||
</module>
|
||||
<module name="InterfaceTypeParameterName">
|
||||
<property name="format" value="(^[A-Z][0-9]?)$|([A-Z][_A-Z0-9]*$)"/>
|
||||
<message key="name.invalidPattern" value="Interface type name ''{0}'' must match pattern ''{1}''."/>
|
||||
</module>
|
||||
<module name="NoFinalizer"/>
|
||||
<module name="GenericWhitespace">
|
||||
<message key="ws.followed" value="GenericWhitespace ''{0}'' is followed by whitespace."/>
|
||||
<message key="ws.preceded" value="GenericWhitespace ''{0}'' is preceded with whitespace."/>
|
||||
<message key="ws.illegalFollow" value="GenericWhitespace ''{0}'' should followed by whitespace."/>
|
||||
<message key="ws.notPreceded" value="GenericWhitespace ''{0}'' is not preceded with whitespace."/>
|
||||
</module>
|
||||
<module name="Indentation">
|
||||
<property name="basicOffset" value="3"/>
|
||||
<property name="braceAdjustment" value="0"/>
|
||||
<property name="caseIndent" value="3"/>
|
||||
<property name="throwsIndent" value="6"/>
|
||||
<property name="lineWrappingIndentation" value="3"/>
|
||||
<property name="arrayInitIndent" value="6"/>
|
||||
</module>
|
||||
<!--
|
||||
<module name="AbbreviationAsWordInName">
|
||||
<property name="ignoreFinal" value="false"/>
|
||||
<property name="allowedAbbreviationLength" value="1"/>
|
||||
</module>
|
||||
<module name="OverloadMethodsDeclarationOrder"/>
|
||||
<module name="VariableDeclarationUsageDistance"/>
|
||||
<module name="CustomImportOrder">
|
||||
<property name="sortImportsInGroupAlphabetically" value="true"/>
|
||||
<property name="separateLineBetweenGroups" value="true"/>
|
||||
<property name="customImportOrderRules" value="STATIC###THIRD_PARTY_PACKAGE"/>
|
||||
</module>
|
||||
-->
|
||||
<module name="CustomImportOrder">
|
||||
<property name="customImportOrderRules" value="SPECIAL_IMPORTS###STANDARD_JAVA_PACKAGE###THIRD_PARTY_PACKAGE###STATIC"/>
|
||||
<property name="specialImportsRegExp" value="^javax\."/>
|
||||
<property name="standardPackageRegExp" value="^java\."/>
|
||||
<property name="sortImportsInGroupAlphabetically" value="true"/>
|
||||
<property name="separateLineBetweenGroups" value="false"/>
|
||||
</module>
|
||||
<module name="MethodParamPad"/>
|
||||
<module name="NoWhitespaceBefore">
|
||||
<property name="tokens" value="COMMA, SEMI, POST_INC, POST_DEC, DOT, ELLIPSIS, METHOD_REF"/>
|
||||
<property name="allowLineBreaks" value="true"/>
|
||||
</module>
|
||||
<module name="ParenPad"/>
|
||||
<module name="OperatorWrap">
|
||||
<property name="option" value="NL"/>
|
||||
<property name="tokens" value="BAND, BOR, BSR, BXOR, DIV, EQUAL, GE, GT, LAND, LE, LITERAL_INSTANCEOF, LOR, LT, MINUS, MOD, NOT_EQUAL, PLUS, QUESTION, SL, SR, STAR, METHOD_REF "/>
|
||||
</module>
|
||||
<module name="AnnotationLocation">
|
||||
<property name="id" value="AnnotationLocationMostCases"/>
|
||||
<property name="tokens" value="CLASS_DEF, INTERFACE_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF"/>
|
||||
</module>
|
||||
<module name="AnnotationLocation">
|
||||
<property name="id" value="AnnotationLocationVariables"/>
|
||||
<property name="tokens" value="VARIABLE_DEF"/>
|
||||
<property name="allowSamelineMultipleAnnotations" value="true"/>
|
||||
</module>
|
||||
<module name="JavadocMethod">
|
||||
<property name="allowMissingParamTags" value="true"/>
|
||||
<property name="allowMissingReturnTag" value="true"/>
|
||||
<property name="allowedAnnotations" value="Override, Test"/>
|
||||
</module>
|
||||
<module name="MissingJavadocMethod">
|
||||
<property name="scope" value="private"/>
|
||||
</module>
|
||||
<module name="MissingJavadocType">
|
||||
<property name="scope" value="private"/>
|
||||
</module>
|
||||
<module name="MethodName">
|
||||
<property name="format" value="^[a-z][a-zA-Z0-9_]*$"/>
|
||||
<message key="name.invalidPattern"
|
||||
value="Method name ''{0}'' must match pattern ''{1}''."/>
|
||||
</module>
|
||||
<module name="EmptyCatchBlock">
|
||||
<property name="exceptionVariableName" value="expected"/>
|
||||
</module>
|
||||
<module name="CommentsIndentation"/>
|
||||
|
||||
<!-- this would be nice, but requires the the @inheritDoc javadoc tag... -->
|
||||
<module name="MissingOverride"/>
|
||||
|
||||
</module>
|
||||
<module name="Header">
|
||||
<property name="headerFile" value="checkstyle/license.txt"/>
|
||||
<property name="fileExtensions" value="java"/>
|
||||
<property name="ignoreLines" value="3"/>
|
||||
</module>
|
||||
<module name="SuppressWarningsFilter"/>
|
||||
</module>
|
||||
@@ -0,0 +1,19 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
@@ -1,42 +0,0 @@
|
||||
# Getting Started
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Java 17+
|
||||
- Maven 3.8+
|
||||
- QQQ 0.23.0+
|
||||
|
||||
## Creating Your Extension QBit
|
||||
|
||||
1. Click "Use this template" on GitHub
|
||||
2. Clone your new repository
|
||||
3. Run the customization script:
|
||||
```bash
|
||||
python scripts/customize_template.py
|
||||
```
|
||||
4. Enter your QBit name when prompted
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
src/main/java/com/kingsrook/qbits/yourextension/
|
||||
├── YourExtensionQBitConfig.java
|
||||
├── YourExtensionQBitProducer.java
|
||||
├── customizers/
|
||||
└── actions/
|
||||
```
|
||||
|
||||
## Using Your QBit
|
||||
|
||||
```java
|
||||
new YourExtensionQBitProducer()
|
||||
.withConfig(new YourExtensionQBitConfig()
|
||||
.withTargetTableName("order"))
|
||||
.produce(qInstance, "myExtension");
|
||||
```
|
||||
|
||||
## Next Steps
|
||||
|
||||
- Read [Extension Patterns](01-extension-patterns.md)
|
||||
- Implement your customization logic
|
||||
- Add tests
|
||||
@@ -1,58 +0,0 @@
|
||||
# Extension Patterns
|
||||
|
||||
## Table Customizers
|
||||
|
||||
Add behavior to existing tables:
|
||||
|
||||
```java
|
||||
public class AuditTableCustomizer
|
||||
{
|
||||
public void customize(QTableMetaData table)
|
||||
{
|
||||
table.withField(new QFieldMetaData("createdAt", QFieldType.DATE_TIME));
|
||||
table.withField(new QFieldMetaData("modifiedAt", QFieldType.DATE_TIME));
|
||||
|
||||
table.withCustomizer(
|
||||
TableCustomizers.PRE_INSERT_RECORD,
|
||||
new QCodeReference(AuditPreInsert.class));
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Action Customizers
|
||||
|
||||
Wrap actions with additional behavior:
|
||||
|
||||
```java
|
||||
public class LoggingActionWrapper implements PreInsertRecordInterface
|
||||
{
|
||||
@Override
|
||||
public void preInsertRecord(InsertInput input, QRecord record) throws QException
|
||||
{
|
||||
LOG.info("Inserting record", logPair("table", input.getTableName()));
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Config Validation
|
||||
|
||||
Validate that required components exist:
|
||||
|
||||
```java
|
||||
@Override
|
||||
public void validate(QInstance qInstance, List<String> errors)
|
||||
{
|
||||
if(qInstance.getTable(targetTableName) == null)
|
||||
{
|
||||
errors.add("Target table required: " + targetTableName);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## When to Use Extension QBits
|
||||
|
||||
- Adding audit logging to tables
|
||||
- Implementing authentication/authorization
|
||||
- Adding encryption to fields
|
||||
- Custom validation rules
|
||||
- Performance monitoring
|
||||
+383
@@ -0,0 +1,383 @@
|
||||
# QBit WORM Audit: design
|
||||
|
||||
Status: approved. **Owner decisions (2026-10-12): all recommendations accepted.** Every open
|
||||
question in section 12 is decided as recommended.
|
||||
|
||||
## 1. Owner request
|
||||
|
||||
> "we also need to ensure that our auditing qbit supports WORM audits as well. and the phi/secure
|
||||
> data qbit needs to ensure that we don't leak phi into audits"
|
||||
|
||||
Context: HIPAA §164.312(b) audit controls, §164.312(c)(1) integrity, §164.316(b)(2)(i) six-year
|
||||
retention of required documentation (audit records are kept to the same horizon), and a
|
||||
HITRUST-style review that asks three things: (1) can anyone change or delete an audit record,
|
||||
(2) would we know if they did, and (3) is there a copy nobody can alter?
|
||||
|
||||
## 2. Goals and non-goals
|
||||
|
||||
Goals:
|
||||
1. **Append-only at the database level.** No application path, and no ordinary database login,
|
||||
can UPDATE, DELETE or TRUNCATE audit rows. The only removal is an audited retention purge,
|
||||
after the retention date, through a separate database role.
|
||||
2. **Tamper evidence.** Every audit row is linked into a hash chain. Signed checkpoints anchor the
|
||||
chain. A verification action and a daily job report the exact first broken row.
|
||||
3. **Sealed copies.** New audit rows and checkpoints are exported on a schedule to an S3 bucket
|
||||
with Object Lock in COMPLIANCE mode (default six years), and can be verified against the live
|
||||
chain.
|
||||
4. **No PHI in the WORM layer.** The chain and the exports seal exactly what core wrote. With
|
||||
qbit-secured-data that means HMACs for classified values, never plaintext.
|
||||
|
||||
Non-goals: changing what core audits (that is core's and qbit-secured-data's job); protecting
|
||||
against a Postgres superuser *preventing* a change (a superuser can disable triggers; the
|
||||
chain, the signatures and the Object Lock copies make that change *detectable*, see 4.5).
|
||||
|
||||
## 3. Scope: which tables
|
||||
|
||||
| Table (QQQ name / kof22 physical) | Treatment |
|
||||
|---|---|
|
||||
| `audit` / `audit` | Append-only (UPDATE, DELETE, TRUNCATE rejected). Chained. |
|
||||
| `auditDetail` / `audit_detail` | Append-only. Chained with its header. |
|
||||
| `auditTable` / `audit_table` | Lookup. DELETE and TRUNCATE rejected; UPDATE of `id` or `name` rejected (label and dates may change). Names are resolved into the chained content, so a later rename is detected. |
|
||||
| `auditUser` / `audit_user` | As `audit_table`. |
|
||||
| `worm_audit_chain`, `worm_audit_checkpoint`, `worm_audit_export`, `worm_audit_verification`, `worm_audit_purge` | The qbit's own tables: all append-only. |
|
||||
|
||||
Physical names come from the host's backend (kof22 maps camelCase to snake_case). The DDL is a
|
||||
template rendered for the host's names (section 5.4).
|
||||
|
||||
## 4. Append-only enforcement (Postgres first)
|
||||
|
||||
### 4.1 Triggers
|
||||
One function, `worm_audit_reject()`, attached as:
|
||||
- `BEFORE UPDATE OR DELETE ... FOR EACH ROW` on every append-only table;
|
||||
- `BEFORE TRUNCATE ... FOR EACH STATEMENT` on every append-only table;
|
||||
- on the two lookups, a row trigger that rejects DELETE and any UPDATE that changes `id` or
|
||||
`name`.
|
||||
|
||||
The error is explicit and greppable:
|
||||
|
||||
```
|
||||
ERROR: worm_audit: audit rows are write-once; UPDATE of "audit" is not allowed (id=4211). Expired rows are removed only by worm_audit_purge().
|
||||
SQLSTATE: P0001 (HINT: see qbit-worm-audit docs/DESIGN.md section 4)
|
||||
```
|
||||
|
||||
In the application the qbit also removes the update and delete capabilities from these tables'
|
||||
QQQ metadata, so the admin UI and both APIs never offer edit or delete, and a direct call gets
|
||||
QQQ's standard "table does not support update/delete" error before reaching the database. The
|
||||
qbit maps a `P0001` with the `worm_audit:` prefix to a `QUserFacingException`: "Audit records are
|
||||
write-once and cannot be changed or deleted."
|
||||
|
||||
### 4.2 The retention purge (the only removal path)
|
||||
`worm_audit_purge(p_cutoff timestamptz)` is a `SECURITY DEFINER` function owned by the schema
|
||||
owner. `EXECUTE` is revoked from `PUBLIC` and granted only to the role `worm_audit_purger`, which
|
||||
the app role is not a member of.
|
||||
|
||||
1. It refuses (and records a `REFUSED` row in `worm_audit_purge`, then returns; no exception, so
|
||||
the record survives) when `p_cutoff > now() - retention`. The retention interval is a constant
|
||||
compiled into the function by the changeset (default `6 years`), so shortening it is a schema
|
||||
migration, reviewed and itself audited.
|
||||
2. It purges only a **chain prefix**: the audits whose chain links all lie at or before the last
|
||||
link whose audit timestamp is older than the cutoff, together with their details and links.
|
||||
3. It sets a transaction-local flag (`set_config('worm_audit.purging', 'on', true)`). The reject
|
||||
trigger allows DELETE only when that flag is on **and** `current_user` is the function owner
|
||||
**and** the row is older than the retention interval. Setting the flag from another session
|
||||
does nothing, because `current_user` is not the owner.
|
||||
4. It writes a `PURGED` row: who (session_user), cutoff, retention, counts, and the **anchor**
|
||||
(the last purged link's id and hash). Verification continues from the anchor.
|
||||
|
||||
The purge runs from an ops runbook (`psql` as `worm_audit_purger`). Version 0.1 has no UI action;
|
||||
the app home lists purge rows, including refusals.
|
||||
|
||||
### 4.3 GRANT model
|
||||
|
||||
| Role | Owns | Grants on append-only tables |
|
||||
|---|---|---|
|
||||
| `<app>_owner` (Liquibase) | tables, functions, triggers | all (owner) |
|
||||
| `<app>` (runtime) | nothing | `SELECT, INSERT`; `USAGE` on their sequences. No UPDATE, DELETE or TRUNCATE. |
|
||||
| `worm_audit_purger` (ops) | nothing | `EXECUTE worm_audit_purge`; nothing else |
|
||||
|
||||
When a host uses one role for both migrations and runtime (kof22 today), the triggers still stop
|
||||
every UPDATE, DELETE and TRUNCATE from the application, but the role can `ALTER TABLE ... DISABLE
|
||||
TRIGGER`. The docs recommend splitting the roles, and the app home shows a warning when the
|
||||
runtime role owns the audit tables (checked through `pg_tables.tableowner = current_user`).
|
||||
|
||||
### 4.4 Changesets
|
||||
The qbit ships `src/main/resources/com/kingsrook/qbits/wormaudit/ddl/postgres/worm-audit.sql`,
|
||||
a Liquibase formatted-SQL template with placeholders for the four core audit table names, the
|
||||
app role and the retention interval, plus a renderer (`WormAuditDdl.postgres(config)`) and the
|
||||
rendered default (`docs/ddl/worm-audit-postgres.sql`, snake_case names). Hosts copy the rendered
|
||||
file into their changelog (kof22: `changesets/features/YYYY-MM-DD-worm-audit.sql`). It creates the
|
||||
qbit tables, the functions, the triggers, the purger role (`CREATE ROLE ... NOLOGIN` when it is
|
||||
missing) and the grants. It is idempotent (`CREATE OR REPLACE`, `DROP TRIGGER IF EXISTS`).
|
||||
|
||||
### 4.5 Threat model summary
|
||||
|
||||
| Actor | Can modify audit rows? | Detected by |
|
||||
|---|---|---|
|
||||
| App user, admin UI, QQQ APIs, assistant | No (capabilities removed; triggers) | n/a |
|
||||
| SQL injection or a bug as the runtime role | No (triggers; no grant) | n/a |
|
||||
| Owner role (migrations) | Only by disabling triggers | chain verification (daily), alert |
|
||||
| Postgres superuser rewriting rows and recomputing the chain | Yes | checkpoint signatures (no signing key in the DB); sealed exports in Object Lock |
|
||||
| Anyone deleting the sealed copies | No (COMPLIANCE mode: not even the bucket root, until expiry) | n/a |
|
||||
|
||||
### 4.6 Other databases (documented, not shipped in 0.1)
|
||||
- **MySQL/MariaDB:** `BEFORE UPDATE`/`BEFORE DELETE` triggers with `SIGNAL SQLSTATE '45000'`;
|
||||
no TRUNCATE trigger, so revoke `DROP` from the app user; purge through a `SQL SECURITY DEFINER`
|
||||
procedure.
|
||||
- **SQL Server:** `INSTEAD OF UPDATE, DELETE` triggers plus `DENY UPDATE, DELETE` to the app user.
|
||||
- **H2** (tests only): no enforcement; the chain and the exports work on any QQQ RDBMS or memory
|
||||
backend, so tamper *evidence* is database-independent.
|
||||
|
||||
## 5. Tamper evidence
|
||||
|
||||
### 5.1 Canonical content (agreed with qbit-secured-data)
|
||||
One canonical document per audit header: the header plus all of its detail rows, as RFC 8785 JCS
|
||||
JSON (sorted keys, no insignificant whitespace, UTF-8):
|
||||
|
||||
```json
|
||||
{"audit":{"clientChannel":null,"id":4211,"message":"Record was Edited","recordId":"17","table":"cms_page","timestamp":"2026-10-13T21:04:05.123456Z","user":"james"},
|
||||
"details":[{"fieldName":"title","id":9001,"message":"Changed Title from ...","newValue":"...","oldValue":"..."}],
|
||||
"v":1}
|
||||
```
|
||||
|
||||
- Values are exactly as stored: no formatting, no re-masking, no lookups other than resolving
|
||||
`auditTableId`/`auditUserId` to their names. `details` are sorted by id.
|
||||
- With qbit-secured-data's core hook (QRun-IO/qqq#1161, `docs/SECURED-DATA.md` §14.2), PII and
|
||||
PHI values are tokenised inside `DMLAuditAction` before `AuditAction` inserts, in the same
|
||||
transaction as the data change, so the token is the only form ever written. Stored forms (ASCII,
|
||||
at most 200 characters) in `oldValue`/`newValue` are
|
||||
`hmac:v1:<keyId>:<b64url>`, `shred:v1:<auditKeyId>:<b64url>`, `changed:v1`,
|
||||
`wiped:v1:<ruleId>` and `shredded:v1:<ruleId>` (the last is its own detail row, written when
|
||||
keys are destroyed, and is chained like any other row). Detail messages for those fields
|
||||
are replaced with value-free ones ("Changed <Label>"); header messages carry no field values.
|
||||
The canonical form, the chain and the exports therefore contain tokens only, and verifying the
|
||||
chain never needs a key. The WORM layer never reads, joins or exports
|
||||
`secured_data_audit_key` or any other `secured_data_*` table. Crypto-shredding (destroying a
|
||||
record's key) leaves the HMACs and the chain intact; they simply become unlinkable.
|
||||
- The canonical JSON is never stored in the database. The chain stores ids and hashes only, so
|
||||
the qbit adds no new copy of audit content to the database.
|
||||
|
||||
### 5.2 The chain: a single asynchronous chainer (recommended)
|
||||
`link_hash = SHA-256(prev_link_hash || 0x00 || canonical_bytes)`, hex, with the genesis
|
||||
`prev_link_hash` = 64 zeros.
|
||||
|
||||
Core inserts audits from many threads, and headers and details may be inserted in separate
|
||||
autocommit statements (core passes the caller's transaction, which may be null). Computing the
|
||||
chain in core's write path would need a global lock on every audited write, and a trigger-based
|
||||
chain would need the canonical form in SQL. The design instead uses **one chainer**:
|
||||
|
||||
- A scheduled job (every 30 s by default) takes `pg_try_advisory_xact_lock` (any other instance
|
||||
skips that tick; on non-Postgres backends, a single-row lock table with `SELECT ... FOR
|
||||
UPDATE`). It selects audit headers that are **older than a settle horizon** (default 60 s, so
|
||||
their details have been committed) and not yet chained (an anti-join on the unique
|
||||
`worm_audit_chain.audit_id`), in `(timestamp, id)` order. Then it appends links in batches.
|
||||
- Link ids are assigned by the chainer as `max(id) + 1` inside the lock, not from a sequence, so
|
||||
the chain is contiguous (a sequence gaps on rollback).
|
||||
- **Late arrivals** (a header whose id sorts before already-chained ones because its transaction
|
||||
committed late) are chained when they appear. The chain order is chaining order, not audit id
|
||||
order, and each link records its `audit_id`.
|
||||
- **Late details** (a detail inserted after its header was chained) get their own link,
|
||||
`kind = LATE_DETAIL`, covering exactly those detail ids. Verification flags any detail not
|
||||
covered by exactly one link.
|
||||
- Unique constraints back the lock: `UNIQUE (audit_id) WHERE kind = 'AUDIT'`, and the detail
|
||||
coverage.
|
||||
|
||||
Why not per-partition chains: they make each partition's chain shorter but multiply heads,
|
||||
checkpoints and verification state, and they still need a writer per partition. One chainer
|
||||
handles thousands of rows a second in batches, which is well above audit volume, and keeps a
|
||||
single head to sign. The table carries no partition column (YAGNI). A future partitioned mode
|
||||
would add one.
|
||||
|
||||
Window: a row is unchained for at most horizon plus interval (about 90 s). During that window the
|
||||
triggers still protect it. The chain defends against privileged tampering, which the window does
|
||||
not change materially.
|
||||
|
||||
### 5.3 Signed checkpoints (`worm_audit_checkpoint`)
|
||||
Every hour, and every 10,000 links, the chainer writes a checkpoint for the links since the last
|
||||
one: `from_link_id`, `to_link_id`, `head_hash` (the last link hash), `merkle_root` (a binary
|
||||
SHA-256 Merkle tree over the range's link hashes, so a single exported range can be proven
|
||||
without the whole chain), `key_id`, `public_key`, `algorithm = Ed25519` and `signature` over the
|
||||
JCS form of those fields. The signing key comes from config (`WormAuditQBitConfig.signingKey`, a
|
||||
supplier; kof22: env `WORM_AUDIT_SIGNING_KEY` from Infisical) and never touches the database. The
|
||||
public key is stored on each checkpoint and pinned in config (`trustedPublicKeys`), so a forged
|
||||
checkpoint signed with another key fails verification. Rotation adds a new key id. Old
|
||||
checkpoints verify against their own pinned key.
|
||||
|
||||
### 5.4 Verification
|
||||
**"Verify audit chain"**, a table action on `worm_audit_chain` (and on selected links), is a
|
||||
standard streamed ETL:
|
||||
- Extract: the links in scope (all since the purge anchor, or the selection), in id order.
|
||||
- Transform: recompute each link from the live audit rows (`prev` continuity, canonical content,
|
||||
`link_hash`); check checkpoint signatures and Merkle roots; for a full run, check completeness
|
||||
(every settled audit and detail is covered exactly once).
|
||||
- Load: one `worm_audit_verification` row.
|
||||
- Review: "1,204,331 chain links will be verified (links 1–1,204,331, 121 checkpoints)."
|
||||
- Result: "1,204,331 links verified" or, on failure, one line per break, each with the exact row:
|
||||
"Break at link 88,412 (audit 90,117, table cms_page, record 17): content differs from the
|
||||
chained hash" / "audit_detail 9,002 is not covered by the chain" / "checkpoint 41: signature
|
||||
does not verify".
|
||||
|
||||
**Daily verification**: a scheduled job runs the same verifier at 03:00. A failure raises
|
||||
`QAlerts.raise(new AlertSignal("worm_audit.chain.break").withDedupKey("chain")...)` with the first
|
||||
break in the title and the verification record as the record ref; a later pass resolves it. Every
|
||||
run sends `QAlerts.heartbeat("worm_audit.verify.daily")`. `QAlerts` is the core producer SPI
|
||||
(QQQ 4.1.0-RC.5), so the qbit needs no dependency on qbit-alerts. With no alerts implementation
|
||||
installed the alerts go to the log.
|
||||
|
||||
## 6. Sealed exports
|
||||
|
||||
### 6.1 Export job
|
||||
Daily at 02:00 (configurable), and by the table action "Export now" on `worm_audit_export`:
|
||||
1. Range: links after the last export's `to_link_id`, up to the latest checkpoint. Exports are
|
||||
contiguous by construction. The app home flags a gap (a range not covered) and lag.
|
||||
2. Objects under `<prefix>/<yyyy>/<mm>/<dd>/<from>-<to>/`:
|
||||
- `links.jsonl`: one line per link: `{"link":{id,kind,auditId,detailIds,prevHash,linkHash},"content":<canonical document>}`;
|
||||
- `checkpoints.jsonl`: the checkpoints in range, with signatures;
|
||||
- `manifest.json`: format version, range, counts, the SHA-256 of each file, `head_hash`,
|
||||
the previous export's manifest SHA-256 (the exports form their own chain), qbit version.
|
||||
3. Each object is put through qbit-object-storage (`SystemStorage`, bucket alias from config,
|
||||
default `worm_audit`) with `LockMode.COMPLIANCE` and `retainUntil = now + retentionYears`. The
|
||||
export refuses to run, and alerts `worm_audit.export.failed`, when the bucket does not report
|
||||
the `OBJECT_LOCK` capability or the put does not echo the lock.
|
||||
4. On success it inserts one `worm_audit_export` row (insert-only; a failed export leaves no row,
|
||||
and its alert carries the reason).
|
||||
|
||||
### 6.2 Restore and verify a sealed export
|
||||
"Verify sealed export", a record action on `worm_audit_export` (standard ETL), downloads the
|
||||
objects. It checks the manifest hashes, the lock mode and retain-until of each object, the link
|
||||
hashes recomputed from the exported content, the checkpoint signatures, and the export's link to
|
||||
the previous manifest. Then it compares each exported link and content document with the live
|
||||
chain and the live audit rows. Result lines name each difference ("link 88,412: live content
|
||||
differs from the sealed copy (sealed 2026-10-14)"). This is also the restore tool: the exported
|
||||
content is the canonical audit content, so a verified export is evidence of the original rows
|
||||
after a live tamper. A purged range stays verifiable from the sealed copy alone. Writing restored
|
||||
rows back into a live table is out of scope (it would need the purge role and break append-only).
|
||||
|
||||
## 7. Tables
|
||||
|
||||
All tables use the host's RDBMS backend, are append-only (section 4), are audited by nothing
|
||||
(auditing the audit tables would recurse), have full record screens, and are searchable in Quick
|
||||
Search. ESB table events are on for `worm_audit_verification` and `worm_audit_export` (the
|
||||
events integrations care about); chain links are high-volume and publish none.
|
||||
|
||||
- **`worm_audit_chain`** [Audit Chain Link]: `id` (contiguous), `kind` (AUDIT, LATE_DETAIL),
|
||||
`audit_id`, `detail_ids` (comma list, at most 250 per link), `prev_hash`, `link_hash`,
|
||||
`chained_at`. Unique `(audit_id) where kind = 'AUDIT'`.
|
||||
- **`worm_audit_checkpoint`** [Checkpoint]: `id`, `from_link_id`, `to_link_id`, `link_count`,
|
||||
`head_hash`, `merkle_root`, `algorithm`, `key_id`, `public_key`, `signature`, `created_at`.
|
||||
- **`worm_audit_export`** [Sealed Export]: `id`, `from_link_id`, `to_link_id`,
|
||||
`from_checkpoint_id`, `to_checkpoint_id`, `bucket_alias`, `object_prefix`, `manifest_key`,
|
||||
`manifest_sha256`, `prev_manifest_sha256`, `object_count`, `byte_count`, `lock_mode`,
|
||||
`retain_until`, `sealed_at`.
|
||||
- **`worm_audit_verification`** [Verification]: `id`, `kind` (CHAIN, SEALED_EXPORT), `trigger`
|
||||
(SCHEDULE, USER), `started_at`, `finished_at`, `from_link_id`, `to_link_id`, `links_checked`,
|
||||
`outcome` (PASS, FAIL), `break_count`, `first_break_link_id`, `first_break_audit_id`,
|
||||
`first_break_reason`, `export_id`, `run_by`.
|
||||
- **`worm_audit_purge`** [Purge]: `id`, `requested_at`, `requested_by`, `cutoff`,
|
||||
`retention_interval`, `outcome` (PURGED, REFUSED), `reason`, `audits_deleted`,
|
||||
`details_deleted`, `links_deleted`, `anchor_link_id`, `anchor_hash`.
|
||||
|
||||
Nav (the app "WORM Audit"): Home, Verifications, Sealed Exports, Checkpoints, Audit Chain, Purges.
|
||||
Checkpoints list their links as a child list. Exports and verifications link their checkpoints
|
||||
and links.
|
||||
|
||||
## 8. Permissions and the assistant wall
|
||||
- Table permissions use the full name (`worm_audit_chain.read`, ...). There are no insert, edit or
|
||||
delete permissions to grant: the capabilities are absent.
|
||||
- Process permissions: `worm_audit.verify`, `worm_audit.export`. The home is visible with read on
|
||||
any `worm_audit_*` table.
|
||||
- The assistant channel is denied `worm_audit_*`. The qbit registers
|
||||
`ClientChannelsMetaData.ofOrWithNew(qInstance).withDeniedPattern("assistant", "worm_audit_*")`
|
||||
(core, QQQ 4.1.0-RC.6, #1141). On RC.5 it does so reflectively when the class is present. A
|
||||
one-line core PR also adds `worm_audit_*` to `ASSISTANT_DEFAULT_DENIED_PATTERNS`, so the wall
|
||||
holds even if the qbit is configured without its producer. Core's `audit*` pattern already
|
||||
covers the core audit tables.
|
||||
|
||||
## 9. App home
|
||||
A real home that owns the data (no duplicate dashboard), built from standard widgets, served
|
||||
through `/qqq/v1/widget` like every QQQ widget:
|
||||
- **Chain status**: links chained, last link time, unchained backlog (with the settle horizon),
|
||||
last verification (time, outcome, links), open breaks with the first break's exact row and a
|
||||
link to the verification record.
|
||||
- **Export status**: last sealed export (range, time, retain-until), lag in links and hours,
|
||||
gaps, and bucket lock state (COMPLIANCE, retention).
|
||||
- **Retention**: configured years, the purge anchor, the oldest live audit, the next eligible
|
||||
purge date.
|
||||
- **Recent purges**: the last 10 purge rows, refusals included.
|
||||
- **Setup warnings**: triggers missing on any table (checked via `pg_trigger`), runtime role owns
|
||||
the audit tables, signing key or trusted keys missing, bucket without Object Lock.
|
||||
|
||||
The home must be readable at every width (390 to 1920 px) in light and dark mode, with no inner
|
||||
horizontal scroll. Screenshots go in `docs/screenshots/`. If a custom endpoint is ever needed, it
|
||||
registers as a `MiddlewareSpecExtension` under `/qqq/v1/qbit/worm-audit/...`
|
||||
(docs/UI-API-EXTENSIONS.md). The design needs none.
|
||||
|
||||
## 10. Configuration (`WormAuditQBitConfig`)
|
||||
`retentionYears` (6), `signingKey` (supplier of an Ed25519 private key), `trustedPublicKeys`
|
||||
(key id to public key), `chainIntervalSeconds` (30), `settleHorizonSeconds` (60),
|
||||
`checkpointEveryLinks` (10,000), `checkpointEveryMinutes` (60), `exportBucketAlias` (`worm_audit`),
|
||||
`exportPrefix` (`worm-audit`), `exportCron` (02:00 daily), `verifyCron` (03:00 daily),
|
||||
`coreAuditTableNames` (physical names, defaulting to QQQ's), `tableMetaDataCustomizer`.
|
||||
|
||||
## 11. Tests (acceptance)
|
||||
1. **Postgres Testcontainers** (`postgres:16`): the rendered DDL applies twice (idempotent);
|
||||
UPDATE, DELETE and TRUNCATE on `audit`, `audit_detail` and each `worm_audit_*` table fail with
|
||||
the `worm_audit:` message; lookup renames fail and label changes succeed; the runtime role has
|
||||
no UPDATE/DELETE grant; `worm_audit_purge` refuses a recent cutoff (and records it), purges an
|
||||
expired prefix as the purger role, records the anchor, and verification passes from the
|
||||
anchor; setting `worm_audit.purging` from the app session does not allow DELETE.
|
||||
2. **Chain**: chain, verify (PASS); a forced tamper (triggers disabled as superuser, one
|
||||
`audit_detail.new_value` changed) is detected, naming that exact link, audit and detail; a
|
||||
deleted row, an inserted row, a reordered link and a forged checkpoint are each detected.
|
||||
3. **Concurrency**: 8 threads inserting audits (with late commits and details in separate
|
||||
statements) while 3 chainers run. The chain is contiguous, every audit and detail is covered
|
||||
exactly once, and verification passes.
|
||||
4. **Exports (MinIO Testcontainers, bucket with Object Lock)**: export, then verify the round
|
||||
trip; every object has COMPLIANCE retention; deleting or overwriting an object version is
|
||||
refused by MinIO; a tampered live row is reported by "Verify sealed export"; a bucket without
|
||||
Object Lock makes the export refuse and alert.
|
||||
5. **PHI**: a field classified PHI (`AiFieldMetaData.withClassification(DataClassification.PHI)`,
|
||||
read through `AiDataClassifications.ofField`) is written with the value `CANARY-PHI-<random>`.
|
||||
With secured-data's audit token hook (a stand-in producing the same `hmac:v1:` form in the
|
||||
qbit's tests until the core hook ships), the audit header, the details (old, new, message),
|
||||
the canonical content and the export JSONL contain neither the canary nor any substring of it
|
||||
longer than 4 characters.
|
||||
6. **Alerts**: a break raises `worm_audit.chain.break` through `QAlerts` (captured by a test
|
||||
sink); a later pass resolves it.
|
||||
7. Coverage gates: 70% instructions, 90% classes.
|
||||
|
||||
## 12. Decisions (all accepted as recommended, 2026-10-12)
|
||||
| # | Question | Decision |
|
||||
|---|---|---|
|
||||
| 1 | Chain construction | A single asynchronous chainer with a settle horizon, contiguous link ids and late-arrival links (5.2), not synchronous or per-partition chains. |
|
||||
| 2 | Chain storage | A sidecar table (`worm_audit_chain`), not columns on core tables, so core's schema is untouched and no UPDATE of audit rows is ever needed. |
|
||||
| 3 | Checkpoints | Hourly or every 10,000 links; Ed25519; Merkle root per range. |
|
||||
| 4 | Lookup tables | DELETE and id/name changes rejected; names resolved into the content. |
|
||||
| 5 | Purge | SQL function plus a purger role only, chain-prefix only, no UI action in 0.1. |
|
||||
| 6 | Export | Daily JSONL plus manifest, COMPLIANCE, six years, contiguous ranges, the exports form their own chain. |
|
||||
| 7 | Databases | Postgres enforced; MySQL and SQL Server documented; evidence on any backend. |
|
||||
|
||||
## 13. Phases
|
||||
- **(a)** DDL and renderer, capabilities guard, chain and chainer, checkpoints, verification
|
||||
action and daily job with alerts, the assistant deny pattern; tests 1–3, 5 and 6.
|
||||
- **(b)** Sealed exports and the export verifier through qbit-object-storage; test 4.
|
||||
- **(c)** App home and setup warnings, screenshots; kof22 adoption (draft Website-Backend PR, a
|
||||
feature changeset) and the infra proposal for an Object Lock bucket (section 14).
|
||||
|
||||
## 14. Adoption infrastructure (kof22, proposal only)
|
||||
- **Bucket:** `kof22-worm-audit` on the kof22 MinIO, created by `minio-bucket-init` with
|
||||
`mc mb --with-lock --ignore-existing` and `mc retention set --default COMPLIANCE 6y`. Object
|
||||
Lock can only be enabled at bucket creation.
|
||||
- **Policy:** `kof22-app` gets `s3:PutObject`, `s3:GetObject`, `s3:ListBucket`,
|
||||
`s3:GetObjectRetention`, `s3:PutObjectRetention` (needed to set the lock on each put),
|
||||
`s3:GetBucketObjectLockConfiguration` on that bucket. No delete, and no
|
||||
`s3:BypassGovernanceRetention` (irrelevant under COMPLIANCE anyway).
|
||||
- **Off-site:** COMPLIANCE on a single homelab MinIO stops every API deletion, but not someone
|
||||
destroying the volume. Recommend extending `media-offsite-copy` to replicate the bucket to a B2
|
||||
bucket with Object Lock (COMPLIANCE, 6 years).
|
||||
- **Secrets:** `WORM_AUDIT_SIGNING_KEY` (Ed25519 PKCS#8, base64) generated in Praetor
|
||||
`terraform/infisical-config` and delivered by ESO. The public key goes in the backend config.
|
||||
- **Database:** split the kof22 owner and runtime roles when convenient (4.3). Until then the
|
||||
home shows the owner warning.
|
||||
|
||||
These are owner-run (Praetor and cluster-gitops are applied by the owner). The qbit only
|
||||
documents them.
|
||||
@@ -0,0 +1,325 @@
|
||||
-- liquibase formatted sql
|
||||
|
||||
-- changeset worm-audit:1
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_chain (
|
||||
id bigint PRIMARY KEY,
|
||||
kind text NOT NULL,
|
||||
audit_id bigint NOT NULL,
|
||||
detail_ids text NOT NULL,
|
||||
prev_hash text NOT NULL,
|
||||
link_hash text NOT NULL,
|
||||
chained_at timestamptz NOT NULL,
|
||||
CHECK (id > 0),
|
||||
CHECK (kind IN ('AUDIT', 'LATE_DETAIL')),
|
||||
CHECK (prev_hash ~ '^[0-9a-f]{64}$' AND link_hash ~ '^[0-9a-f]{64}$'),
|
||||
CHECK (detail_ids = '' OR (detail_ids ~ '^(0|[1-9][0-9]*)(,(0|[1-9][0-9]*))*$' AND cardinality(string_to_array(detail_ids, ',')) <= 250))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_checkpoint (
|
||||
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
from_link_id bigint,
|
||||
to_link_id bigint,
|
||||
link_count bigint,
|
||||
head_hash text,
|
||||
merkle_root text,
|
||||
algorithm text,
|
||||
key_id text,
|
||||
public_key text,
|
||||
signature text,
|
||||
created_at timestamptz
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_export (
|
||||
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
from_link_id bigint,
|
||||
to_link_id bigint,
|
||||
from_checkpoint_id bigint,
|
||||
to_checkpoint_id bigint,
|
||||
bucket_alias text,
|
||||
object_prefix text,
|
||||
manifest_key text,
|
||||
manifest_sha256 text,
|
||||
prev_manifest_sha256 text,
|
||||
object_count bigint,
|
||||
byte_count bigint,
|
||||
lock_mode text,
|
||||
retain_until timestamptz,
|
||||
sealed_at timestamptz
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_verification (
|
||||
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
kind text,
|
||||
"trigger" text,
|
||||
started_at timestamptz,
|
||||
finished_at timestamptz,
|
||||
from_link_id bigint,
|
||||
to_link_id bigint,
|
||||
links_checked bigint,
|
||||
outcome text,
|
||||
break_count bigint,
|
||||
first_break_link_id bigint,
|
||||
first_break_audit_id bigint,
|
||||
first_break_reason text,
|
||||
export_id bigint,
|
||||
run_by text
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_purge (
|
||||
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
requested_at timestamptz,
|
||||
requested_by text,
|
||||
cutoff timestamptz,
|
||||
retention_interval text,
|
||||
outcome text,
|
||||
reason text,
|
||||
audits_deleted bigint,
|
||||
details_deleted bigint,
|
||||
links_deleted bigint,
|
||||
anchor_link_id bigint,
|
||||
anchor_hash text
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS worm_audit_chain_audit ON public.worm_audit_chain(audit_id) WHERE kind = 'AUDIT';
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_chain_audit_all ON public.worm_audit_chain(audit_id);
|
||||
|
||||
-- changeset worm-audit:2 splitStatements:false
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_reject() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp SET timezone = 'UTC' AS $$
|
||||
DECLARE
|
||||
old_time timestamptz;
|
||||
function_owner name;
|
||||
stack text;
|
||||
row_identity text := 'all';
|
||||
BEGIN
|
||||
IF TG_OP = 'DELETE' AND current_setting('worm_audit.purging', true) = 'on' THEN
|
||||
SELECT pg_get_userbyid(proowner) INTO function_owner FROM pg_proc
|
||||
WHERE oid = 'public.worm_audit_purge(timestamptz)'::regprocedure;
|
||||
GET DIAGNOSTICS stack = PG_CONTEXT;
|
||||
IF current_user = function_owner AND position('function worm_audit_purge(timestamp with time zone)' in stack) > 0 THEN
|
||||
IF TG_RELID = '"audit"'::regclass THEN
|
||||
old_time := OLD.timestamp;
|
||||
ELSIF TG_RELID = '"audit_detail"'::regclass THEN
|
||||
SELECT timestamp INTO old_time FROM "audit" WHERE id = OLD.audit_id;
|
||||
ELSIF TG_RELID = 'public.worm_audit_chain'::regclass THEN
|
||||
SELECT timestamp INTO old_time FROM "audit" WHERE id = OLD.audit_id;
|
||||
END IF;
|
||||
IF old_time < now() - interval '6 years' THEN
|
||||
RETURN OLD;
|
||||
END IF;
|
||||
END IF;
|
||||
END IF;
|
||||
IF TG_LEVEL = 'ROW' THEN
|
||||
row_identity := OLD.id::text;
|
||||
END IF;
|
||||
RAISE EXCEPTION USING ERRCODE = 'P0001',
|
||||
MESSAGE = format('worm_audit: audit rows are write-once; %s of "%s" is not allowed (id=%s). Expired rows are removed only by worm_audit_purge().',
|
||||
TG_OP, TG_TABLE_NAME, row_identity),
|
||||
HINT = 'see qbit-worm-audit docs/DESIGN.md section 4';
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:3 splitStatements:false
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_lookup_reject() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $$
|
||||
BEGIN
|
||||
IF TG_OP = 'UPDATE' AND NEW.id IS NOT DISTINCT FROM OLD.id AND NEW.name IS NOT DISTINCT FROM OLD.name THEN
|
||||
RETURN NEW;
|
||||
END IF;
|
||||
RAISE EXCEPTION USING ERRCODE = 'P0001',
|
||||
MESSAGE = format('worm_audit: audit rows are write-once; %s of "%s" is not allowed (id=%s). Expired rows are removed only by worm_audit_purge().',
|
||||
TG_OP, TG_TABLE_NAME, OLD.id),
|
||||
HINT = 'see qbit-worm-audit docs/DESIGN.md section 4';
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:4 splitStatements:false
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'worm_audit_purger') THEN
|
||||
CREATE ROLE worm_audit_purger NOLOGIN;
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:5 splitStatements:false
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_purge(p_cutoff timestamptz) RETURNS void
|
||||
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, public, pg_temp SET timezone = 'UTC' AS $$
|
||||
DECLARE
|
||||
boundary bigint;
|
||||
smaller bigint;
|
||||
anchor text;
|
||||
audits_count bigint;
|
||||
details_count bigint;
|
||||
links_count bigint;
|
||||
BEGIN
|
||||
IF p_cutoff IS NULL OR p_cutoff > now() - interval '6 years' THEN
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome, reason)
|
||||
VALUES(now(), session_user, p_cutoff, '6 years', 'REFUSED', 'Cutoff is within retention or null');
|
||||
RETURN;
|
||||
END IF;
|
||||
-- Same lock as the Java chainer; table locks also exclude concurrent audit inserts.
|
||||
PERFORM pg_advisory_xact_lock(86753090421);
|
||||
LOCK TABLE "audit", "audit_detail", public.worm_audit_chain IN SHARE ROW EXCLUSIVE MODE;
|
||||
SELECT coalesce(min(c.id) - 1, (SELECT max(id) FROM public.worm_audit_chain)) INTO boundary
|
||||
FROM public.worm_audit_chain c LEFT JOIN "audit" a ON a.id = c.audit_id
|
||||
WHERE a.id IS NULL OR a.timestamp IS NULL OR a.timestamp >= p_cutoff;
|
||||
-- Shrink to a prefix closed over every audit's links AND all its stored details.
|
||||
LOOP
|
||||
SELECT min(c.id) - 1 INTO smaller FROM public.worm_audit_chain c
|
||||
WHERE c.id <= boundary AND (
|
||||
EXISTS (SELECT FROM public.worm_audit_chain later WHERE later.audit_id = c.audit_id AND later.id > boundary)
|
||||
OR EXISTS (SELECT FROM "audit_detail" d WHERE d.audit_id = c.audit_id AND NOT EXISTS (
|
||||
SELECT FROM public.worm_audit_chain covered WHERE covered.audit_id = c.audit_id
|
||||
AND d.id::text = ANY(string_to_array(covered.detail_ids, ',')))));
|
||||
EXIT WHEN smaller IS NULL;
|
||||
boundary := smaller;
|
||||
END LOOP;
|
||||
SELECT link_hash INTO anchor FROM public.worm_audit_chain WHERE id = boundary;
|
||||
IF anchor IS NULL THEN
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome, reason)
|
||||
VALUES(now(), session_user, p_cutoff, '6 years', 'REFUSED', 'No complete expired chain prefix');
|
||||
RETURN;
|
||||
END IF;
|
||||
PERFORM set_config('worm_audit.purging', 'on', true);
|
||||
DELETE FROM "audit_detail" d WHERE EXISTS
|
||||
(SELECT FROM public.worm_audit_chain c WHERE c.audit_id = d.audit_id AND c.id <= boundary);
|
||||
GET DIAGNOSTICS details_count = ROW_COUNT;
|
||||
-- Keep ids locally because links must be deleted before their audit headers.
|
||||
WITH removed AS (DELETE FROM public.worm_audit_chain WHERE id <= boundary RETURNING audit_id),
|
||||
removed_audits AS (DELETE FROM "audit" WHERE id IN (SELECT audit_id FROM removed) RETURNING id)
|
||||
SELECT (SELECT count(*) FROM removed), (SELECT count(*) FROM removed_audits) INTO links_count, audits_count;
|
||||
PERFORM set_config('worm_audit.purging', 'off', true);
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome,
|
||||
audits_deleted, details_deleted, links_deleted, anchor_link_id, anchor_hash)
|
||||
VALUES(now(), session_user, p_cutoff, '6 years', 'PURGED', audits_count, details_count, links_count, boundary, anchor);
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:6
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON "audit";
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON "audit"
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON "audit";
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON "audit"
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON "audit" FROM "app";
|
||||
GRANT SELECT, INSERT ON "audit" TO "app";
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON "audit_detail";
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON "audit_detail"
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON "audit_detail";
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON "audit_detail"
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON "audit_detail" FROM "app";
|
||||
GRANT SELECT, INSERT ON "audit_detail" TO "app";
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_chain;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_chain
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_chain;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_chain
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_chain FROM "app";
|
||||
GRANT SELECT, INSERT ON public.worm_audit_chain TO "app";
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_checkpoint;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_checkpoint
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_checkpoint;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_checkpoint
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_checkpoint FROM "app";
|
||||
GRANT SELECT, INSERT ON public.worm_audit_checkpoint TO "app";
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_export;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_export
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_export;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_export
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_export FROM "app";
|
||||
GRANT SELECT, INSERT ON public.worm_audit_export TO "app";
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_verification;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_verification
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_verification;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_verification
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_verification FROM "app";
|
||||
GRANT SELECT, INSERT ON public.worm_audit_verification TO "app";
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_purge;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_purge
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_purge;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_purge
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_purge FROM "app";
|
||||
GRANT SELECT, INSERT ON public.worm_audit_purge TO "app";
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON "audit_table";
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON "audit_table"
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_lookup_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON "audit_table";
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON "audit_table"
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON "audit_table" FROM "app";
|
||||
GRANT SELECT, INSERT ON "audit_table" TO "app";
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON "audit_user";
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON "audit_user"
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_lookup_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON "audit_user";
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON "audit_user"
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON "audit_user" FROM "app";
|
||||
GRANT SELECT, INSERT ON "audit_user" TO "app";
|
||||
|
||||
-- Lookups may refresh display metadata, but never identities or names.
|
||||
GRANT UPDATE (label, create_date, modify_date) ON "audit_table" TO "app";
|
||||
GRANT UPDATE (create_date, modify_date) ON "audit_user" TO "app";
|
||||
|
||||
REVOKE ALL ON FUNCTION public.worm_audit_purge(timestamptz) FROM PUBLIC;
|
||||
REVOKE ALL ON FUNCTION public.worm_audit_purge(timestamptz) FROM "app";
|
||||
GRANT EXECUTE ON FUNCTION public.worm_audit_purge(timestamptz) TO worm_audit_purger;
|
||||
|
||||
-- changeset worm-audit:7 splitStatements:false
|
||||
DO $$
|
||||
DECLARE
|
||||
table_name text;
|
||||
sequence_name text;
|
||||
BEGIN
|
||||
FOREACH table_name IN ARRAY ARRAY['"audit"', '"audit_detail"', '"audit_table"', '"audit_user"',
|
||||
'public.worm_audit_checkpoint', 'public.worm_audit_export', 'public.worm_audit_verification', 'public.worm_audit_purge'] LOOP
|
||||
sequence_name := pg_get_serial_sequence(table_name, 'id');
|
||||
IF sequence_name IS NOT NULL THEN
|
||||
EXECUTE format('GRANT USAGE ON SEQUENCE %s TO "app"', sequence_name);
|
||||
END IF;
|
||||
END LOOP;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:8 splitStatements:false
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_detail_coverage() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $$
|
||||
BEGIN
|
||||
-- Serialize even direct SQL appends; the GIN index makes overlap checks indexed.
|
||||
PERFORM pg_advisory_xact_lock(86753090421);
|
||||
IF NEW.detail_ids <> '' AND (
|
||||
EXISTS (SELECT FROM unnest(string_to_array(NEW.detail_ids, ',')) AS d(id) GROUP BY d.id HAVING count(*) > 1)
|
||||
OR EXISTS (SELECT FROM public.worm_audit_chain c
|
||||
WHERE string_to_array(c.detail_ids, ',') && string_to_array(NEW.detail_ids, ','))) THEN
|
||||
RAISE EXCEPTION USING ERRCODE = '23505', MESSAGE = 'worm_audit: audit detail is already covered by the chain';
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:9
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_chain_detail_coverage ON public.worm_audit_chain USING gin (string_to_array(detail_ids, ','));
|
||||
DROP TRIGGER IF EXISTS worm_audit_detail_coverage_guard ON public.worm_audit_chain;
|
||||
CREATE TRIGGER worm_audit_detail_coverage_guard BEFORE INSERT ON public.worm_audit_chain
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_detail_coverage();
|
||||
@@ -0,0 +1,295 @@
|
||||
#!/bin/sh
|
||||
# ----------------------------------------------------------------------------
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Apache Maven Wrapper startup batch script, version 3.3.4
|
||||
#
|
||||
# Optional ENV vars
|
||||
# -----------------
|
||||
# JAVA_HOME - location of a JDK home dir, required when download maven via java source
|
||||
# MVNW_REPOURL - repo url base for downloading maven distribution
|
||||
# MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven
|
||||
# MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
set -euf
|
||||
[ "${MVNW_VERBOSE-}" != debug ] || set -x
|
||||
|
||||
# OS specific support.
|
||||
native_path() { printf %s\\n "$1"; }
|
||||
case "$(uname)" in
|
||||
CYGWIN* | MINGW*)
|
||||
[ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")"
|
||||
native_path() { cygpath --path --windows "$1"; }
|
||||
;;
|
||||
esac
|
||||
|
||||
# set JAVACMD and JAVACCMD
|
||||
set_java_home() {
|
||||
# For Cygwin and MinGW, ensure paths are in Unix format before anything is touched
|
||||
if [ -n "${JAVA_HOME-}" ]; then
|
||||
if [ -x "$JAVA_HOME/jre/sh/java" ]; then
|
||||
# IBM's JDK on AIX uses strange locations for the executables
|
||||
JAVACMD="$JAVA_HOME/jre/sh/java"
|
||||
JAVACCMD="$JAVA_HOME/jre/sh/javac"
|
||||
else
|
||||
JAVACMD="$JAVA_HOME/bin/java"
|
||||
JAVACCMD="$JAVA_HOME/bin/javac"
|
||||
|
||||
if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then
|
||||
echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2
|
||||
echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
else
|
||||
JAVACMD="$(
|
||||
'set' +e
|
||||
'unset' -f command 2>/dev/null
|
||||
'command' -v java
|
||||
)" || :
|
||||
JAVACCMD="$(
|
||||
'set' +e
|
||||
'unset' -f command 2>/dev/null
|
||||
'command' -v javac
|
||||
)" || :
|
||||
|
||||
if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then
|
||||
echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# hash string like Java String::hashCode
|
||||
hash_string() {
|
||||
str="${1:-}" h=0
|
||||
while [ -n "$str" ]; do
|
||||
char="${str%"${str#?}"}"
|
||||
h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296))
|
||||
str="${str#?}"
|
||||
done
|
||||
printf %x\\n $h
|
||||
}
|
||||
|
||||
verbose() { :; }
|
||||
[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; }
|
||||
|
||||
die() {
|
||||
printf %s\\n "$1" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
trim() {
|
||||
# MWRAPPER-139:
|
||||
# Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds.
|
||||
# Needed for removing poorly interpreted newline sequences when running in more
|
||||
# exotic environments such as mingw bash on Windows.
|
||||
printf "%s" "${1}" | tr -d '[:space:]'
|
||||
}
|
||||
|
||||
scriptDir="$(dirname "$0")"
|
||||
scriptName="$(basename "$0")"
|
||||
|
||||
# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties
|
||||
while IFS="=" read -r key value; do
|
||||
case "${key-}" in
|
||||
distributionUrl) distributionUrl=$(trim "${value-}") ;;
|
||||
distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;;
|
||||
esac
|
||||
done <"$scriptDir/.mvn/wrapper/maven-wrapper.properties"
|
||||
[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties"
|
||||
|
||||
case "${distributionUrl##*/}" in
|
||||
maven-mvnd-*bin.*)
|
||||
MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/
|
||||
case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in
|
||||
*AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;;
|
||||
:Darwin*x86_64) distributionPlatform=darwin-amd64 ;;
|
||||
:Darwin*arm64) distributionPlatform=darwin-aarch64 ;;
|
||||
:Linux*x86_64*) distributionPlatform=linux-amd64 ;;
|
||||
*)
|
||||
echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2
|
||||
distributionPlatform=linux-amd64
|
||||
;;
|
||||
esac
|
||||
distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip"
|
||||
;;
|
||||
maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;;
|
||||
*) MVN_CMD="mvn${scriptName#mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;;
|
||||
esac
|
||||
|
||||
# apply MVNW_REPOURL and calculate MAVEN_HOME
|
||||
# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-<version>,maven-mvnd-<version>-<platform>}/<hash>
|
||||
[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}"
|
||||
distributionUrlName="${distributionUrl##*/}"
|
||||
distributionUrlNameMain="${distributionUrlName%.*}"
|
||||
distributionUrlNameMain="${distributionUrlNameMain%-bin}"
|
||||
MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}"
|
||||
MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")"
|
||||
|
||||
exec_maven() {
|
||||
unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || :
|
||||
exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD"
|
||||
}
|
||||
|
||||
if [ -d "$MAVEN_HOME" ]; then
|
||||
verbose "found existing MAVEN_HOME at $MAVEN_HOME"
|
||||
exec_maven "$@"
|
||||
fi
|
||||
|
||||
case "${distributionUrl-}" in
|
||||
*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;;
|
||||
*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;;
|
||||
esac
|
||||
|
||||
# prepare tmp dir
|
||||
if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then
|
||||
clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; }
|
||||
trap clean HUP INT TERM EXIT
|
||||
else
|
||||
die "cannot create temp dir"
|
||||
fi
|
||||
|
||||
mkdir -p -- "${MAVEN_HOME%/*}"
|
||||
|
||||
# Download and Install Apache Maven
|
||||
verbose "Couldn't find MAVEN_HOME, downloading and installing it ..."
|
||||
verbose "Downloading from: $distributionUrl"
|
||||
verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName"
|
||||
|
||||
# select .zip or .tar.gz
|
||||
if ! command -v unzip >/dev/null; then
|
||||
distributionUrl="${distributionUrl%.zip}.tar.gz"
|
||||
distributionUrlName="${distributionUrl##*/}"
|
||||
fi
|
||||
|
||||
# verbose opt
|
||||
__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR=''
|
||||
[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v
|
||||
|
||||
# normalize http auth
|
||||
case "${MVNW_PASSWORD:+has-password}" in
|
||||
'') MVNW_USERNAME='' MVNW_PASSWORD='' ;;
|
||||
has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;;
|
||||
esac
|
||||
|
||||
if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then
|
||||
verbose "Found wget ... using wget"
|
||||
wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl"
|
||||
elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then
|
||||
verbose "Found curl ... using curl"
|
||||
curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl"
|
||||
elif set_java_home; then
|
||||
verbose "Falling back to use Java to download"
|
||||
javaSource="$TMP_DOWNLOAD_DIR/Downloader.java"
|
||||
targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName"
|
||||
cat >"$javaSource" <<-END
|
||||
public class Downloader extends java.net.Authenticator
|
||||
{
|
||||
protected java.net.PasswordAuthentication getPasswordAuthentication()
|
||||
{
|
||||
return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() );
|
||||
}
|
||||
public static void main( String[] args ) throws Exception
|
||||
{
|
||||
setDefault( new Downloader() );
|
||||
java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() );
|
||||
}
|
||||
}
|
||||
END
|
||||
# For Cygwin/MinGW, switch paths to Windows format before running javac and java
|
||||
verbose " - Compiling Downloader.java ..."
|
||||
"$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java"
|
||||
verbose " - Running Downloader.java ..."
|
||||
"$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")"
|
||||
fi
|
||||
|
||||
# If specified, validate the SHA-256 sum of the Maven distribution zip file
|
||||
if [ -n "${distributionSha256Sum-}" ]; then
|
||||
distributionSha256Result=false
|
||||
if [ "$MVN_CMD" = mvnd.sh ]; then
|
||||
echo "Checksum validation is not supported for maven-mvnd." >&2
|
||||
echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2
|
||||
exit 1
|
||||
elif command -v sha256sum >/dev/null; then
|
||||
if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c - >/dev/null 2>&1; then
|
||||
distributionSha256Result=true
|
||||
fi
|
||||
elif command -v shasum >/dev/null; then
|
||||
if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then
|
||||
distributionSha256Result=true
|
||||
fi
|
||||
else
|
||||
echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2
|
||||
echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ $distributionSha256Result = false ]; then
|
||||
echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2
|
||||
echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# unzip and move
|
||||
if command -v unzip >/dev/null; then
|
||||
unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip"
|
||||
else
|
||||
tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar"
|
||||
fi
|
||||
|
||||
# Find the actual extracted directory name (handles snapshots where filename != directory name)
|
||||
actualDistributionDir=""
|
||||
|
||||
# First try the expected directory name (for regular distributions)
|
||||
if [ -d "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" ]; then
|
||||
if [ -f "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/bin/$MVN_CMD" ]; then
|
||||
actualDistributionDir="$distributionUrlNameMain"
|
||||
fi
|
||||
fi
|
||||
|
||||
# If not found, search for any directory with the Maven executable (for snapshots)
|
||||
if [ -z "$actualDistributionDir" ]; then
|
||||
# enable globbing to iterate over items
|
||||
set +f
|
||||
for dir in "$TMP_DOWNLOAD_DIR"/*; do
|
||||
if [ -d "$dir" ]; then
|
||||
if [ -f "$dir/bin/$MVN_CMD" ]; then
|
||||
actualDistributionDir="$(basename "$dir")"
|
||||
break
|
||||
fi
|
||||
fi
|
||||
done
|
||||
set -f
|
||||
fi
|
||||
|
||||
if [ -z "$actualDistributionDir" ]; then
|
||||
verbose "Contents of $TMP_DOWNLOAD_DIR:"
|
||||
verbose "$(ls -la "$TMP_DOWNLOAD_DIR")"
|
||||
die "Could not find Maven distribution directory in extracted archive"
|
||||
fi
|
||||
|
||||
verbose "Found extracted Maven distribution directory: $actualDistributionDir"
|
||||
printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$actualDistributionDir/mvnw.url"
|
||||
mv -- "$TMP_DOWNLOAD_DIR/$actualDistributionDir" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME"
|
||||
|
||||
clean || :
|
||||
exec_maven "$@"
|
||||
@@ -0,0 +1,189 @@
|
||||
<# : batch portion
|
||||
@REM ----------------------------------------------------------------------------
|
||||
@REM Licensed to the Apache Software Foundation (ASF) under one
|
||||
@REM or more contributor license agreements. See the NOTICE file
|
||||
@REM distributed with this work for additional information
|
||||
@REM regarding copyright ownership. The ASF licenses this file
|
||||
@REM to you under the Apache License, Version 2.0 (the
|
||||
@REM "License"); you may not use this file except in compliance
|
||||
@REM with the License. You may obtain a copy of the License at
|
||||
@REM
|
||||
@REM http://www.apache.org/licenses/LICENSE-2.0
|
||||
@REM
|
||||
@REM Unless required by applicable law or agreed to in writing,
|
||||
@REM software distributed under the License is distributed on an
|
||||
@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
@REM KIND, either express or implied. See the License for the
|
||||
@REM specific language governing permissions and limitations
|
||||
@REM under the License.
|
||||
@REM ----------------------------------------------------------------------------
|
||||
|
||||
@REM ----------------------------------------------------------------------------
|
||||
@REM Apache Maven Wrapper startup batch script, version 3.3.4
|
||||
@REM
|
||||
@REM Optional ENV vars
|
||||
@REM MVNW_REPOURL - repo url base for downloading maven distribution
|
||||
@REM MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven
|
||||
@REM MVNW_VERBOSE - true: enable verbose log; others: silence the output
|
||||
@REM ----------------------------------------------------------------------------
|
||||
|
||||
@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0)
|
||||
@SET __MVNW_CMD__=
|
||||
@SET __MVNW_ERROR__=
|
||||
@SET __MVNW_PSMODULEP_SAVE=%PSModulePath%
|
||||
@SET PSModulePath=
|
||||
@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @(
|
||||
IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B)
|
||||
)
|
||||
@SET PSModulePath=%__MVNW_PSMODULEP_SAVE%
|
||||
@SET __MVNW_PSMODULEP_SAVE=
|
||||
@SET __MVNW_ARG0_NAME__=
|
||||
@SET MVNW_USERNAME=
|
||||
@SET MVNW_PASSWORD=
|
||||
@IF NOT "%__MVNW_CMD__%"=="" ("%__MVNW_CMD__%" %*)
|
||||
@echo Cannot start maven from wrapper >&2 && exit /b 1
|
||||
@GOTO :EOF
|
||||
: end batch / begin powershell #>
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
if ($env:MVNW_VERBOSE -eq "true") {
|
||||
$VerbosePreference = "Continue"
|
||||
}
|
||||
|
||||
# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties
|
||||
$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl
|
||||
if (!$distributionUrl) {
|
||||
Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties"
|
||||
}
|
||||
|
||||
switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) {
|
||||
"maven-mvnd-*" {
|
||||
$USE_MVND = $true
|
||||
$distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip"
|
||||
$MVN_CMD = "mvnd.cmd"
|
||||
break
|
||||
}
|
||||
default {
|
||||
$USE_MVND = $false
|
||||
$MVN_CMD = $script -replace '^mvnw','mvn'
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
# apply MVNW_REPOURL and calculate MAVEN_HOME
|
||||
# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-<version>,maven-mvnd-<version>-<platform>}/<hash>
|
||||
if ($env:MVNW_REPOURL) {
|
||||
$MVNW_REPO_PATTERN = if ($USE_MVND -eq $False) { "/org/apache/maven/" } else { "/maven/mvnd/" }
|
||||
$distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace "^.*$MVNW_REPO_PATTERN",'')"
|
||||
}
|
||||
$distributionUrlName = $distributionUrl -replace '^.*/',''
|
||||
$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$',''
|
||||
|
||||
$MAVEN_M2_PATH = "$HOME/.m2"
|
||||
if ($env:MAVEN_USER_HOME) {
|
||||
$MAVEN_M2_PATH = "$env:MAVEN_USER_HOME"
|
||||
}
|
||||
|
||||
if (-not (Test-Path -Path $MAVEN_M2_PATH)) {
|
||||
New-Item -Path $MAVEN_M2_PATH -ItemType Directory | Out-Null
|
||||
}
|
||||
|
||||
$MAVEN_WRAPPER_DISTS = $null
|
||||
if ((Get-Item $MAVEN_M2_PATH).Target[0] -eq $null) {
|
||||
$MAVEN_WRAPPER_DISTS = "$MAVEN_M2_PATH/wrapper/dists"
|
||||
} else {
|
||||
$MAVEN_WRAPPER_DISTS = (Get-Item $MAVEN_M2_PATH).Target[0] + "/wrapper/dists"
|
||||
}
|
||||
|
||||
$MAVEN_HOME_PARENT = "$MAVEN_WRAPPER_DISTS/$distributionUrlNameMain"
|
||||
$MAVEN_HOME_NAME = ([System.Security.Cryptography.SHA256]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join ''
|
||||
$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME"
|
||||
|
||||
if (Test-Path -Path "$MAVEN_HOME" -PathType Container) {
|
||||
Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME"
|
||||
Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD"
|
||||
exit $?
|
||||
}
|
||||
|
||||
if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) {
|
||||
Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl"
|
||||
}
|
||||
|
||||
# prepare tmp dir
|
||||
$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile
|
||||
$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir"
|
||||
$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null
|
||||
trap {
|
||||
if ($TMP_DOWNLOAD_DIR.Exists) {
|
||||
try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null }
|
||||
catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" }
|
||||
}
|
||||
}
|
||||
|
||||
New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null
|
||||
|
||||
# Download and Install Apache Maven
|
||||
Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..."
|
||||
Write-Verbose "Downloading from: $distributionUrl"
|
||||
Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName"
|
||||
|
||||
$webclient = New-Object System.Net.WebClient
|
||||
if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) {
|
||||
$webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD)
|
||||
}
|
||||
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
|
||||
$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null
|
||||
|
||||
# If specified, validate the SHA-256 sum of the Maven distribution zip file
|
||||
$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum
|
||||
if ($distributionSha256Sum) {
|
||||
if ($USE_MVND) {
|
||||
Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties."
|
||||
}
|
||||
Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash
|
||||
if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) {
|
||||
Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property."
|
||||
}
|
||||
}
|
||||
|
||||
# unzip and move
|
||||
Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null
|
||||
|
||||
# Find the actual extracted directory name (handles snapshots where filename != directory name)
|
||||
$actualDistributionDir = ""
|
||||
|
||||
# First try the expected directory name (for regular distributions)
|
||||
$expectedPath = Join-Path "$TMP_DOWNLOAD_DIR" "$distributionUrlNameMain"
|
||||
$expectedMvnPath = Join-Path "$expectedPath" "bin/$MVN_CMD"
|
||||
if ((Test-Path -Path $expectedPath -PathType Container) -and (Test-Path -Path $expectedMvnPath -PathType Leaf)) {
|
||||
$actualDistributionDir = $distributionUrlNameMain
|
||||
}
|
||||
|
||||
# If not found, search for any directory with the Maven executable (for snapshots)
|
||||
if (!$actualDistributionDir) {
|
||||
Get-ChildItem -Path "$TMP_DOWNLOAD_DIR" -Directory | ForEach-Object {
|
||||
$testPath = Join-Path $_.FullName "bin/$MVN_CMD"
|
||||
if (Test-Path -Path $testPath -PathType Leaf) {
|
||||
$actualDistributionDir = $_.Name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!$actualDistributionDir) {
|
||||
Write-Error "Could not find Maven distribution directory in extracted archive"
|
||||
}
|
||||
|
||||
Write-Verbose "Found extracted Maven distribution directory: $actualDistributionDir"
|
||||
Rename-Item -Path "$TMP_DOWNLOAD_DIR/$actualDistributionDir" -NewName $MAVEN_HOME_NAME | Out-Null
|
||||
try {
|
||||
Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null
|
||||
} catch {
|
||||
if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) {
|
||||
Write-Error "fail to move MAVEN_HOME"
|
||||
}
|
||||
} finally {
|
||||
try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null }
|
||||
catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" }
|
||||
}
|
||||
|
||||
Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD"
|
||||
@@ -1,42 +1,137 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
|
||||
http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<!--
|
||||
~ QQQ - Low-code Application Framework for Engineers.
|
||||
~ Copyright (C) 2021-2026. Kingsrook, LLC
|
||||
~ 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
~ contact@kingsrook.com
|
||||
~ https://github.com/Kingsrook/
|
||||
~
|
||||
~ Licensed under the Apache License, Version 2.0 (the "License");
|
||||
~ you may not use this file except in compliance with the License.
|
||||
~ You may obtain a copy of the License at
|
||||
~
|
||||
~ https://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
|
||||
<groupId>com.kingsrook.qbits</groupId>
|
||||
<artifactId>qbit-example-extension</artifactId>
|
||||
<version>0.1.0-SNAPSHOT</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>QBit Example Extension</name>
|
||||
<description>Example Extension QBit - rename this for your QBit</description>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
|
||||
<properties>
|
||||
<revision>0.1.0-SNAPSHOT</revision>
|
||||
<maven.compiler.source>21</maven.compiler.source>
|
||||
<maven.compiler.target>21</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<qqq.version>0.35.0</qqq.version>
|
||||
<jacoco.version>0.8.15</jacoco.version>
|
||||
<h2.version>2.2.224</h2.version>
|
||||
<testcontainers.version>2.0.5</testcontainers.version>
|
||||
</properties>
|
||||
|
||||
<parent>
|
||||
<groupId>com.kingsrook</groupId>
|
||||
<artifactId>qbit-build-parent</artifactId>
|
||||
<version>2.0.0</version>
|
||||
</parent>
|
||||
|
||||
<groupId>com.kingsrook.qbits</groupId>
|
||||
<artifactId>qbit-worm-audit</artifactId>
|
||||
<version>${revision}</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>QBit WORM Audit</name>
|
||||
<description>QBit for write-once (WORM) audit in QQQ applications: database-enforced append-only audit tables, hash-chained tamper evidence with signed checkpoints, verification, and sealed exports to S3 Object Lock</description>
|
||||
<url>https://github.com/QRun-IO/qbit-worm-audit</url>
|
||||
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<licenses>
|
||||
<license>
|
||||
<name>Apache License, Version 2.0</name>
|
||||
<url>https://www.apache.org/licenses/LICENSE-2.0</url>
|
||||
<distribution>repo</distribution>
|
||||
</license>
|
||||
</licenses>
|
||||
|
||||
<developers>
|
||||
<developer>
|
||||
<id>kingsrook</id>
|
||||
<name>Kingsrook Team</name>
|
||||
<email>contact@kingsrook.com</email>
|
||||
<organization>Kingsrook, LLC</organization>
|
||||
<organizationUrl>https://kingsrook.com</organizationUrl>
|
||||
</developer>
|
||||
</developers>
|
||||
|
||||
<scm>
|
||||
<connection>scm:git:git://github.com/QRun-IO/qbit-worm-audit.git</connection>
|
||||
<developerConnection>scm:git:ssh://github.com:QRun-IO/qbit-worm-audit.git</developerConnection>
|
||||
<url>https://github.com/QRun-IO/qbit-worm-audit</url>
|
||||
<tag>HEAD</tag>
|
||||
</scm>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
<artifactId>qqq-backend-core</artifactId>
|
||||
<version>${qqq.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- The memory backend is included in qqq-backend-core. -->
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
<artifactId>qqq-backend-module-rdbms</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
<artifactId>qqq-middleware-javalin</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
<artifactId>qqq-frontend-next</artifactId>
|
||||
<version>1.0.0-RC.14</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.h2database</groupId>
|
||||
<artifactId>h2</artifactId>
|
||||
<version>${h2.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.postgresql</groupId>
|
||||
<artifactId>postgresql</artifactId>
|
||||
<version>42.7.14</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.testcontainers</groupId>
|
||||
<artifactId>testcontainers-postgresql</artifactId>
|
||||
<version>${testcontainers.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.testcontainers</groupId>
|
||||
<artifactId>testcontainers-minio</artifactId>
|
||||
<version>${testcontainers.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.testcontainers</groupId>
|
||||
<artifactId>testcontainers-junit-jupiter</artifactId>
|
||||
<version>${testcontainers.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter</artifactId>
|
||||
<version>5.10.0</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.assertj</groupId>
|
||||
<artifactId>assertj-core</artifactId>
|
||||
<version>3.24.2</version>
|
||||
<version>3.27.7</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
@@ -45,14 +140,105 @@
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.11.0</version>
|
||||
<artifactId>maven-checkstyle-plugin</artifactId>
|
||||
<version>3.3.1</version>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.puppycrawl.tools</groupId>
|
||||
<artifactId>checkstyle</artifactId>
|
||||
<version>10.12.6</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<configuration>
|
||||
<configLocation>checkstyle/config.xml</configLocation>
|
||||
<headerLocation>checkstyle/license.txt</headerLocation>
|
||||
<encoding>UTF-8</encoding>
|
||||
<consoleOutput>true</consoleOutput>
|
||||
<failsOnError>false</failsOnError>
|
||||
<failOnViolation>true</failOnViolation>
|
||||
<violationSeverity>warning</violationSeverity>
|
||||
<includeTestSourceDirectory>false</includeTestSourceDirectory>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>validate</id>
|
||||
<phase>validate</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<version>3.1.2</version>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>${jacoco.version}</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>check</id>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.70</minimum>
|
||||
</limit>
|
||||
<limit>
|
||||
<counter>CLASS</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.90</minimum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
|
||||
<profiles>
|
||||
<!-- QQQ 4.1 APIs until qbit-build-parent imports the 4.1 BOM (as in qbit-quick-search and qbit-cdn, ADR-0007). -->
|
||||
<profile>
|
||||
<id>qqq-snapshot</id>
|
||||
<activation>
|
||||
<activeByDefault>true</activeByDefault>
|
||||
</activation>
|
||||
<properties>
|
||||
<!-- Latest QQQ RC on Maven Central (2026-10-13). -->
|
||||
<qqq.snapshot.version>4.1.0-RC.5</qqq.snapshot.version>
|
||||
</properties>
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
<artifactId>qqq-bom-pom</artifactId>
|
||||
<version>${qqq.snapshot.version}</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
</profile>
|
||||
</profiles>
|
||||
</project>
|
||||
@@ -1,75 +0,0 @@
|
||||
/*******************************************************************************
|
||||
** Configuration for the Example Extension QBit.
|
||||
*******************************************************************************/
|
||||
package com.kingsrook.qbits.example;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configuration class for the Example Extension QBit.
|
||||
*******************************************************************************/
|
||||
public class ExampleExtensionQBitConfig implements QBitConfig
|
||||
{
|
||||
private String targetTableName;
|
||||
private Boolean enableFeatureX = true;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Validate configuration before QBit is produced.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void validate(QInstance qInstance, List<String> errors)
|
||||
{
|
||||
if(targetTableName != null && qInstance.getTable(targetTableName) == null)
|
||||
{
|
||||
errors.add("Target table not found: " + targetTableName);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for targetTableName
|
||||
*******************************************************************************/
|
||||
public String getTargetTableName()
|
||||
{
|
||||
return targetTableName;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for targetTableName
|
||||
*******************************************************************************/
|
||||
public ExampleExtensionQBitConfig withTargetTableName(String targetTableName)
|
||||
{
|
||||
this.targetTableName = targetTableName;
|
||||
return this;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for enableFeatureX
|
||||
*******************************************************************************/
|
||||
public Boolean getEnableFeatureX()
|
||||
{
|
||||
return enableFeatureX;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for enableFeatureX
|
||||
*******************************************************************************/
|
||||
public ExampleExtensionQBitConfig withEnableFeatureX(Boolean enableFeatureX)
|
||||
{
|
||||
this.enableFeatureX = enableFeatureX;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
/*******************************************************************************
|
||||
** Producer for the Example Extension QBit.
|
||||
*******************************************************************************/
|
||||
package com.kingsrook.qbits.example;
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitProducer;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Producer class for the Example Extension QBit.
|
||||
*******************************************************************************/
|
||||
public class ExampleExtensionQBitProducer implements QBitProducer
|
||||
{
|
||||
public static final String GROUP_ID = "com.kingsrook.qbits";
|
||||
public static final String ARTIFACT_ID = "qbit-example-extension";
|
||||
public static final String VERSION = "0.1.0";
|
||||
|
||||
private ExampleExtensionQBitConfig config;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Produce this QBit into the given QInstance.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void produce(QInstance qInstance, String namespace) throws QException
|
||||
{
|
||||
QBitMetaData qBitMetaData = new QBitMetaData()
|
||||
.withGroupId(GROUP_ID)
|
||||
.withArtifactId(ARTIFACT_ID)
|
||||
.withVersion(VERSION)
|
||||
.withNamespace(namespace)
|
||||
.withConfig(config);
|
||||
|
||||
qInstance.addQBit(qBitMetaData);
|
||||
|
||||
if(config != null && config.getTargetTableName() != null)
|
||||
{
|
||||
applyTableCustomizations(qInstance, config.getTargetTableName());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Apply customizations to the target table.
|
||||
*******************************************************************************/
|
||||
private void applyTableCustomizations(QInstance qInstance, String tableName)
|
||||
{
|
||||
// Implement your extension logic here
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for config
|
||||
*******************************************************************************/
|
||||
public ExampleExtensionQBitProducer withConfig(ExampleExtensionQBitConfig config)
|
||||
{
|
||||
this.config = config;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
/*******************************************************************************
|
||||
** Example table customizer.
|
||||
*******************************************************************************/
|
||||
package com.kingsrook.qbits.example.customizers;
|
||||
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Example table customizer that adds behavior to existing tables.
|
||||
*******************************************************************************/
|
||||
public class ExampleTableCustomizer
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
** Apply customizations to the given table.
|
||||
*******************************************************************************/
|
||||
public void customize(QTableMetaData table)
|
||||
{
|
||||
// Example: Add audit tracking fields
|
||||
// table.withField(new QFieldMetaData("createdAt", QFieldType.DATE_TIME));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,361 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import java.security.PrivateKey;
|
||||
import java.security.PublicKey;
|
||||
import java.util.Map;
|
||||
import java.util.function.Supplier;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Host configuration. Private signing material never enters metadata records.
|
||||
*******************************************************************************/
|
||||
public class WormAuditQBitConfig implements QBitConfig
|
||||
{
|
||||
public static final int DEFAULT_RETENTION_YEARS = 6;
|
||||
|
||||
/*******************************************************************************
|
||||
** Signing identity; the matching public key must be pinned in trustedPublicKeys.
|
||||
*******************************************************************************/
|
||||
public record SigningKey(String keyId, PrivateKey privateKey)
|
||||
{
|
||||
}
|
||||
|
||||
private String backendName = null;
|
||||
private int retentionYears = 6;
|
||||
private Supplier<SigningKey> signingKey = null;
|
||||
private Map<String, PublicKey> trustedPublicKeys = Map.of();
|
||||
private int chainIntervalSeconds = 30;
|
||||
private int settleHorizonSeconds = 60;
|
||||
private int batchSize = 1000;
|
||||
private int lateDetailLookbackHours = 24;
|
||||
private int checkpointEveryLinks = 10000;
|
||||
private int checkpointEveryMinutes = 60;
|
||||
private String verifyCron = "0 0 3 * * ?";
|
||||
private String schedulerName = "worm_audit";
|
||||
private Map<String, String> coreAuditTableNames = Map.of();
|
||||
private String appRole = "app";
|
||||
private MetaDataCustomizerInterface<QTableMetaData> tableMetaDataCustomizer = null;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read backendName.
|
||||
*******************************************************************************/
|
||||
public String getBackendName()
|
||||
{
|
||||
return (backendName);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure backendName.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withBackendName(String backendName)
|
||||
{
|
||||
this.backendName = backendName;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read retentionYears.
|
||||
*******************************************************************************/
|
||||
public int getRetentionYears()
|
||||
{
|
||||
return (retentionYears);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure retentionYears.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withRetentionYears(int retentionYears)
|
||||
{
|
||||
this.retentionYears = retentionYears;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read signingKey.
|
||||
*******************************************************************************/
|
||||
public Supplier<SigningKey> getSigningKey()
|
||||
{
|
||||
return (signingKey);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure signingKey.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withSigningKey(Supplier<SigningKey> signingKey)
|
||||
{
|
||||
this.signingKey = signingKey;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read trustedPublicKeys.
|
||||
*******************************************************************************/
|
||||
public Map<String, PublicKey> getTrustedPublicKeys()
|
||||
{
|
||||
return (trustedPublicKeys);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure trustedPublicKeys.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withTrustedPublicKeys(Map<String, PublicKey> trustedPublicKeys)
|
||||
{
|
||||
this.trustedPublicKeys = trustedPublicKeys;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read chainIntervalSeconds.
|
||||
*******************************************************************************/
|
||||
public int getChainIntervalSeconds()
|
||||
{
|
||||
return (chainIntervalSeconds);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure chainIntervalSeconds.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withChainIntervalSeconds(int chainIntervalSeconds)
|
||||
{
|
||||
this.chainIntervalSeconds = chainIntervalSeconds;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read settleHorizonSeconds.
|
||||
*******************************************************************************/
|
||||
public int getSettleHorizonSeconds()
|
||||
{
|
||||
return (settleHorizonSeconds);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure settleHorizonSeconds.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withSettleHorizonSeconds(int settleHorizonSeconds)
|
||||
{
|
||||
this.settleHorizonSeconds = settleHorizonSeconds;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the bounded late-detail polling window in hours.
|
||||
*******************************************************************************/
|
||||
public int getLateDetailLookbackHours()
|
||||
{
|
||||
return (lateDetailLookbackHours);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure late-detail polling; older omissions are reported by verification.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withLateDetailLookbackHours(int lateDetailLookbackHours)
|
||||
{
|
||||
this.lateDetailLookbackHours = lateDetailLookbackHours;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read batchSize.
|
||||
*******************************************************************************/
|
||||
public int getBatchSize()
|
||||
{
|
||||
return (batchSize);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure batchSize.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withBatchSize(int batchSize)
|
||||
{
|
||||
this.batchSize = batchSize;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read checkpointEveryLinks.
|
||||
*******************************************************************************/
|
||||
public int getCheckpointEveryLinks()
|
||||
{
|
||||
return (checkpointEveryLinks);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure checkpointEveryLinks.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withCheckpointEveryLinks(int checkpointEveryLinks)
|
||||
{
|
||||
this.checkpointEveryLinks = checkpointEveryLinks;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read checkpointEveryMinutes.
|
||||
*******************************************************************************/
|
||||
public int getCheckpointEveryMinutes()
|
||||
{
|
||||
return (checkpointEveryMinutes);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure checkpointEveryMinutes.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withCheckpointEveryMinutes(int checkpointEveryMinutes)
|
||||
{
|
||||
this.checkpointEveryMinutes = checkpointEveryMinutes;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read verifyCron.
|
||||
*******************************************************************************/
|
||||
public String getVerifyCron()
|
||||
{
|
||||
return (verifyCron);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure verifyCron.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withVerifyCron(String verifyCron)
|
||||
{
|
||||
this.verifyCron = verifyCron;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read schedulerName.
|
||||
*******************************************************************************/
|
||||
public String getSchedulerName()
|
||||
{
|
||||
return (schedulerName);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure schedulerName.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withSchedulerName(String schedulerName)
|
||||
{
|
||||
this.schedulerName = schedulerName;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read coreAuditTableNames.
|
||||
*******************************************************************************/
|
||||
public Map<String, String> getCoreAuditTableNames()
|
||||
{
|
||||
return (coreAuditTableNames);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure coreAuditTableNames.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withCoreAuditTableNames(Map<String, String> coreAuditTableNames)
|
||||
{
|
||||
this.coreAuditTableNames = coreAuditTableNames;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read appRole.
|
||||
*******************************************************************************/
|
||||
public String getAppRole()
|
||||
{
|
||||
return (appRole);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure appRole.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withAppRole(String appRole)
|
||||
{
|
||||
this.appRole = appRole;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read tableMetaDataCustomizer.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public MetaDataCustomizerInterface<QTableMetaData> getTableMetaDataCustomizer()
|
||||
{
|
||||
return (tableMetaDataCustomizer);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Configure tableMetaDataCustomizer.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitConfig withTableMetaDataCustomizer(MetaDataCustomizerInterface<QTableMetaData> tableMetaDataCustomizer)
|
||||
{
|
||||
this.tableMetaDataCustomizer = tableMetaDataCustomizer;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reject invalid runtime limits before producing metadata.
|
||||
*******************************************************************************/
|
||||
public void validate()
|
||||
{
|
||||
if(backendName == null || backendName.isBlank() || retentionYears < 1 || chainIntervalSeconds < 1
|
||||
|| lateDetailLookbackHours < 1 || settleHorizonSeconds < 0 || batchSize < 1 || checkpointEveryLinks < 1 || checkpointEveryMinutes < 1)
|
||||
{
|
||||
throw (new IllegalArgumentException("backendName is required and WORM audit limits must be positive (settle horizon may be zero)"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
|
||||
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
|
||||
import com.kingsrook.qbits.wormaudit.guard.AppendOnlyGuard;
|
||||
import com.kingsrook.qbits.wormaudit.model.LinkRangeWidget;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.WormAuditProcesses;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaDataProducer;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.quartz.QuartzSchedulerMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Produces append-only audit metadata, verification actions and schedules.
|
||||
*******************************************************************************/
|
||||
public class WormAuditQBitProducer implements QBitMetaDataProducer<WormAuditQBitConfig>
|
||||
{
|
||||
public static final String GROUP_ID = "com.kingsrook.qbits";
|
||||
public static final String ARTIFACT_ID = "qbit-worm-audit";
|
||||
public static final String VERSION = "0.1.0";
|
||||
|
||||
private WormAuditQBitConfig qBitConfig;
|
||||
private String namespace;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for the config.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitProducer withQBitConfig(WormAuditQBitConfig config)
|
||||
{
|
||||
this.qBitConfig = config;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for the config.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public WormAuditQBitConfig getQBitConfig()
|
||||
{
|
||||
return (qBitConfig);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for the namespace.
|
||||
*******************************************************************************/
|
||||
public WormAuditQBitProducer withNamespace(String namespace)
|
||||
{
|
||||
this.namespace = namespace;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for the namespace.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getNamespace()
|
||||
{
|
||||
return (namespace);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Identity and configuration of this qbit.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QBitMetaData getQBitMetaData()
|
||||
{
|
||||
return (new QBitMetaData().withGroupId(GROUP_ID).withArtifactId(ARTIFACT_ID).withVersion(VERSION).withNamespace(namespace).withConfig(qBitConfig));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Package-scanned producers have no config: skip them.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean isEnabled()
|
||||
{
|
||||
return (qBitConfig != null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Register phase (a) tables, processes and schedules on the host backend.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public MetaDataProducerMultiOutput produce(QInstance qInstance) throws QException
|
||||
{
|
||||
qBitConfig.validate();
|
||||
if(qInstance.getBackend(qBitConfig.getBackendName()) == null)
|
||||
{
|
||||
throw (new IllegalArgumentException("Configured audit backend is missing"));
|
||||
}
|
||||
WormAuditDdl.resolveNames(qBitConfig, qInstance);
|
||||
for(String core : AppendOnlyGuard.CORE_TABLES)
|
||||
{
|
||||
if(!qBitConfig.getBackendName().equals(qInstance.getTable(core).getBackendName()))
|
||||
{
|
||||
throw (new IllegalArgumentException("Core and WORM audit tables must share one backend"));
|
||||
}
|
||||
}
|
||||
QBitMetaData qBit = getQBitMetaData();
|
||||
MetaDataProducerMultiOutput output = new MetaDataProducerMultiOutput().with(qBit);
|
||||
AppendOnlyGuard guard = new AppendOnlyGuard();
|
||||
for(QTableMetaData table : WormAuditTables.tables(qBitConfig))
|
||||
{
|
||||
if(qBitConfig.getTableMetaDataCustomizer() != null)
|
||||
{
|
||||
table = qBitConfig.getTableMetaDataCustomizer().customizeMetaData(qInstance, table);
|
||||
}
|
||||
guard.customizeMetaData(qInstance, table);
|
||||
table.setSourceQBitName(qBit.getName());
|
||||
output.add(table);
|
||||
}
|
||||
for(String core : AppendOnlyGuard.CORE_TABLES)
|
||||
{
|
||||
guard.customizeMetaData(qInstance, qInstance.getTable(core));
|
||||
}
|
||||
for(String name : WormAuditTables.ALL)
|
||||
{
|
||||
output.add(QPossibleValueSource.newForTable(name));
|
||||
}
|
||||
for(String name : java.util.List.of(WormAuditTables.CHECKPOINT, WormAuditTables.EXPORT, WormAuditTables.VERIFICATION))
|
||||
{
|
||||
output.add(new QWidgetMetaData().withName(name + "_links").withLabel("Chain links").withType("childRecordList")
|
||||
.withCodeReference(new QCodeReference(LinkRangeWidget.class)).withDefaultValue("parentTable", name));
|
||||
}
|
||||
for(var process : WormAuditProcesses.processes(qBitConfig))
|
||||
{
|
||||
process.setSourceQBitName(qBit.getName());
|
||||
output.add(process);
|
||||
}
|
||||
if(qInstance.getSchedulers() == null || !qInstance.getSchedulers().containsKey(qBitConfig.getSchedulerName()))
|
||||
{
|
||||
qInstance.addScheduler(new QuartzSchedulerMetaData().withName(qBitConfig.getSchedulerName()));
|
||||
}
|
||||
QAppMetaData app = new QAppMetaData().withName("worm_audit").withLabel("WORM Audit").withIcon(new QIcon("policy"));
|
||||
for(String name : WormAuditTables.NAV)
|
||||
{
|
||||
QTableMetaData table = output.getEach(QTableMetaData.class).stream().filter(t -> t.getName().equals(name)).findFirst().orElseThrow();
|
||||
app.withSectionOfChildren(new QAppSection().withName(name).withLabel(table.getLabel()), table);
|
||||
}
|
||||
output.add(app);
|
||||
// RC.5 rejects underscores in declared alert type names. QAlerts still
|
||||
// dispatches the specified worm_audit.chain.break signal to installed sinks.
|
||||
output.addSelfToInstance(qInstance);
|
||||
denyAssistant(qInstance);
|
||||
return (new MetaDataProducerMultiOutput());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** RC.5 compatibility: replace with a direct call when the qbit builds on RC.6 (qqq#1141, #1163).
|
||||
*******************************************************************************/
|
||||
private void denyAssistant(QInstance instance) throws QException
|
||||
{
|
||||
try
|
||||
{
|
||||
Class<?> type = Class.forName("com.kingsrook.qqq.backend.core.model.metadata.permissions.ClientChannelsMetaData");
|
||||
Object metadata = type.getMethod("ofOrWithNew", QInstance.class).invoke(null, instance);
|
||||
type.getMethod("withDeniedPattern", String.class, String.class).invoke(metadata, "assistant", "worm_audit_*");
|
||||
}
|
||||
catch(ClassNotFoundException expected)
|
||||
{
|
||||
// The assistant wall API is absent in RC.5.
|
||||
}
|
||||
catch(ReflectiveOperationException e)
|
||||
{
|
||||
throw (new QException("Cannot register the WORM audit assistant deny pattern", e));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.canonical;
|
||||
|
||||
|
||||
import java.time.ZoneOffset;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Comparator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** v1 audit content, exactly as stored. Subsets use the same document schema for
|
||||
** LATE_DETAIL links; no canonical content is persisted by this qbit.
|
||||
*******************************************************************************/
|
||||
public final class AuditCanonical
|
||||
{
|
||||
private static final DateTimeFormatter TIMESTAMP = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'").withZone(ZoneOffset.UTC);
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private AuditCanonical()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve names outside this function, without looking up application data.
|
||||
*******************************************************************************/
|
||||
public static byte[] bytes(QRecord audit, String table, String user, List<QRecord> details)
|
||||
{
|
||||
Map<String, Object> header = new LinkedHashMap<>();
|
||||
header.put("clientChannel", audit.getValueString("clientChannel"));
|
||||
header.put("id", audit.getValueLong("id"));
|
||||
header.put("message", audit.getValueString("message"));
|
||||
header.put("recordId", audit.getValueString("recordId"));
|
||||
header.put("table", table);
|
||||
header.put("timestamp", TIMESTAMP.format(audit.getValueInstant("timestamp")));
|
||||
header.put("user", user);
|
||||
List<Map<String, Object>> rows = details.stream().sorted(Comparator.comparing(r -> r.getValueLong("id"))).map(detail ->
|
||||
{
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
row.put("id", detail.getValueLong("id"));
|
||||
for(String field : List.of("fieldName", "message", "newValue", "oldValue"))
|
||||
{
|
||||
row.put(field, detail.getValueString(field));
|
||||
}
|
||||
return (row);
|
||||
}).toList();
|
||||
return (Jcs.bytes(Map.of("audit", header, "details", rows, "v", 1)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.canonical;
|
||||
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** RFC 8785 serialization of the audit schema: strings, integer longs, null,
|
||||
** objects and arrays. Java String ordering is UTF-16 code-unit ordering. Reject
|
||||
** malformed surrogate pairs instead of silently replacing them during UTF-8 encoding.
|
||||
*******************************************************************************/
|
||||
public final class Jcs
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private Jcs()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Encode a document with no insignificant whitespace.
|
||||
*******************************************************************************/
|
||||
public static byte[] bytes(Object value)
|
||||
{
|
||||
return (string(value).getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Serialize only the value types supported by the v1 schema.
|
||||
*******************************************************************************/
|
||||
public static String string(Object value)
|
||||
{
|
||||
if(value == null)
|
||||
{
|
||||
return ("null");
|
||||
}
|
||||
if(value instanceof String text)
|
||||
{
|
||||
return (quote(text));
|
||||
}
|
||||
if(value instanceof Long || value instanceof Integer)
|
||||
{
|
||||
return (value.toString());
|
||||
}
|
||||
if(value instanceof List<?> list)
|
||||
{
|
||||
return ("[" + String.join(",", list.stream().map(Jcs::string).toList()) + "]");
|
||||
}
|
||||
if(value instanceof Map<?, ?> map)
|
||||
{
|
||||
TreeMap<String, Object> sorted = new TreeMap<>();
|
||||
map.forEach((key, item) -> sorted.put((String) key, item));
|
||||
return ("{" + String.join(",", sorted.entrySet().stream().map(e -> quote(e.getKey()) + ":" + string(e.getValue())).toList()) + "}");
|
||||
}
|
||||
throw (new IllegalArgumentException("Unsupported canonical value type"));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** JSON escaping mandated by JCS, including lowercase control escapes.
|
||||
*******************************************************************************/
|
||||
private static String quote(String text)
|
||||
{
|
||||
StringBuilder out = new StringBuilder("\"");
|
||||
for(int i = 0; i < text.length(); i++)
|
||||
{
|
||||
char c = text.charAt(i);
|
||||
switch(c)
|
||||
{
|
||||
case '"' -> out.append("\\\"");
|
||||
case '\\' -> out.append("\\\\");
|
||||
case '\b' -> out.append("\\b");
|
||||
case '\f' -> out.append("\\f");
|
||||
case '\n' -> out.append("\\n");
|
||||
case '\r' -> out.append("\\r");
|
||||
case '\t' -> out.append("\\t");
|
||||
default ->
|
||||
{
|
||||
if(c < 32)
|
||||
{
|
||||
out.append(String.format(java.util.Locale.ROOT, "\\u%04x", (int) c));
|
||||
}
|
||||
else if(Character.isHighSurrogate(c))
|
||||
{
|
||||
if(i + 1 >= text.length() || !Character.isLowSurrogate(text.charAt(i + 1)))
|
||||
{
|
||||
throw (new IllegalArgumentException("Unpaired surrogate in canonical string"));
|
||||
}
|
||||
out.append(c).append(text.charAt(++i));
|
||||
}
|
||||
else if(Character.isLowSurrogate(c))
|
||||
{
|
||||
throw (new IllegalArgumentException("Unpaired surrogate in canonical string"));
|
||||
}
|
||||
else
|
||||
{
|
||||
out.append(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return (out.append('"').toString());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,479 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.chain;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.Timestamp;
|
||||
import java.sql.Types;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import com.kingsrook.qbits.wormaudit.canonical.AuditCanonical;
|
||||
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
|
||||
import com.kingsrook.qbits.wormaudit.guard.AppendOnlyGuard;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.VerificationResult;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSTransaction;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** QQQ storage shared by chainer, verifier and future export code. All reads use
|
||||
** the caller's transaction; only insert operations exist here. Bypass applies
|
||||
** to internal catalog work after process permission checks, never user requests.
|
||||
*******************************************************************************/
|
||||
public class AuditStore
|
||||
{
|
||||
private final QBackendTransaction transaction;
|
||||
private final Map<String, String> names = new LinkedHashMap<>(16, 0.75F, true);
|
||||
private Boolean timestampWithZone;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Bind operations to one snapshot/transaction, or the memory backend.
|
||||
*******************************************************************************/
|
||||
public AuditStore(QBackendTransaction transaction)
|
||||
{
|
||||
this.transaction = transaction;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read in deterministic id order, explicitly fetching TEXT fields.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> query(String table, QQueryFilter filter) throws QException
|
||||
{
|
||||
QueryInput input = new QueryInput(table).withFilter(filter).withTransaction(transaction).withShouldFetchHeavyFields(true);
|
||||
input.withPermissionBypass("WORM audit internal read");
|
||||
return (new QueryAction().execute(input).getRecords());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** All records ordered by id.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> all(String table) throws QException
|
||||
{
|
||||
return (query(table, new QQueryFilter().withOrderBy(new QFilterOrderBy("id", true))));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** An equality query.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> where(String table, String field, Serializable value) throws QException
|
||||
{
|
||||
// Core lookup keys are INTEGER while audit and chain keys are LONG.
|
||||
// Memory equality is type-sensitive; normalize to the declared field type.
|
||||
if(value instanceof Number number && QContext.getQInstance().getTable(table).getField(field).getType()
|
||||
== com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType.INTEGER)
|
||||
{
|
||||
value = Math.toIntExact(number.longValue());
|
||||
}
|
||||
return (query(table, new QQueryFilter(new QFilterCriteria(field, QCriteriaOperator.EQUALS, value)).withOrderBy(new QFilterOrderBy("id", true))));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Return a row or null if a tamper removed it.
|
||||
*******************************************************************************/
|
||||
public QRecord get(String table, long id) throws QException
|
||||
{
|
||||
return (where(table, "id", id).stream().findFirst().orElse(null));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Insert a batch, propagating failures so a transaction can roll back.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> insert(String table, List<QRecord> rows) throws QException
|
||||
{
|
||||
if(rows.isEmpty())
|
||||
{
|
||||
return (rows);
|
||||
}
|
||||
try
|
||||
{
|
||||
if(WormAuditTables.ALL.contains(table) && QContext.getQInstance().getBackendForTable(table)
|
||||
instanceof com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData)
|
||||
{
|
||||
// RC.5 binds Instant as a UTC LocalDateTime, which shifts timestamptz
|
||||
// in non-UTC sessions. Its Timestamp path preserves the actual instant.
|
||||
rows.forEach(row -> row.getValues().replaceAll((field, value) -> value instanceof Instant instant ? Timestamp.from(instant) : value));
|
||||
}
|
||||
InsertInput input = new InsertInput(table).withRecords(rows).withTransaction(transaction).withOmitDmlAudit(true);
|
||||
input.withPermissionBypass("WORM audit internal append");
|
||||
List<QRecord> inserted = new InsertAction().execute(input).getRecords();
|
||||
for(QRecord row : inserted)
|
||||
{
|
||||
if(row.getErrors() != null && !row.getErrors().isEmpty())
|
||||
{
|
||||
// Never echo a rejected record value into logs or alerts.
|
||||
throw (new QException("WORM audit append failed for " + table));
|
||||
}
|
||||
}
|
||||
return (inserted);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw (AppendOnlyGuard.map(e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Parse explicit coverage; duplicates are retained so verification sees them.
|
||||
*******************************************************************************/
|
||||
public static List<Long> detailIds(QRecord link)
|
||||
{
|
||||
String ids = link.getValueString("detail_ids");
|
||||
return (ids == null || ids.isEmpty() ? List.of() : java.util.Arrays.stream(ids.split(",")).map(Long::valueOf).toList());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve a lookup name without touching application tables.
|
||||
*******************************************************************************/
|
||||
public String name(String table, Long id) throws QException
|
||||
{
|
||||
if(id == null)
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
String key = table + ":" + id;
|
||||
if(names.containsKey(key))
|
||||
{
|
||||
return (names.get(key));
|
||||
}
|
||||
QRecord row = get(table, id);
|
||||
if(row == null)
|
||||
{
|
||||
throw (new QException("Missing " + table + " " + id));
|
||||
}
|
||||
if(names.size() >= 2000)
|
||||
{
|
||||
names.remove(names.keySet().iterator().next());
|
||||
}
|
||||
names.put(key, row.getValueString("name"));
|
||||
return (row.getValueString("name"));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Canonical bytes for the exact detail ids recorded on a link.
|
||||
*******************************************************************************/
|
||||
public byte[] canonical(QRecord audit, List<Long> ids) throws QException
|
||||
{
|
||||
return (canonical(audit, ids, index(in("auditDetail", "id", ids))));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Settled unchained headers. RDBMS uses an anti-join under the writer lock;
|
||||
** memory uses QQQ queries and the same ordering and cutoff semantics.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> unchained(Instant cutoff, int limit) throws QException
|
||||
{
|
||||
if(isRdbms())
|
||||
{
|
||||
String sql = "SELECT a." + column("audit", "id") + " FROM " + physical("audit") + " a WHERE a." + column("audit", "timestamp")
|
||||
+ " < ? AND NOT EXISTS (SELECT 1 FROM worm_audit_chain c WHERE c.audit_id=a." + column("audit", "id")
|
||||
+ " AND c.kind='AUDIT') ORDER BY a." + column("audit", "timestamp") + ", a." + column("audit", "id") + " LIMIT " + limit;
|
||||
Map<Long, QRecord> rows = index(in("audit", "id", sqlIds(sql, cutoff)));
|
||||
return (rows.values().stream().sorted(Comparator.comparing((QRecord r) -> r.getValueInstant("timestamp"))
|
||||
.thenComparing(r -> r.getValueLong("id"))).toList());
|
||||
}
|
||||
var covered = all(WormAuditTables.CHAIN).stream().filter(r -> "AUDIT".equals(r.getValueString("kind"))).map(r -> r.getValueLong("audit_id")).collect(java.util.stream.Collectors.toSet());
|
||||
return (all("audit").stream().filter(r -> r.getValueInstant("timestamp").isBefore(cutoff) && !covered.contains(r.getValueLong("id")))
|
||||
.sorted(Comparator.comparing((QRecord r) -> r.getValueInstant("timestamp")).thenComparing(r -> r.getValueLong("id"))).limit(limit).toList());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Latest successful retention anchor, or genesis.
|
||||
*******************************************************************************/
|
||||
public QRecord anchor() throws QException
|
||||
{
|
||||
List<QRecord> rows = query(WormAuditTables.PURGE, new QQueryFilter(new QFilterCriteria("outcome", QCriteriaOperator.EQUALS, "PURGED"))
|
||||
.withOrderBy(new QFilterOrderBy("id", false)).withLimit(1));
|
||||
return (rows.isEmpty() ? new QRecord().withValue("anchor_link_id", 0L).withValue("anchor_hash", Hashes.GENESIS) : rows.getFirst());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether SQL anti-joins are available in this snapshot.
|
||||
*******************************************************************************/
|
||||
public boolean isRdbms()
|
||||
{
|
||||
return (transaction instanceof RDBMSTransaction);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fetch one keyset page, never using an offset or a whole-chain read.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> page(String table, long after, int limit) throws QException
|
||||
{
|
||||
return (query(table, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, after))
|
||||
.withOrderBy(new QFilterOrderBy("id", true)).withLimit(limit)));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fetch just the tail row.
|
||||
*******************************************************************************/
|
||||
public QRecord last(String table) throws QException
|
||||
{
|
||||
List<QRecord> rows = query(table, new QQueryFilter().withOrderBy(new QFilterOrderBy("id", false)).withLimit(1));
|
||||
return (rows.isEmpty() ? null : rows.getFirst());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One batched IN query, with memory-backend integer key normalization.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> in(String table, String field, Collection<Long> ids) throws QException
|
||||
{
|
||||
if(ids.isEmpty())
|
||||
{
|
||||
return (List.of());
|
||||
}
|
||||
boolean integer = QContext.getQInstance().getTable(table).getField(field).getType()
|
||||
== com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType.INTEGER;
|
||||
List<Serializable> values = ids.stream().distinct().map(id -> integer ? (Serializable) Math.toIntExact(id) : id).toList();
|
||||
return (query(table, new QQueryFilter(new QFilterCriteria(field, QCriteriaOperator.IN, values)).withOrderBy(new QFilterOrderBy("id", true))));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Index one bounded batch by primary key.
|
||||
*******************************************************************************/
|
||||
public static Map<Long, QRecord> index(List<QRecord> rows)
|
||||
{
|
||||
return (rows.stream().collect(Collectors.toMap(r -> r.getValueLong("id"), r -> r)));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Canonicalize already fetched content; a missing detail never triggers a query.
|
||||
*******************************************************************************/
|
||||
public byte[] canonical(QRecord audit, List<Long> ids, Map<Long, QRecord> details) throws QException
|
||||
{
|
||||
List<QRecord> rows = new ArrayList<>();
|
||||
for(long id : ids)
|
||||
{
|
||||
QRecord detail = details.get(id);
|
||||
if(detail == null || !audit.getValueLong("id").equals(detail.getValueLong("auditId")))
|
||||
{
|
||||
throw (new QException("audit_detail " + id + " is missing or belongs to another audit"));
|
||||
}
|
||||
rows.add(detail);
|
||||
}
|
||||
return (AuditCanonical.bytes(audit, name("auditTable", audit.getValueLong("auditTableId")), name("auditUser", audit.getValueLong("auditUserId")), rows));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Poll uncovered details in a bounded time window; memory retains its simple fallback.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> lateDetails(Instant cutoff, int hours, int limit) throws QException
|
||||
{
|
||||
if(isRdbms())
|
||||
{
|
||||
String sql = "SELECT d." + column("auditDetail", "id") + " FROM " + physical("auditDetail") + " d JOIN " + physical("audit")
|
||||
+ " a ON a." + column("audit", "id") + "=d." + column("auditDetail", "auditId")
|
||||
+ " WHERE a." + column("audit", "timestamp") + " < ? AND a." + column("audit", "timestamp") + " >= ?"
|
||||
+ " AND EXISTS (SELECT 1 FROM worm_audit_chain h WHERE h.kind='AUDIT' AND h.audit_id=a." + column("audit", "id") + ")"
|
||||
+ " AND NOT EXISTS (SELECT 1 FROM worm_audit_chain c WHERE " + containsDetail() + ") ORDER BY d." + column("auditDetail", "id") + " LIMIT " + limit;
|
||||
return (in("auditDetail", "id", sqlIds(sql, cutoff, cutoff.minusSeconds(hours * 3600L))));
|
||||
}
|
||||
List<QRecord> links = all(WormAuditTables.CHAIN);
|
||||
Set<Long> headers = links.stream().filter(r -> "AUDIT".equals(r.getValueString("kind"))).map(r -> r.getValueLong("audit_id")).collect(Collectors.toSet());
|
||||
Set<Long> covered = new HashSet<>();
|
||||
links.forEach(link -> covered.addAll(detailIds(link)));
|
||||
Set<Long> settled = all("audit").stream().filter(a -> headers.contains(a.getValueLong("id")) && a.getValueInstant("timestamp").isBefore(cutoff))
|
||||
.map(a -> a.getValueLong("id")).collect(Collectors.toSet());
|
||||
return (all("auditDetail").stream().filter(d -> settled.contains(d.getValueLong("auditId")) && !covered.contains(d.getValueLong("id"))).toList());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** SQL completeness counts all offenders but returns only the first 100 of each type.
|
||||
*******************************************************************************/
|
||||
public void completeness(Instant cutoff, VerificationResult result) throws QException
|
||||
{
|
||||
String auditId = "a." + column("audit", "id");
|
||||
String time = "a." + column("audit", "timestamp");
|
||||
String headers = "SELECT " + auditId + " AS id, (SELECT count(*) FROM worm_audit_chain c WHERE c.kind='AUDIT' AND c.audit_id=" + auditId
|
||||
+ ") AS coverage FROM " + physical("audit") + " a WHERE " + time + " < ? OR " + time + " IS NULL";
|
||||
String details = "SELECT d." + column("auditDetail", "id") + " AS id, (SELECT count(*) FROM worm_audit_chain c WHERE " + containsDetail()
|
||||
+ ") AS coverage FROM " + physical("auditDetail") + " d LEFT JOIN " + physical("audit") + " a ON " + auditId + "=d."
|
||||
+ column("auditDetail", "auditId") + " WHERE " + time + " < ? OR " + time + " IS NULL";
|
||||
reportCoverage(headers, "audit", cutoff, result);
|
||||
reportCoverage(details, "audit_detail", cutoff, result);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Keep exact offender ids and total counts without materializing the result set.
|
||||
*******************************************************************************/
|
||||
private void reportCoverage(String source, String table, Instant cutoff, VerificationResult result) throws QException
|
||||
{
|
||||
String sql = "SELECT id, coverage, count(*) OVER() AS total FROM (" + source + ") offenders WHERE coverage <> 1 ORDER BY id LIMIT 100";
|
||||
try(PreparedStatement statement = ((RDBMSTransaction) transaction).getConnection().prepareStatement(sql))
|
||||
{
|
||||
bindCutoff(statement, 1, cutoff);
|
||||
long total = 0;
|
||||
int reported = 0;
|
||||
try(ResultSet rows = statement.executeQuery())
|
||||
{
|
||||
while(rows.next())
|
||||
{
|
||||
long id = rows.getLong(1);
|
||||
total = rows.getLong(3);
|
||||
result.add(null, table.equals("audit") ? id : null, table + " " + com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier.number(id)
|
||||
+ (rows.getLong(2) == 0 ? " is not covered by the chain" : " is covered twice or more by the chain"));
|
||||
reported++;
|
||||
}
|
||||
}
|
||||
result.omitted(total - reported);
|
||||
if(total > reported)
|
||||
{
|
||||
result.warn(table + ": " + total + " coverage offenders; first 100 ids reported");
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw (AppendOnlyGuard.map(e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** PostgreSQL's containment expression matches the GIN expression index.
|
||||
*******************************************************************************/
|
||||
private String containsDetail() throws QException
|
||||
{
|
||||
try
|
||||
{
|
||||
String id = "d." + column("auditDetail", "id");
|
||||
if("PostgreSQL".equals(((RDBMSTransaction) transaction).getConnection().getMetaData().getDatabaseProductName()))
|
||||
{
|
||||
return ("string_to_array(c.detail_ids, ',') @> ARRAY[" + id + "::text]");
|
||||
}
|
||||
return ("POSITION(',' || CAST(" + id + " AS varchar) || ',' IN ',' || c.detail_ids || ',') > 0");
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw (AppendOnlyGuard.map(e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read ids from an anti-join, binding all instants according to column type.
|
||||
*******************************************************************************/
|
||||
private List<Long> sqlIds(String sql, Instant... cutoffs) throws QException
|
||||
{
|
||||
try(PreparedStatement statement = ((RDBMSTransaction) transaction).getConnection().prepareStatement(sql))
|
||||
{
|
||||
for(int i = 0; i < cutoffs.length; i++)
|
||||
{
|
||||
bindCutoff(statement, i + 1, cutoffs[i]);
|
||||
}
|
||||
List<Long> ids = new ArrayList<>();
|
||||
try(ResultSet rows = statement.executeQuery())
|
||||
{
|
||||
while(rows.next())
|
||||
{
|
||||
ids.add(rows.getLong(1));
|
||||
}
|
||||
}
|
||||
return (ids);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw (AppendOnlyGuard.map(e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** UTC local timestamps for without-tz columns; offset timestamps for timestamptz.
|
||||
*******************************************************************************/
|
||||
private void bindCutoff(PreparedStatement statement, int index, Instant cutoff) throws Exception
|
||||
{
|
||||
if(timestampWithZone == null)
|
||||
{
|
||||
try(var probe = ((RDBMSTransaction) transaction).getConnection().createStatement();
|
||||
var rows = probe.executeQuery("SELECT " + column("audit", "timestamp") + " FROM " + physical("audit") + " WHERE 1=0"))
|
||||
{
|
||||
var metadata = rows.getMetaData();
|
||||
timestampWithZone = metadata.getColumnType(1) == Types.TIMESTAMP_WITH_TIMEZONE
|
||||
|| "timestamptz".equalsIgnoreCase(metadata.getColumnTypeName(1));
|
||||
}
|
||||
}
|
||||
statement.setObject(index, timestampWithZone ? cutoff.atOffset(ZoneOffset.UTC) : LocalDateTime.ofInstant(cutoff, ZoneOffset.UTC));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve and quote physical core table names.
|
||||
*******************************************************************************/
|
||||
private String physical(String table)
|
||||
{
|
||||
return (WormAuditDdl.identifier(((RDBMSTableBackendDetails) QContext.getQInstance().getTable(table).getBackendDetails()).getTableName()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve and quote physical field names for runtime queries.
|
||||
*******************************************************************************/
|
||||
private String column(String table, String field)
|
||||
{
|
||||
String name = QContext.getQInstance().getTable(table).getField(field).getBackendName();
|
||||
return (WormAuditDdl.identifier(name == null ? field : name));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.chain;
|
||||
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HexFormat;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** SHA-256 primitives. Link predecessors are the 64 lowercase ASCII hex bytes;
|
||||
** Merkle leaves are the decoded 32-byte link hashes, with odd nodes promoted.
|
||||
*******************************************************************************/
|
||||
public final class Hashes
|
||||
{
|
||||
public static final String GENESIS = "0".repeat(64);
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private Hashes()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** SHA-256 digest of concatenated byte arrays.
|
||||
*******************************************************************************/
|
||||
public static byte[] digest(byte[]... parts)
|
||||
{
|
||||
try
|
||||
{
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
for(byte[] part : parts)
|
||||
{
|
||||
digest.update(part);
|
||||
}
|
||||
return (digest.digest());
|
||||
}
|
||||
catch(NoSuchAlgorithmException e)
|
||||
{
|
||||
throw (new IllegalStateException("JDK lacks SHA-256", e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Link hash over predecessor, NUL delimiter and canonical bytes.
|
||||
*******************************************************************************/
|
||||
public static String link(String previous, byte[] canonical)
|
||||
{
|
||||
return (HexFormat.of().formatHex(digest(previous.getBytes(StandardCharsets.US_ASCII), new byte[] { 0 }, canonical)));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Binary Merkle root; an empty range cannot be checkpointed.
|
||||
*******************************************************************************/
|
||||
public static String merkle(List<String> hashes)
|
||||
{
|
||||
if(hashes.isEmpty())
|
||||
{
|
||||
throw (new IllegalArgumentException("A Merkle range must contain links"));
|
||||
}
|
||||
List<byte[]> level = hashes.stream().map(HexFormat.of()::parseHex).toList();
|
||||
while(level.size() > 1)
|
||||
{
|
||||
List<byte[]> next = new ArrayList<>();
|
||||
for(int i = 0; i < level.size(); i += 2)
|
||||
{
|
||||
next.add(i + 1 == level.size() ? level.get(i) : digest(level.get(i), level.get(i + 1)));
|
||||
}
|
||||
level = next;
|
||||
}
|
||||
return (HexFormat.of().formatHex(level.getFirst()));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.chain;
|
||||
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.Statement;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import java.util.stream.Collectors;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.checkpoint.Checkpoints;
|
||||
import com.kingsrook.qbits.wormaudit.guard.AppendOnlyGuard;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSTransaction;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Single asynchronous writer. PostgreSQL uses a transaction-scoped advisory
|
||||
** lock shared with the retention function. Other backends use a JVM-wide lock:
|
||||
** those deployments must run their chainer in one JVM only.
|
||||
*******************************************************************************/
|
||||
public class WormAuditChainer
|
||||
{
|
||||
public static final long ADVISORY_LOCK = 86753090421L;
|
||||
public static final ReentrantLock JVM_LOCK = new ReentrantLock();
|
||||
private final WormAuditQBitConfig config;
|
||||
|
||||
/*******************************************************************************
|
||||
** Counts describe committed work; a contended writer tick is skipped.
|
||||
*******************************************************************************/
|
||||
public record Counts(int audits, int lateDetails, int links, int checkpoints, boolean skipped)
|
||||
{
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Bind host configuration.
|
||||
*******************************************************************************/
|
||||
public WormAuditChainer(WormAuditQBitConfig config)
|
||||
{
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Run a writer tick under one transaction; link ids cannot gap on rollback.
|
||||
*******************************************************************************/
|
||||
public Counts chainOnce() throws QException
|
||||
{
|
||||
return (chainOnce(Instant.now()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Clock-injected tick for deterministic settling and checkpoint tests.
|
||||
*******************************************************************************/
|
||||
public Counts chainOnce(Instant now) throws QException
|
||||
{
|
||||
config.validate();
|
||||
JVM_LOCK.lock();
|
||||
try(QBackendTransaction transaction = QBackendTransaction.openFor(new QueryInput(WormAuditTables.CHAIN)))
|
||||
{
|
||||
try
|
||||
{
|
||||
if(transaction instanceof RDBMSTransaction rdbms && "PostgreSQL".equals(rdbms.getConnection().getMetaData().getDatabaseProductName()))
|
||||
{
|
||||
try(Statement statement = rdbms.getConnection().createStatement(); ResultSet result = statement.executeQuery("SELECT pg_try_advisory_xact_lock(" + ADVISORY_LOCK + ")"))
|
||||
{
|
||||
result.next();
|
||||
if(!result.getBoolean(1))
|
||||
{
|
||||
return (new Counts(0, 0, 0, 0, true));
|
||||
}
|
||||
}
|
||||
}
|
||||
Counts counts = append(new AuditStore(transaction), now);
|
||||
transaction.commit();
|
||||
return (counts);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
transaction.rollback();
|
||||
throw (AppendOnlyGuard.map(e));
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
JVM_LOCK.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Append settled headers and uncovered detail chunks, then checkpoint.
|
||||
*******************************************************************************/
|
||||
private Counts append(AuditStore store, Instant now) throws Exception
|
||||
{
|
||||
QRecord tail = store.last(WormAuditTables.CHAIN);
|
||||
QRecord anchor = store.anchor();
|
||||
long nextId = tail == null ? anchor.getValueLong("anchor_link_id") + 1 : tail.getValueLong("id") + 1;
|
||||
String previous = tail == null ? anchor.getValueString("anchor_hash") : tail.getValueString("link_hash");
|
||||
List<QRecord> appended = new ArrayList<>();
|
||||
Instant cutoff = now.minusSeconds(config.getSettleHorizonSeconds());
|
||||
List<QRecord> audits = store.unchained(cutoff, config.getBatchSize());
|
||||
Map<Long, QRecord> details = AuditStore.index(store.in("auditDetail", "auditId", audits.stream().map(r -> r.getValueLong("id")).toList()));
|
||||
Map<Long, List<QRecord>> grouped = details.values().stream().collect(Collectors.groupingBy(r -> r.getValueLong("auditId")));
|
||||
int lateDetails = 0;
|
||||
for(QRecord audit : audits)
|
||||
{
|
||||
List<Long> ids = grouped.getOrDefault(audit.getValueLong("id"), List.of()).stream().map(r -> r.getValueLong("id")).sorted().toList();
|
||||
for(int start = 0; start == 0 || start < ids.size(); start += 250)
|
||||
{
|
||||
List<Long> chunk = ids.subList(start, Math.min(start + 250, ids.size()));
|
||||
QRecord link = link(store, nextId++, start == 0 ? "AUDIT" : "LATE_DETAIL", audit, chunk, details, previous, now);
|
||||
appended.add(link);
|
||||
previous = link.getValueString("link_hash");
|
||||
if(start > 0)
|
||||
{
|
||||
lateDetails += chunk.size();
|
||||
}
|
||||
}
|
||||
}
|
||||
insertLinks(store, appended);
|
||||
int linkCount = appended.size();
|
||||
appended.clear();
|
||||
details = AuditStore.index(store.lateDetails(cutoff, config.getLateDetailLookbackHours(), config.getBatchSize()));
|
||||
grouped = details.values().stream().collect(Collectors.groupingBy(r -> r.getValueLong("auditId"), TreeMap::new, Collectors.toList()));
|
||||
Map<Long, QRecord> headers = AuditStore.index(store.in("audit", "id", grouped.keySet()));
|
||||
for(var entry : grouped.entrySet())
|
||||
{
|
||||
List<Long> ids = entry.getValue().stream().map(r -> r.getValueLong("id")).sorted().toList();
|
||||
for(int start = 0; start < ids.size(); start += 250)
|
||||
{
|
||||
List<Long> chunk = ids.subList(start, Math.min(start + 250, ids.size()));
|
||||
QRecord link = link(store, nextId++, "LATE_DETAIL", headers.get(entry.getKey()), chunk, details, previous, now);
|
||||
appended.add(link);
|
||||
previous = link.getValueString("link_hash");
|
||||
lateDetails += chunk.size();
|
||||
}
|
||||
}
|
||||
insertLinks(store, appended);
|
||||
return (new Counts(audits.size(), lateDetails, linkCount + appended.size(), checkpoint(store, now), false));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Persist bounded insert batches inside the writer transaction.
|
||||
*******************************************************************************/
|
||||
private void insertLinks(AuditStore store, List<QRecord> links) throws QException
|
||||
{
|
||||
for(int start = 0; start < links.size(); start += config.getBatchSize())
|
||||
{
|
||||
store.insert(WormAuditTables.CHAIN, links.subList(start, Math.min(start + config.getBatchSize(), links.size())));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build one immutable link from live stored content.
|
||||
*******************************************************************************/
|
||||
private QRecord link(AuditStore store, long id, String kind, QRecord audit, List<Long> details, Map<Long, QRecord> rows, String previous, Instant now) throws QException
|
||||
{
|
||||
return (new QRecord().withValue("id", id).withValue("kind", kind).withValue("audit_id", audit.getValueLong("id"))
|
||||
.withValue("detail_ids", details.stream().map(Object::toString).collect(Collectors.joining(",")))
|
||||
.withValue("prev_hash", previous).withValue("link_hash", Hashes.link(previous, store.canonical(audit, details, rows))).withValue("chained_at", now));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Checkpoint the uncheckpointed suffix by count or elapsed time.
|
||||
*******************************************************************************/
|
||||
private int checkpoint(AuditStore store, Instant now) throws Exception
|
||||
{
|
||||
if(config.getSigningKey() == null)
|
||||
{
|
||||
return (0);
|
||||
}
|
||||
QRecord latest = store.last(WormAuditTables.CHECKPOINT);
|
||||
long lastId = latest == null ? store.anchor().getValueLong("anchor_link_id") : latest.getValueLong("to_link_id");
|
||||
int count = 0;
|
||||
while(true)
|
||||
{
|
||||
List<QRecord> pending = store.page(WormAuditTables.CHAIN, lastId, config.getCheckpointEveryLinks());
|
||||
if(pending.isEmpty())
|
||||
{
|
||||
return (count);
|
||||
}
|
||||
Instant since = latest == null ? pending.getFirst().getValueInstant("chained_at") : latest.getValueInstant("created_at");
|
||||
if(pending.size() < config.getCheckpointEveryLinks() && now.isBefore(since.plusSeconds(config.getCheckpointEveryMinutes() * 60L)))
|
||||
{
|
||||
return (count);
|
||||
}
|
||||
latest = store.insert(WormAuditTables.CHECKPOINT, List.of(Checkpoints.sign(pending, config, now))).getFirst();
|
||||
lastId = latest.getValueLong("to_link_id");
|
||||
count++;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.checkpoint;
|
||||
|
||||
|
||||
import java.security.GeneralSecurityException;
|
||||
import java.security.PublicKey;
|
||||
import java.security.Signature;
|
||||
import java.time.Instant;
|
||||
import java.util.Base64;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.canonical.Jcs;
|
||||
import com.kingsrook.qbits.wormaudit.chain.Hashes;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Ed25519 checkpoint envelopes. Trust comes exclusively from host-pinned keys,
|
||||
** never the public_key supplied by an untrusted checkpoint row.
|
||||
*******************************************************************************/
|
||||
public final class Checkpoints
|
||||
{
|
||||
private static final List<String> SIGNED_FIELDS = List.of("from_link_id", "to_link_id", "link_count", "head_hash", "merkle_root", "algorithm", "key_id");
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private Checkpoints()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The exact signed payload; public_key and signature are envelope fields.
|
||||
*******************************************************************************/
|
||||
public static byte[] payload(QRecord checkpoint)
|
||||
{
|
||||
Map<String, Object> fields = new LinkedHashMap<>();
|
||||
for(String field : SIGNED_FIELDS)
|
||||
{
|
||||
fields.put(field, field.endsWith("_id") && !field.equals("key_id") || field.equals("link_count")
|
||||
? checkpoint.getValueLong(field) : checkpoint.getValueString(field));
|
||||
}
|
||||
return (Jcs.bytes(fields));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Create a checkpoint only when the configured private key matches its pin.
|
||||
*******************************************************************************/
|
||||
public static QRecord sign(List<QRecord> links, WormAuditQBitConfig config, Instant now) throws GeneralSecurityException
|
||||
{
|
||||
WormAuditQBitConfig.SigningKey key = config.getSigningKey().get();
|
||||
PublicKey publicKey = config.getTrustedPublicKeys().get(key.keyId());
|
||||
if(publicKey == null)
|
||||
{
|
||||
throw (new GeneralSecurityException("Signing key id is not pinned"));
|
||||
}
|
||||
QRecord record = new QRecord().withValue("from_link_id", links.getFirst().getValueLong("id"))
|
||||
.withValue("to_link_id", links.getLast().getValueLong("id")).withValue("link_count", (long) links.size())
|
||||
.withValue("head_hash", links.getLast().getValueString("link_hash"))
|
||||
.withValue("merkle_root", Hashes.merkle(links.stream().map(r -> r.getValueString("link_hash")).toList()))
|
||||
.withValue("algorithm", "Ed25519").withValue("key_id", key.keyId())
|
||||
.withValue("public_key", Base64.getEncoder().encodeToString(publicKey.getEncoded())).withValue("created_at", now);
|
||||
Signature signature = Signature.getInstance("Ed25519");
|
||||
signature.initSign(key.privateKey());
|
||||
signature.update(payload(record));
|
||||
record.setValue("signature", Base64.getEncoder().encodeToString(signature.sign()));
|
||||
if(!verify(record, publicKey))
|
||||
{
|
||||
throw (new GeneralSecurityException("Signing private key does not match its pin"));
|
||||
}
|
||||
return (record);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify with a pinned key, rejecting substituted algorithms and public keys.
|
||||
*******************************************************************************/
|
||||
public static boolean verify(QRecord record, PublicKey pin)
|
||||
{
|
||||
try
|
||||
{
|
||||
if(pin == null || !"Ed25519".equals(record.getValueString("algorithm"))
|
||||
|| !Base64.getEncoder().encodeToString(pin.getEncoded()).equals(record.getValueString("public_key")))
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
Signature signature = Signature.getInstance("Ed25519");
|
||||
signature.initVerify(pin);
|
||||
signature.update(payload(record));
|
||||
return (signature.verify(Base64.getDecoder().decode(record.getValueString("signature"))));
|
||||
}
|
||||
catch(GeneralSecurityException | IllegalArgumentException e)
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.ddl;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Render a reviewed migration for the host's physical audit table names.
|
||||
** Core physical columns must use snake_case: audit.id/timestamp,
|
||||
** audit_detail.id/audit_id, audit_table.id/name/label/create_date/modify_date,
|
||||
** and audit_user.id/name/create_date/modify_date. Only table names are rendered;
|
||||
** QInstance field backend-name overrides do not change this DDL contract.
|
||||
** Identifiers are restricted and quoted; configuration is never executable SQL.
|
||||
*******************************************************************************/
|
||||
public final class WormAuditDdl
|
||||
{
|
||||
public static final Map<String, String> DEFAULT_NAMES = Map.of("audit", "audit", "auditDetail", "audit_detail", "auditTable", "audit_table", "auditUser", "audit_user");
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private WormAuditDdl()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A safe SQL identifier, optionally schema qualified.
|
||||
*******************************************************************************/
|
||||
public static String identifier(String value)
|
||||
{
|
||||
if(value == null || !value.matches("[A-Za-z_][A-Za-z0-9_]*(\\.[A-Za-z_][A-Za-z0-9_]*)?"))
|
||||
{
|
||||
throw (new IllegalArgumentException("Invalid SQL identifier"));
|
||||
}
|
||||
return ("\"" + value.replace(".", "\".\"") + "\"");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve backend names while preserving explicit DDL overrides.
|
||||
*******************************************************************************/
|
||||
public static void resolveNames(WormAuditQBitConfig config, QInstance instance)
|
||||
{
|
||||
Map<String, String> names = new LinkedHashMap<>();
|
||||
for(String name : DEFAULT_NAMES.keySet())
|
||||
{
|
||||
QTableMetaData table = instance.getTable(name);
|
||||
if(table == null)
|
||||
{
|
||||
throw (new IllegalArgumentException("Core audit table is required: " + name));
|
||||
}
|
||||
names.put(name, table.getBackendDetails() instanceof RDBMSTableBackendDetails details ? details.getTableName() : DEFAULT_NAMES.get(name));
|
||||
}
|
||||
names.putAll(config.getCoreAuditTableNames());
|
||||
config.withCoreAuditTableNames(Map.copyOf(names));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Render the template. Retention is compiled into the function, never a GUC.
|
||||
*******************************************************************************/
|
||||
public static String postgres(WormAuditQBitConfig config)
|
||||
{
|
||||
if(config.getRetentionYears() < 1 || config.getAppRole().contains("."))
|
||||
{
|
||||
throw (new IllegalArgumentException("Invalid retention or app role"));
|
||||
}
|
||||
try(var stream = WormAuditDdl.class.getResourceAsStream("/com/kingsrook/qbits/wormaudit/ddl/postgres/worm-audit.sql"))
|
||||
{
|
||||
String sql = new String(java.util.Objects.requireNonNull(stream).readAllBytes(), StandardCharsets.UTF_8);
|
||||
Map<String, String> names = new LinkedHashMap<>(DEFAULT_NAMES);
|
||||
names.putAll(config.getCoreAuditTableNames());
|
||||
for(var entry : DEFAULT_NAMES.entrySet())
|
||||
{
|
||||
sql = sql.replace("${" + entry.getValue() + "}", identifier(names.get(entry.getKey())));
|
||||
}
|
||||
return (sql.replace("${app_role}", identifier(config.getAppRole())).replace("${retention}", config.getRetentionYears() + " years"));
|
||||
}
|
||||
catch(IOException e)
|
||||
{
|
||||
throw (new IllegalStateException("Cannot read WORM audit DDL", e));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.guard;
|
||||
|
||||
|
||||
import java.sql.SQLException;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Metadata defense and a value-free mapping of the database write-once error.
|
||||
*******************************************************************************/
|
||||
public class AppendOnlyGuard implements MetaDataCustomizerInterface<QTableMetaData>
|
||||
{
|
||||
public static final List<String> CORE_TABLES = List.of("audit", "auditDetail", "auditTable", "auditUser");
|
||||
public static final String MESSAGE = "Audit records are write-once and cannot be changed or deleted.";
|
||||
|
||||
/*******************************************************************************
|
||||
** Protect core and sidecar audit metadata after host customization.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QTableMetaData customizeMetaData(QInstance instance, QTableMetaData table)
|
||||
{
|
||||
if(CORE_TABLES.contains(table.getName()) || table.getName().startsWith("worm_audit_"))
|
||||
{
|
||||
table.withoutCapabilities(Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
|
||||
}
|
||||
if(table.getName().startsWith("worm_audit_"))
|
||||
{
|
||||
table.withoutCapabilities(Capability.TABLE_INSERT);
|
||||
}
|
||||
return (table);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Preserve unrelated failures, map only the documented SQLSTATE and prefix.
|
||||
*******************************************************************************/
|
||||
public static QException map(Exception error)
|
||||
{
|
||||
for(Throwable cause = error; cause != null; cause = cause.getCause())
|
||||
{
|
||||
if(cause instanceof SQLException sql && "P0001".equals(sql.getSQLState())
|
||||
&& sql.getMessage() != null && sql.getMessage().startsWith("worm_audit:"))
|
||||
{
|
||||
return (new QUserFacingException(MESSAGE));
|
||||
}
|
||||
// The PostgreSQL driver prepends ERROR: to server error messages.
|
||||
if(cause instanceof SQLException sql && "P0001".equals(sql.getSQLState())
|
||||
&& sql.getMessage() != null && sql.getMessage().startsWith("ERROR: worm_audit:"))
|
||||
{
|
||||
return (new QUserFacingException(MESSAGE));
|
||||
}
|
||||
}
|
||||
return (error instanceof QException q ? q : new QException("WORM audit operation failed", error));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.model;
|
||||
|
||||
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChildRecordListData;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A standard child-record list for a checkpoint/export/verification link range.
|
||||
** This is a record-screen relation, not an app-home widget. Reads retain user
|
||||
** permissions and use a bounded page with a filtered View All link.
|
||||
*******************************************************************************/
|
||||
public class LinkRangeWidget extends AbstractWidgetRenderer
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Render the inclusive range stored on the parent record.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public RenderWidgetOutput render(RenderWidgetInput input) throws QException
|
||||
{
|
||||
String parent = (String) input.getWidgetMetaData().getDefaultValues().get("parentTable");
|
||||
GetInput get = new GetInput().withPrimaryKey(input.getQueryParams().get("id")).withInputSource(QInputSource.USER);
|
||||
get.setTableName(parent);
|
||||
get.withPermissionEnforcement(PermissionEnforcement.ENFORCE);
|
||||
QRecord record = new GetAction().execute(get).getRecord();
|
||||
if(record == null)
|
||||
{
|
||||
throw (new QNotFoundException("Audit range record was not found"));
|
||||
}
|
||||
Long from = record.getValueLong("from_link_id");
|
||||
Long to = record.getValueLong("to_link_id");
|
||||
QQueryFilter filter = new QQueryFilter()
|
||||
.withCriteria(new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, from == null ? 1L : from))
|
||||
.withCriteria(new QFilterCriteria("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, to == null ? 0L : to))
|
||||
.withOrderBy(new QFilterOrderBy("id", true));
|
||||
CountInput count = new CountInput(WormAuditTables.CHAIN).withFilter(filter).withInputSource(QInputSource.USER);
|
||||
count.withPermissionEnforcement(PermissionEnforcement.ENFORCE);
|
||||
int total = new CountAction().execute(count).getCount();
|
||||
String path = QContext.getQInstance().getTablePath(WormAuditTables.CHAIN);
|
||||
String viewAll = path + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), StandardCharsets.UTF_8);
|
||||
QueryInput query = new QueryInput(WormAuditTables.CHAIN).withFilter(filter.clone().withLimit(100)).withInputSource(QInputSource.USER);
|
||||
query.withPermissionEnforcement(PermissionEnforcement.ENFORCE);
|
||||
return (new RenderWidgetOutput(new ChildRecordListData("Chain links", new QueryAction().execute(query),
|
||||
QContext.getQInstance().getTable(WormAuditTables.CHAIN), path, viewAll, total)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.model;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Append-only sidecar tables. Snake-case fields map directly to DDL columns.
|
||||
** Export metadata is the integration hook for phase (b); no export runs here.
|
||||
*******************************************************************************/
|
||||
public final class WormAuditTables
|
||||
{
|
||||
public static final String CHAIN = "worm_audit_chain";
|
||||
public static final String CHECKPOINT = "worm_audit_checkpoint";
|
||||
public static final String EXPORT = "worm_audit_export";
|
||||
public static final String VERIFICATION = "worm_audit_verification";
|
||||
public static final String PURGE = "worm_audit_purge";
|
||||
public static final List<String> ALL = List.of(CHAIN, CHECKPOINT, EXPORT, VERIFICATION, PURGE);
|
||||
public static final List<String> NAV = List.of(VERIFICATION, EXPORT, CHECKPOINT, CHAIN, PURGE);
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private WormAuditTables()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Construct full record screens, with no recursive audit writes.
|
||||
*******************************************************************************/
|
||||
public static List<QTableMetaData> tables(WormAuditQBitConfig config)
|
||||
{
|
||||
List<QTableMetaData> tables = new ArrayList<>();
|
||||
tables.add(table(config, CHAIN, "Audit Chain Link", "id:L kind:S audit_id:L detail_ids:T prev_hash:S link_hash:S chained_at:D"));
|
||||
tables.add(table(config, CHECKPOINT, "Checkpoint", "id:L from_link_id:L to_link_id:L link_count:L head_hash:S merkle_root:S algorithm:S key_id:S public_key:T signature:T created_at:D"));
|
||||
tables.add(table(config, EXPORT, "Sealed Export", "id:L from_link_id:L to_link_id:L from_checkpoint_id:L to_checkpoint_id:L bucket_alias:S object_prefix:T manifest_key:T manifest_sha256:S prev_manifest_sha256:S object_count:L byte_count:L lock_mode:S retain_until:D sealed_at:D"));
|
||||
tables.add(table(config, VERIFICATION, "Verification", "id:L kind:S trigger:S started_at:D finished_at:D from_link_id:L to_link_id:L links_checked:L outcome:S break_count:L first_break_link_id:L first_break_audit_id:L first_break_reason:T export_id:L run_by:S"));
|
||||
tables.add(table(config, PURGE, "Purge", "id:L requested_at:D requested_by:S cutoff:D retention_interval:S outcome:S reason:T audits_deleted:L details_deleted:L links_deleted:L anchor_link_id:L anchor_hash:S"));
|
||||
return (tables);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Common metadata for a process-written table.
|
||||
*******************************************************************************/
|
||||
private static QTableMetaData table(WormAuditQBitConfig config, String name, String label, String fields)
|
||||
{
|
||||
QTableMetaData table = new QTableMetaData().withName(name).withLabel(label)
|
||||
.withBackendName(config.getBackendName()).withBackendDetails(new RDBMSTableBackendDetails().withTableName(name))
|
||||
.withPrimaryKeyField("id").withRecordLabelFields("id").withRecordLabelFormat("%s")
|
||||
.withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.NONE))
|
||||
.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_INSERT_EDIT_DELETE_PERMISSIONS).withPermissionBaseName(name));
|
||||
for(String spec : fields.split(" "))
|
||||
{
|
||||
String[] parts = spec.split(":");
|
||||
QFieldType type = switch(parts[1])
|
||||
{
|
||||
case "L" -> QFieldType.LONG;
|
||||
case "D" -> QFieldType.DATE_TIME;
|
||||
case "T" -> QFieldType.TEXT;
|
||||
default -> QFieldType.STRING;
|
||||
};
|
||||
table.withField(new QFieldMetaData(parts[0], type).withBackendName(parts[0]));
|
||||
}
|
||||
for(String reference : List.of("from_link_id", "to_link_id", "first_break_link_id", "anchor_link_id"))
|
||||
{
|
||||
if(table.getFields().containsKey(reference))
|
||||
{
|
||||
table.getField(reference).withPossibleValueSourceName(CHAIN);
|
||||
}
|
||||
}
|
||||
for(String reference : List.of("from_checkpoint_id", "to_checkpoint_id"))
|
||||
{
|
||||
if(table.getFields().containsKey(reference))
|
||||
{
|
||||
table.getField(reference).withPossibleValueSourceName(CHECKPOINT);
|
||||
}
|
||||
}
|
||||
if(table.getFields().containsKey("export_id"))
|
||||
{
|
||||
table.getField("export_id").withPossibleValueSourceName(EXPORT);
|
||||
}
|
||||
table.withSection(new QFieldSection("record", label, new QIcon("fact_check"), Tier.T1, new ArrayList<>(table.getFields().keySet())));
|
||||
if(List.of(CHECKPOINT, EXPORT, VERIFICATION).contains(name))
|
||||
{
|
||||
table.withSection(new QFieldSection("links", new QIcon("link"), Tier.T2).withLabel("Chain links").withWidgetName(name + "_links"));
|
||||
}
|
||||
return (table);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.processes;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.VerificationResult;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertRecordRef;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertSignal;
|
||||
import com.kingsrook.qqq.backend.core.alerts.QAlerts;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verification run persistence and the core alerts SPI. Both UI and schedule
|
||||
** write exactly one immutable result row; only schedules change alert state.
|
||||
*******************************************************************************/
|
||||
public final class VerificationRuns
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private VerificationRuns()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve this instance's configuration, never a global cross-instance cache.
|
||||
*******************************************************************************/
|
||||
public static WormAuditQBitConfig config()
|
||||
{
|
||||
return (QContext.getQInstance().getQBits().values().stream().map(q -> q.getConfig()).filter(WormAuditQBitConfig.class::isInstance)
|
||||
.map(WormAuditQBitConfig.class::cast).findFirst().orElseThrow(() -> new IllegalStateException("WORM audit qbit is not installed")));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify and persist, sending a heartbeat even if a scheduled run errors.
|
||||
*******************************************************************************/
|
||||
public static VerificationResult run(Set<Long> selection, String trigger) throws QException
|
||||
{
|
||||
boolean scheduled = "SCHEDULE".equals(trigger);
|
||||
try
|
||||
{
|
||||
VerificationResult result = new WormAuditVerifier(config()).verify(selection);
|
||||
QRecord row = save(result, trigger);
|
||||
if(scheduled)
|
||||
{
|
||||
if(result.passed())
|
||||
{
|
||||
QAlerts.resolve("worm_audit.chain.break", "chain", "Audit chain verification passed");
|
||||
}
|
||||
else
|
||||
{
|
||||
QAlerts.raise(new AlertSignal("worm_audit.chain.break").withDedupKey("chain")
|
||||
.withTitle(result.getBreaks().getFirst().reason()).withRecordRef(new AlertRecordRef(WormAuditTables.VERIFICATION, row.getValue("id"))));
|
||||
}
|
||||
}
|
||||
return (result);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if(scheduled)
|
||||
{
|
||||
QAlerts.heartbeat("worm_audit.verify.daily");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Persist identifiers and a value-free first reason, never audit content.
|
||||
*******************************************************************************/
|
||||
public static QRecord save(VerificationResult result, String trigger) throws QException
|
||||
{
|
||||
String user = QContext.getQSession() == null || QContext.getQSession().getUser() == null ? "system" : QContext.getQSession().getUser().getIdReference();
|
||||
QRecord row = new QRecord().withValue("kind", "CHAIN").withValue("trigger", trigger)
|
||||
.withValue("started_at", result.getStartedAt()).withValue("finished_at", Instant.now())
|
||||
.withValue("from_link_id", result.getFromLinkId()).withValue("to_link_id", result.getToLinkId())
|
||||
.withValue("links_checked", result.getLinksChecked()).withValue("outcome", result.passed() ? "PASS" : "FAIL")
|
||||
.withValue("break_count", result.getBreakCount()).withValue("run_by", user);
|
||||
if(!result.passed())
|
||||
{
|
||||
var first = result.getBreaks().getFirst();
|
||||
row.withValue("first_break_link_id", first.linkId()).withValue("first_break_audit_id", first.auditId()).withValue("first_break_reason", first.reason());
|
||||
}
|
||||
return (new AuditStore(null).insert(WormAuditTables.VERIFICATION, List.of(row)).getFirst());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.processes;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.VerificationResult;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.UseOtherPermissionNameCustomPermissionChecker;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunInput;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunOutput;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.CouldNotFindQueryFilterForExtractStepException;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Standard streamed ETL Review/Result action plus two scheduled-only processes.
|
||||
** No process is an app navigation entry.
|
||||
*******************************************************************************/
|
||||
public final class WormAuditProcesses
|
||||
{
|
||||
public static final String VERIFY = "worm_audit_verify_chain";
|
||||
public static final String CHAIN = "worm_audit_chain_tick";
|
||||
public static final String DAILY = "worm_audit_verify_daily";
|
||||
public static final String PERMISSION = "worm_audit.verify";
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private WormAuditProcesses()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Metadata for table/record verification and scheduled background work.
|
||||
*******************************************************************************/
|
||||
public static List<QProcessMetaData> processes(WormAuditQBitConfig config)
|
||||
{
|
||||
Map<String, Serializable> values = new HashMap<>();
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, WormAuditTables.CHAIN);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, WormAuditTables.CHAIN);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION, true);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, "Verify live audit content, link continuity and signed checkpoints.");
|
||||
QProcessMetaData verify = StreamedETLWithFrontendProcess.defineProcessMetaData(Extract.class, Review.class, Load.class, values)
|
||||
.withName(VERIFY).withLabel("Verify audit chain").withTableName(WormAuditTables.CHAIN).withPermissionRules(permission());
|
||||
QProcessMetaData chain = new QProcessMetaData().withName(CHAIN).withLabel("Chain settled audit records")
|
||||
.withStep(new QBackendStepMetaData().withName("chain").withCode(new QCodeReference(ChainTick.class)))
|
||||
.withPermissionRules(permission()).withSchedule(new QScheduleMetaData().withSchedulerName(config.getSchedulerName())
|
||||
.withRepeatSeconds(config.getChainIntervalSeconds()).withInitialDelaySeconds(config.getChainIntervalSeconds()));
|
||||
QProcessMetaData daily = new QProcessMetaData().withName(DAILY).withLabel("Daily audit chain verification")
|
||||
.withStep(new QBackendStepMetaData().withName("verify").withCode(new QCodeReference(Daily.class)))
|
||||
.withPermissionRules(permission()).withSchedule(new QScheduleMetaData().withSchedulerName(config.getSchedulerName())
|
||||
.withCronExpression(config.getVerifyCron()).withCronTimeZoneId("UTC"));
|
||||
return (List.of(verify, chain, daily));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One explicit permission, not process-name access suffixes.
|
||||
*******************************************************************************/
|
||||
private static QPermissionRules permission()
|
||||
{
|
||||
return (new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION)
|
||||
.withCustomPermissionChecker(UseOtherPermissionNameCustomPermissionChecker.build(PERMISSION)));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Use core selection/filter interpretation, with a full-chain default. An
|
||||
** empty frontend filter means a full run, so completeness is still checked.
|
||||
*******************************************************************************/
|
||||
public static class Extract extends ExtractViaQueryStep
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Extract links in id order and make the effective scope available to Load.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QQueryFilter getQueryFilter(RunBackendStepInput input) throws QException
|
||||
{
|
||||
input.addValue(FIELD_SOURCE_TABLE, WormAuditTables.CHAIN);
|
||||
QQueryFilter filter;
|
||||
try
|
||||
{
|
||||
filter = super.getQueryFilter(input).clone();
|
||||
}
|
||||
catch(CouldNotFindQueryFilterForExtractStepException expected)
|
||||
{
|
||||
filter = new QQueryFilter();
|
||||
}
|
||||
filter.setOrderBys(List.of(new QFilterOrderBy("id", true)));
|
||||
return (filter);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Review counts the streamed selection without verifying or writing rows.
|
||||
*******************************************************************************/
|
||||
public static class Review extends AbstractTransformStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private long count;
|
||||
private long min = Long.MAX_VALUE;
|
||||
private long max = Long.MIN_VALUE;
|
||||
private Long checkpoints;
|
||||
|
||||
/*******************************************************************************
|
||||
** Accumulate the scope and forward links to the loader.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
for(QRecord record : input.getRecords())
|
||||
{
|
||||
count++;
|
||||
min = Math.min(min, record.getValueLong("id"));
|
||||
max = Math.max(max, record.getValueLong("id"));
|
||||
}
|
||||
checkpoints = null;
|
||||
output.getRecords().addAll(input.getRecords());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Standard Review line, including the link range and checkpoint count.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
if(checkpoints == null)
|
||||
{
|
||||
try
|
||||
{
|
||||
CountInput input = new CountInput(WormAuditTables.CHECKPOINT).withFilter(new QQueryFilter(
|
||||
new QFilterCriteria("from_link_id", QCriteriaOperator.LESS_THAN_OR_EQUALS, max),
|
||||
new QFilterCriteria("to_link_id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, min)));
|
||||
input.withPermissionBypass("WORM audit internal review");
|
||||
checkpoints = count == 0 ? 0L : new CountAction().execute(input).getCount().longValue();
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
return (new ArrayList<>(List.of(new ProcessSummaryLine(Status.ERROR, "Cannot count overlapping audit checkpoints"))));
|
||||
}
|
||||
}
|
||||
String range = count == 0 ? "none" : WormAuditVerifier.number(min) + "–" + WormAuditVerifier.number(max);
|
||||
return (new ArrayList<>(List.of(new ProcessSummaryLine(Status.INFO, WormAuditVerifier.number(count)
|
||||
+ " chain links will be verified (links " + range + ", " + WormAuditVerifier.number(checkpoints) + " checkpoints)"))));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Load collects selected ids, then performs one snapshot verification in
|
||||
** postRun and appends one result row, including for an empty chain.
|
||||
*******************************************************************************/
|
||||
public static class Load extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private Set<Long> ids;
|
||||
private VerificationResult result;
|
||||
|
||||
/*******************************************************************************
|
||||
** Retain ids only, not audit content, across streamed pages.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
if(selected(new Extract().getQueryFilter(input)))
|
||||
{
|
||||
if(ids == null)
|
||||
{
|
||||
ids = new HashSet<>();
|
||||
}
|
||||
input.getRecords().forEach(r -> ids.add(r.getValueLong("id")));
|
||||
}
|
||||
output.getRecords().addAll(input.getRecords());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Run once after all pages. Explicit filters are selection runs.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void postRun(BackendStepPostRunInput input, BackendStepPostRunOutput output) throws QException
|
||||
{
|
||||
QQueryFilter filter = new Extract().getQueryFilter(input);
|
||||
if(selected(filter) && ids == null)
|
||||
{
|
||||
// Some callers supply the effective selection only to postRun.
|
||||
ids = new HashSet<>();
|
||||
new AuditStore(null).query(WormAuditTables.CHAIN, filter).forEach(r -> ids.add(r.getValueLong("id")));
|
||||
}
|
||||
result = VerificationRuns.run(selected(filter) ? ids : null, "USER");
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Empty filters preserve full completeness verification and retain no ids.
|
||||
*******************************************************************************/
|
||||
private boolean selected(QQueryFilter filter)
|
||||
{
|
||||
return ((filter.getCriteria() != null && !filter.getCriteria().isEmpty())
|
||||
|| (filter.getSubFilters() != null && !filter.getSubFilters().isEmpty())
|
||||
|| filter.getLimit() != null || (filter.getSkip() != null && filter.getSkip() > 0));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One ERROR per exact break; warnings remain visible on a passing run.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
if(result == null)
|
||||
{
|
||||
return (lines);
|
||||
}
|
||||
if(result.passed())
|
||||
{
|
||||
lines.add(new ProcessSummaryLine(Status.OK, WormAuditVerifier.number(result.getLinksChecked()) + " links verified"));
|
||||
}
|
||||
result.getBreaks().forEach(b -> lines.add(new ProcessSummaryLine(Status.ERROR, b.reason())));
|
||||
result.getWarnings().forEach(w -> lines.add(new ProcessSummaryLine(Status.WARNING, w)));
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Scheduled writer tick.
|
||||
*******************************************************************************/
|
||||
public static class ChainTick implements BackendStep
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Append settled work under the shared writer lock.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
output.addValue("linksChained", new WormAuditChainer(VerificationRuns.config()).chainOnce().links());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Daily verification, including alerts and heartbeat.
|
||||
*******************************************************************************/
|
||||
public static class Daily implements BackendStep
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Verify the full settled chain.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
output.addValue("outcome", VerificationRuns.run(null, "SCHEDULE").passed() ? "PASS" : "FAIL");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.verify;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Ordered, value-free findings. Break identifiers are separately available for
|
||||
** record navigation and alert correlation without parsing human-readable text.
|
||||
*******************************************************************************/
|
||||
public class VerificationResult
|
||||
{
|
||||
/*******************************************************************************
|
||||
** One break; null ids apply to coverage and checkpoint findings.
|
||||
*******************************************************************************/
|
||||
public record Break(Long linkId, Long auditId, String reason)
|
||||
{
|
||||
}
|
||||
|
||||
private final List<Break> breaks = new ArrayList<>();
|
||||
private final List<String> warnings = new ArrayList<>();
|
||||
private final Instant startedAt = Instant.now();
|
||||
private long breakCount;
|
||||
private long linksChecked;
|
||||
private Long fromLinkId;
|
||||
private Long toLinkId;
|
||||
|
||||
/*******************************************************************************
|
||||
** Add a finding in deterministic verification order.
|
||||
*******************************************************************************/
|
||||
public void add(Long linkId, Long auditId, String reason)
|
||||
{
|
||||
breakCount++;
|
||||
if(breaks.size() < 1000)
|
||||
{
|
||||
breaks.add(new Break(linkId, auditId, reason));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Count additional SQL offenders without retaining their content.
|
||||
*******************************************************************************/
|
||||
public void omitted(long count)
|
||||
{
|
||||
breakCount += count;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Total findings, including diagnostics omitted by the bounded display limit.
|
||||
*******************************************************************************/
|
||||
public long getBreakCount()
|
||||
{
|
||||
return (breakCount);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Keep warnings bounded even when many checkpoint ranges have been purged.
|
||||
*******************************************************************************/
|
||||
public void warn(String warning)
|
||||
{
|
||||
if(warnings.size() < 100)
|
||||
{
|
||||
warnings.add(warning);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Record the scope that was checked.
|
||||
*******************************************************************************/
|
||||
public void scope(long count, Long from, Long to)
|
||||
{
|
||||
linksChecked = count;
|
||||
fromLinkId = from;
|
||||
toLinkId = to;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Ordered findings.
|
||||
*******************************************************************************/
|
||||
public List<Break> getBreaks()
|
||||
{
|
||||
return (List.copyOf(breaks));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Warnings do not break the chain.
|
||||
*******************************************************************************/
|
||||
public List<String> getWarnings()
|
||||
{
|
||||
return (warnings);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether all checked invariants hold.
|
||||
*******************************************************************************/
|
||||
public boolean passed()
|
||||
{
|
||||
return (breakCount == 0);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Number of checked links.
|
||||
*******************************************************************************/
|
||||
public long getLinksChecked()
|
||||
{
|
||||
return (linksChecked);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** First checked link, null for an empty scope.
|
||||
*******************************************************************************/
|
||||
public Long getFromLinkId()
|
||||
{
|
||||
return (fromLinkId);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Last checked link, null for an empty scope.
|
||||
*******************************************************************************/
|
||||
public Long getToLinkId()
|
||||
{
|
||||
return (toLinkId);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verification start time.
|
||||
*******************************************************************************/
|
||||
public Instant getStartedAt()
|
||||
{
|
||||
return (startedAt);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,477 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit.verify;
|
||||
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.chain.Hashes;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.checkpoint.Checkpoints;
|
||||
import com.kingsrook.qbits.wormaudit.guard.AppendOnlyGuard;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSTransaction;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Snapshot verification of live content, continuity, exact coverage and signed
|
||||
** checkpoints. Selection runs verify predecessor boundaries but deliberately
|
||||
** omit global completeness; full runs begin at the latest retention anchor.
|
||||
*******************************************************************************/
|
||||
public class WormAuditVerifier
|
||||
{
|
||||
public static final int PAGE_SIZE = 1000;
|
||||
private final WormAuditQBitConfig config;
|
||||
|
||||
/*******************************************************************************
|
||||
** Bind trust pins and settle horizon.
|
||||
*******************************************************************************/
|
||||
public WormAuditVerifier(WormAuditQBitConfig config)
|
||||
{
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify all live links and settled coverage.
|
||||
*******************************************************************************/
|
||||
public VerificationResult verify() throws QException
|
||||
{
|
||||
return (verify(null));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Null means full verification; an empty set is an empty selection.
|
||||
*******************************************************************************/
|
||||
public VerificationResult verify(Set<Long> selection) throws QException
|
||||
{
|
||||
WormAuditChainer.JVM_LOCK.lock();
|
||||
try(QBackendTransaction transaction = QBackendTransaction.openFor(new QueryInput(WormAuditTables.CHAIN)))
|
||||
{
|
||||
try
|
||||
{
|
||||
if(transaction instanceof RDBMSTransaction rdbms)
|
||||
{
|
||||
rdbms.getConnection().setTransactionIsolation(Connection.TRANSACTION_REPEATABLE_READ);
|
||||
}
|
||||
VerificationResult result = verify(new AuditStore(transaction), selection, Instant.now());
|
||||
transaction.commit();
|
||||
return (result);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
transaction.rollback();
|
||||
throw (AppendOnlyGuard.map(e));
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
WormAuditChainer.JVM_LOCK.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify one consistent storage snapshot.
|
||||
*******************************************************************************/
|
||||
private VerificationResult verify(AuditStore store, Set<Long> selection, Instant now) throws QException
|
||||
{
|
||||
VerificationResult result = new VerificationResult();
|
||||
if(config.getSigningKey() == null)
|
||||
{
|
||||
result.warn("no signing key configured");
|
||||
}
|
||||
QRecord anchor = store.anchor();
|
||||
long anchorId = anchor.getValueLong("anchor_link_id");
|
||||
long lastId = anchorId;
|
||||
String previous = anchor.getValueString("anchor_hash");
|
||||
long checked = 0;
|
||||
Long first = null;
|
||||
Long last = null;
|
||||
Set<Long> missing = selection == null ? null : new HashSet<>(selection);
|
||||
CheckpointCursor checkpoints = new CheckpointCursor(store, selection, anchor, result);
|
||||
while(true)
|
||||
{
|
||||
List<QRecord> page = store.page(WormAuditTables.CHAIN, lastId, PAGE_SIZE);
|
||||
if(page.isEmpty())
|
||||
{
|
||||
break;
|
||||
}
|
||||
List<QRecord> selected = page.stream().filter(r -> selection == null || selection.contains(r.getValueLong("id"))).toList();
|
||||
Map<Long, QRecord> headers = AuditStore.index(store.in("audit", "id", selected.stream().map(r -> r.getValueLong("audit_id")).toList()));
|
||||
Set<Long> detailIds = new HashSet<>();
|
||||
for(QRecord link : selected)
|
||||
{
|
||||
try
|
||||
{
|
||||
List<Long> ids = AuditStore.detailIds(link);
|
||||
if(ids.size() <= 250)
|
||||
{
|
||||
detailIds.addAll(ids);
|
||||
}
|
||||
}
|
||||
catch(IllegalArgumentException e)
|
||||
{
|
||||
// Content verification reports malformed coverage with its link id.
|
||||
}
|
||||
}
|
||||
Map<Long, QRecord> details = new HashMap<>();
|
||||
List<Long> ids = new ArrayList<>(detailIds);
|
||||
for(int start = 0; start < ids.size(); start += PAGE_SIZE)
|
||||
{
|
||||
details.putAll(AuditStore.index(store.in("auditDetail", "id", ids.subList(start, Math.min(start + PAGE_SIZE, ids.size())))));
|
||||
}
|
||||
for(QRecord link : page)
|
||||
{
|
||||
long id = link.getValueLong("id");
|
||||
long auditId = link.getValueLong("audit_id");
|
||||
if(selection == null || selection.contains(id))
|
||||
{
|
||||
if(missing != null)
|
||||
{
|
||||
missing.remove(id);
|
||||
}
|
||||
checked++;
|
||||
first = first == null ? id : first;
|
||||
last = id;
|
||||
if(selection == null && id != lastId + 1)
|
||||
{
|
||||
result.add(id, auditId, "Missing link ids " + number(lastId + 1) + "–" + number(id - 1) + " (gap)");
|
||||
}
|
||||
if(id != lastId + 1 || previous == null || !previous.equals(link.getValueString("prev_hash")))
|
||||
{
|
||||
result.add(id, auditId, "Break at link " + number(id) + " (audit " + number(auditId) + "): prev-hash discontinuity");
|
||||
}
|
||||
verifyContent(store, link, headers.get(auditId), details, result);
|
||||
}
|
||||
previous = link.getValueString("link_hash");
|
||||
lastId = id;
|
||||
}
|
||||
checkpoints.through(lastId);
|
||||
}
|
||||
checkpoints.through(Long.MAX_VALUE);
|
||||
result.scope(checked, first, last);
|
||||
if(missing != null)
|
||||
{
|
||||
missing.stream().sorted().forEach(id -> result.add(id, null, "Missing selected link " + number(id)));
|
||||
}
|
||||
if(selection == null)
|
||||
{
|
||||
Instant cutoff = now.minusSeconds(config.getSettleHorizonSeconds());
|
||||
if(store.isRdbms())
|
||||
{
|
||||
store.completeness(cutoff, result);
|
||||
}
|
||||
else
|
||||
{
|
||||
completeness(store, store.all(WormAuditTables.CHAIN), cutoff, result);
|
||||
}
|
||||
}
|
||||
return (result);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Recompute exact stored content. Diagnostics name covered detail ids because
|
||||
** v1 stores a document hash, not per-field hashes or prior values.
|
||||
*******************************************************************************/
|
||||
private void verifyContent(AuditStore store, QRecord link, QRecord audit, Map<Long, QRecord> rows, VerificationResult result) throws QException
|
||||
{
|
||||
long id = link.getValueLong("id");
|
||||
long auditId = link.getValueLong("audit_id");
|
||||
if(List.of("kind", "detail_ids", "prev_hash", "link_hash", "chained_at").stream().anyMatch(field -> link.getValue(field) == null))
|
||||
{
|
||||
result.add(id, auditId, "Break at link " + number(id) + " (audit " + number(auditId) + "): malformed link");
|
||||
return;
|
||||
}
|
||||
if(audit == null)
|
||||
{
|
||||
result.add(id, auditId, "Break at link " + number(id) + " (audit " + number(auditId) + "): audit header is missing");
|
||||
return;
|
||||
}
|
||||
if(audit.getValueInstant("timestamp") == null)
|
||||
{
|
||||
result.add(id, auditId, "Break at link " + number(id) + " (audit " + number(auditId) + "): missing audit timestamp");
|
||||
return;
|
||||
}
|
||||
try
|
||||
{
|
||||
List<Long> details = AuditStore.detailIds(link);
|
||||
String kind = link.getValueString("kind");
|
||||
if(!List.of("AUDIT", "LATE_DETAIL").contains(kind) || details.size() > 250 || ("LATE_DETAIL".equals(kind) && details.isEmpty()))
|
||||
{
|
||||
result.add(id, auditId, "Break at link " + number(id) + ": invalid kind or detail coverage");
|
||||
}
|
||||
if(details.size() > 250)
|
||||
{
|
||||
return;
|
||||
}
|
||||
Set<Long> unique = new HashSet<>();
|
||||
for(long detail : details)
|
||||
{
|
||||
if(!unique.add(detail))
|
||||
{
|
||||
result.add(id, auditId, "audit_detail " + number(detail) + " is covered twice or more by the chain");
|
||||
}
|
||||
}
|
||||
String hash = Hashes.link(link.getValueString("prev_hash"), store.canonical(audit, details, rows));
|
||||
if(!hash.equals(link.getValueString("link_hash")))
|
||||
{
|
||||
String table = store.name("auditTable", audit.getValueLong("auditTableId"));
|
||||
result.add(id, auditId, "Break at link " + number(id) + " (audit " + number(auditId) + ", table " + table + ", record "
|
||||
+ audit.getValueString("recordId") + "): content differs from the chained hash"
|
||||
+ (details.isEmpty() ? "" : " (audit_detail " + details.stream().map(WormAuditVerifier::number).collect(java.util.stream.Collectors.joining(", ")) + ")"));
|
||||
}
|
||||
}
|
||||
catch(QException | IllegalArgumentException e)
|
||||
{
|
||||
result.add(id, auditId, "Break at link " + number(id) + " (audit " + number(auditId) + "): " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every settled header and detail must occur once; orphan details also fail.
|
||||
*******************************************************************************/
|
||||
private void completeness(AuditStore store, List<QRecord> links, Instant cutoff, VerificationResult result) throws QException
|
||||
{
|
||||
Map<Long, Integer> headers = new HashMap<>();
|
||||
Map<Long, Integer> details = new HashMap<>();
|
||||
for(QRecord link : links)
|
||||
{
|
||||
if("AUDIT".equals(link.getValueString("kind")))
|
||||
{
|
||||
headers.merge(link.getValueLong("audit_id"), 1, Integer::sum);
|
||||
}
|
||||
try
|
||||
{
|
||||
AuditStore.detailIds(link).forEach(id -> details.merge(id, 1, Integer::sum));
|
||||
}
|
||||
catch(IllegalArgumentException e)
|
||||
{
|
||||
// Invalid coverage was already reported while checking link content.
|
||||
}
|
||||
}
|
||||
Map<Long, QRecord> audits = new HashMap<>();
|
||||
for(QRecord audit : store.all("audit"))
|
||||
{
|
||||
long id = audit.getValueLong("id");
|
||||
audits.put(id, audit);
|
||||
if(audit.getValueInstant("timestamp") == null)
|
||||
{
|
||||
result.add(null, id, "audit " + number(id) + " has no timestamp");
|
||||
}
|
||||
else if(audit.getValueInstant("timestamp").isBefore(cutoff))
|
||||
{
|
||||
coverage("audit", id, headers.getOrDefault(id, 0), result);
|
||||
}
|
||||
}
|
||||
for(QRecord detail : store.all("auditDetail"))
|
||||
{
|
||||
QRecord audit = audits.get(detail.getValueLong("auditId"));
|
||||
if(audit == null || audit.getValueInstant("timestamp") == null || audit.getValueInstant("timestamp").isBefore(cutoff))
|
||||
{
|
||||
coverage("audit_detail", detail.getValueLong("id"), details.getOrDefault(detail.getValueLong("id"), 0), result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Report coverage without printing any audit content.
|
||||
*******************************************************************************/
|
||||
private void coverage(String table, long id, int count, VerificationResult result)
|
||||
{
|
||||
if(count != 1)
|
||||
{
|
||||
result.add(null, table.equals("audit") ? id : null, table + " " + number(id)
|
||||
+ (count == 0 ? " is not covered by the chain" : " is covered twice or more by the chain"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Check pinned signatures even for purged ranges. A partially purged range's
|
||||
** Merkle tree requires its sealed export; never claim to recompute it live.
|
||||
*******************************************************************************/
|
||||
private class CheckpointCursor
|
||||
{
|
||||
private final AuditStore store;
|
||||
private final Set<Long> selection;
|
||||
private final QRecord anchor;
|
||||
private final VerificationResult result;
|
||||
private long after;
|
||||
private QRecord next;
|
||||
|
||||
/*******************************************************************************
|
||||
** Keep only one checkpoint pending as pages complete.
|
||||
*******************************************************************************/
|
||||
CheckpointCursor(AuditStore store, Set<Long> selection, QRecord anchor, VerificationResult result)
|
||||
{
|
||||
this.store = store;
|
||||
this.selection = selection;
|
||||
this.anchor = anchor;
|
||||
this.result = result;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify completed ranges, including missing tails at end of the scan.
|
||||
*******************************************************************************/
|
||||
void through(long lastId) throws QException
|
||||
{
|
||||
while(true)
|
||||
{
|
||||
if(next == null)
|
||||
{
|
||||
List<QRecord> rows = store.page(WormAuditTables.CHECKPOINT, after, 1);
|
||||
if(rows.isEmpty())
|
||||
{
|
||||
return;
|
||||
}
|
||||
next = rows.getFirst();
|
||||
}
|
||||
Long to = next.getValueLong("to_link_id");
|
||||
if(to != null && to > lastId)
|
||||
{
|
||||
return;
|
||||
}
|
||||
check(next);
|
||||
after = next.getValueLong("id");
|
||||
next = null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Re-read at most one capped range of hashes in the same snapshot.
|
||||
*******************************************************************************/
|
||||
private void check(QRecord checkpoint) throws QException
|
||||
{
|
||||
String prefix = "checkpoint " + number(checkpoint.getValueLong("id")) + ": ";
|
||||
if(List.of("from_link_id", "to_link_id", "link_count", "head_hash", "merkle_root", "algorithm", "key_id", "public_key", "signature")
|
||||
.stream().anyMatch(field -> checkpoint.getValue(field) == null))
|
||||
{
|
||||
result.add(null, null, prefix + "malformed checkpoint");
|
||||
return;
|
||||
}
|
||||
long from = checkpoint.getValueLong("from_link_id");
|
||||
long to = checkpoint.getValueLong("to_link_id");
|
||||
long anchorId = anchor.getValueLong("anchor_link_id");
|
||||
if(selection != null && selection.stream().noneMatch(id -> id >= from && id <= to))
|
||||
{
|
||||
return;
|
||||
}
|
||||
var pin = config.getTrustedPublicKeys().get(checkpoint.getValueString("key_id"));
|
||||
if(pin == null)
|
||||
{
|
||||
result.add(null, null, prefix + "untrusted key");
|
||||
}
|
||||
else if(!Checkpoints.verify(checkpoint, pin))
|
||||
{
|
||||
result.add(null, null, prefix + "signature does not verify");
|
||||
}
|
||||
if(from < 1 || to < from || checkpoint.getValueLong("link_count") != to - from + 1
|
||||
|| to - from + 1 > config.getCheckpointEveryLinks())
|
||||
{
|
||||
result.add(null, null, prefix + "invalid link range or count");
|
||||
return;
|
||||
}
|
||||
if(from <= anchorId)
|
||||
{
|
||||
QRecord head = to > anchorId ? store.get(WormAuditTables.CHAIN, to) : null;
|
||||
if(to > anchorId && (head == null || !Objects.equals(head.getValueString("link_hash"), checkpoint.getValueString("head_hash"))))
|
||||
{
|
||||
result.add(to, null, prefix + "head mismatch");
|
||||
}
|
||||
if(to == anchorId && !Objects.equals(anchor.getValueString("anchor_hash"), checkpoint.getValueString("head_hash")))
|
||||
{
|
||||
result.add(to, null, prefix + "head mismatch with purge anchor");
|
||||
}
|
||||
result.warn(prefix + "purged range; Merkle proof requires the sealed export");
|
||||
return;
|
||||
}
|
||||
List<String> hashes = new ArrayList<>();
|
||||
long cursor = from - 1;
|
||||
String head = null;
|
||||
while(cursor < to)
|
||||
{
|
||||
List<QRecord> page = store.page(WormAuditTables.CHAIN, cursor, (int) Math.min(PAGE_SIZE, to - cursor));
|
||||
if(page.isEmpty())
|
||||
{
|
||||
break;
|
||||
}
|
||||
for(QRecord link : page)
|
||||
{
|
||||
cursor = link.getValueLong("id");
|
||||
if(cursor <= to)
|
||||
{
|
||||
hashes.add(link.getValueString("link_hash"));
|
||||
}
|
||||
if(cursor == to)
|
||||
{
|
||||
head = link.getValueString("link_hash");
|
||||
}
|
||||
}
|
||||
}
|
||||
if(!Objects.equals(head, checkpoint.getValueString("head_hash")))
|
||||
{
|
||||
result.add(to, null, prefix + "head mismatch");
|
||||
}
|
||||
try
|
||||
{
|
||||
if(hashes.size() != checkpoint.getValueLong("link_count") || hashes.isEmpty()
|
||||
|| hashes.stream().anyMatch(Objects::isNull) || !Hashes.merkle(hashes).equals(checkpoint.getValueString("merkle_root")))
|
||||
{
|
||||
result.add(null, null, prefix + "merkle mismatch");
|
||||
}
|
||||
}
|
||||
catch(IllegalArgumentException e)
|
||||
{
|
||||
result.add(null, null, prefix + "merkle mismatch");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stable human-readable ids matching the process summary contract.
|
||||
*******************************************************************************/
|
||||
public static String number(long value)
|
||||
{
|
||||
return (String.format(Locale.US, "%,d", value));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,325 @@
|
||||
-- liquibase formatted sql
|
||||
|
||||
-- changeset worm-audit:1
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_chain (
|
||||
id bigint PRIMARY KEY,
|
||||
kind text NOT NULL,
|
||||
audit_id bigint NOT NULL,
|
||||
detail_ids text NOT NULL,
|
||||
prev_hash text NOT NULL,
|
||||
link_hash text NOT NULL,
|
||||
chained_at timestamptz NOT NULL,
|
||||
CHECK (id > 0),
|
||||
CHECK (kind IN ('AUDIT', 'LATE_DETAIL')),
|
||||
CHECK (prev_hash ~ '^[0-9a-f]{64}$' AND link_hash ~ '^[0-9a-f]{64}$'),
|
||||
CHECK (detail_ids = '' OR (detail_ids ~ '^(0|[1-9][0-9]*)(,(0|[1-9][0-9]*))*$' AND cardinality(string_to_array(detail_ids, ',')) <= 250))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_checkpoint (
|
||||
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
from_link_id bigint,
|
||||
to_link_id bigint,
|
||||
link_count bigint,
|
||||
head_hash text,
|
||||
merkle_root text,
|
||||
algorithm text,
|
||||
key_id text,
|
||||
public_key text,
|
||||
signature text,
|
||||
created_at timestamptz
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_export (
|
||||
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
from_link_id bigint,
|
||||
to_link_id bigint,
|
||||
from_checkpoint_id bigint,
|
||||
to_checkpoint_id bigint,
|
||||
bucket_alias text,
|
||||
object_prefix text,
|
||||
manifest_key text,
|
||||
manifest_sha256 text,
|
||||
prev_manifest_sha256 text,
|
||||
object_count bigint,
|
||||
byte_count bigint,
|
||||
lock_mode text,
|
||||
retain_until timestamptz,
|
||||
sealed_at timestamptz
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_verification (
|
||||
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
kind text,
|
||||
"trigger" text,
|
||||
started_at timestamptz,
|
||||
finished_at timestamptz,
|
||||
from_link_id bigint,
|
||||
to_link_id bigint,
|
||||
links_checked bigint,
|
||||
outcome text,
|
||||
break_count bigint,
|
||||
first_break_link_id bigint,
|
||||
first_break_audit_id bigint,
|
||||
first_break_reason text,
|
||||
export_id bigint,
|
||||
run_by text
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.worm_audit_purge (
|
||||
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
requested_at timestamptz,
|
||||
requested_by text,
|
||||
cutoff timestamptz,
|
||||
retention_interval text,
|
||||
outcome text,
|
||||
reason text,
|
||||
audits_deleted bigint,
|
||||
details_deleted bigint,
|
||||
links_deleted bigint,
|
||||
anchor_link_id bigint,
|
||||
anchor_hash text
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS worm_audit_chain_audit ON public.worm_audit_chain(audit_id) WHERE kind = 'AUDIT';
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_chain_audit_all ON public.worm_audit_chain(audit_id);
|
||||
|
||||
-- changeset worm-audit:2 splitStatements:false
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_reject() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp SET timezone = 'UTC' AS $$
|
||||
DECLARE
|
||||
old_time timestamptz;
|
||||
function_owner name;
|
||||
stack text;
|
||||
row_identity text := 'all';
|
||||
BEGIN
|
||||
IF TG_OP = 'DELETE' AND current_setting('worm_audit.purging', true) = 'on' THEN
|
||||
SELECT pg_get_userbyid(proowner) INTO function_owner FROM pg_proc
|
||||
WHERE oid = 'public.worm_audit_purge(timestamptz)'::regprocedure;
|
||||
GET DIAGNOSTICS stack = PG_CONTEXT;
|
||||
IF current_user = function_owner AND position('function worm_audit_purge(timestamp with time zone)' in stack) > 0 THEN
|
||||
IF TG_RELID = '${audit}'::regclass THEN
|
||||
old_time := OLD.timestamp;
|
||||
ELSIF TG_RELID = '${audit_detail}'::regclass THEN
|
||||
SELECT timestamp INTO old_time FROM ${audit} WHERE id = OLD.audit_id;
|
||||
ELSIF TG_RELID = 'public.worm_audit_chain'::regclass THEN
|
||||
SELECT timestamp INTO old_time FROM ${audit} WHERE id = OLD.audit_id;
|
||||
END IF;
|
||||
IF old_time < now() - interval '${retention}' THEN
|
||||
RETURN OLD;
|
||||
END IF;
|
||||
END IF;
|
||||
END IF;
|
||||
IF TG_LEVEL = 'ROW' THEN
|
||||
row_identity := OLD.id::text;
|
||||
END IF;
|
||||
RAISE EXCEPTION USING ERRCODE = 'P0001',
|
||||
MESSAGE = format('worm_audit: audit rows are write-once; %s of "%s" is not allowed (id=%s). Expired rows are removed only by worm_audit_purge().',
|
||||
TG_OP, TG_TABLE_NAME, row_identity),
|
||||
HINT = 'see qbit-worm-audit docs/DESIGN.md section 4';
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:3 splitStatements:false
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_lookup_reject() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $$
|
||||
BEGIN
|
||||
IF TG_OP = 'UPDATE' AND NEW.id IS NOT DISTINCT FROM OLD.id AND NEW.name IS NOT DISTINCT FROM OLD.name THEN
|
||||
RETURN NEW;
|
||||
END IF;
|
||||
RAISE EXCEPTION USING ERRCODE = 'P0001',
|
||||
MESSAGE = format('worm_audit: audit rows are write-once; %s of "%s" is not allowed (id=%s). Expired rows are removed only by worm_audit_purge().',
|
||||
TG_OP, TG_TABLE_NAME, OLD.id),
|
||||
HINT = 'see qbit-worm-audit docs/DESIGN.md section 4';
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:4 splitStatements:false
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'worm_audit_purger') THEN
|
||||
CREATE ROLE worm_audit_purger NOLOGIN;
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:5 splitStatements:false
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_purge(p_cutoff timestamptz) RETURNS void
|
||||
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, public, pg_temp SET timezone = 'UTC' AS $$
|
||||
DECLARE
|
||||
boundary bigint;
|
||||
smaller bigint;
|
||||
anchor text;
|
||||
audits_count bigint;
|
||||
details_count bigint;
|
||||
links_count bigint;
|
||||
BEGIN
|
||||
IF p_cutoff IS NULL OR p_cutoff > now() - interval '${retention}' THEN
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome, reason)
|
||||
VALUES(now(), session_user, p_cutoff, '${retention}', 'REFUSED', 'Cutoff is within retention or null');
|
||||
RETURN;
|
||||
END IF;
|
||||
-- Same lock as the Java chainer; table locks also exclude concurrent audit inserts.
|
||||
PERFORM pg_advisory_xact_lock(86753090421);
|
||||
LOCK TABLE ${audit}, ${audit_detail}, public.worm_audit_chain IN SHARE ROW EXCLUSIVE MODE;
|
||||
SELECT coalesce(min(c.id) - 1, (SELECT max(id) FROM public.worm_audit_chain)) INTO boundary
|
||||
FROM public.worm_audit_chain c LEFT JOIN ${audit} a ON a.id = c.audit_id
|
||||
WHERE a.id IS NULL OR a.timestamp IS NULL OR a.timestamp >= p_cutoff;
|
||||
-- Shrink to a prefix closed over every audit's links AND all its stored details.
|
||||
LOOP
|
||||
SELECT min(c.id) - 1 INTO smaller FROM public.worm_audit_chain c
|
||||
WHERE c.id <= boundary AND (
|
||||
EXISTS (SELECT FROM public.worm_audit_chain later WHERE later.audit_id = c.audit_id AND later.id > boundary)
|
||||
OR EXISTS (SELECT FROM ${audit_detail} d WHERE d.audit_id = c.audit_id AND NOT EXISTS (
|
||||
SELECT FROM public.worm_audit_chain covered WHERE covered.audit_id = c.audit_id
|
||||
AND d.id::text = ANY(string_to_array(covered.detail_ids, ',')))));
|
||||
EXIT WHEN smaller IS NULL;
|
||||
boundary := smaller;
|
||||
END LOOP;
|
||||
SELECT link_hash INTO anchor FROM public.worm_audit_chain WHERE id = boundary;
|
||||
IF anchor IS NULL THEN
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome, reason)
|
||||
VALUES(now(), session_user, p_cutoff, '${retention}', 'REFUSED', 'No complete expired chain prefix');
|
||||
RETURN;
|
||||
END IF;
|
||||
PERFORM set_config('worm_audit.purging', 'on', true);
|
||||
DELETE FROM ${audit_detail} d WHERE EXISTS
|
||||
(SELECT FROM public.worm_audit_chain c WHERE c.audit_id = d.audit_id AND c.id <= boundary);
|
||||
GET DIAGNOSTICS details_count = ROW_COUNT;
|
||||
-- Keep ids locally because links must be deleted before their audit headers.
|
||||
WITH removed AS (DELETE FROM public.worm_audit_chain WHERE id <= boundary RETURNING audit_id),
|
||||
removed_audits AS (DELETE FROM ${audit} WHERE id IN (SELECT audit_id FROM removed) RETURNING id)
|
||||
SELECT (SELECT count(*) FROM removed), (SELECT count(*) FROM removed_audits) INTO links_count, audits_count;
|
||||
PERFORM set_config('worm_audit.purging', 'off', true);
|
||||
INSERT INTO public.worm_audit_purge(requested_at, requested_by, cutoff, retention_interval, outcome,
|
||||
audits_deleted, details_deleted, links_deleted, anchor_link_id, anchor_hash)
|
||||
VALUES(now(), session_user, p_cutoff, '${retention}', 'PURGED', audits_count, details_count, links_count, boundary, anchor);
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:6
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON ${audit};
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON ${audit}
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON ${audit};
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON ${audit}
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON ${audit} FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON ${audit} TO ${app_role};
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON ${audit_detail};
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON ${audit_detail}
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON ${audit_detail};
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON ${audit_detail}
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON ${audit_detail} FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON ${audit_detail} TO ${app_role};
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_chain;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_chain
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_chain;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_chain
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_chain FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON public.worm_audit_chain TO ${app_role};
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_checkpoint;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_checkpoint
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_checkpoint;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_checkpoint
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_checkpoint FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON public.worm_audit_checkpoint TO ${app_role};
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_export;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_export
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_export;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_export
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_export FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON public.worm_audit_export TO ${app_role};
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_verification;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_verification
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_verification;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_verification
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_verification FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON public.worm_audit_verification TO ${app_role};
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON public.worm_audit_purge;
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON public.worm_audit_purge
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON public.worm_audit_purge;
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON public.worm_audit_purge
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON public.worm_audit_purge FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON public.worm_audit_purge TO ${app_role};
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON ${audit_table};
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON ${audit_table}
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_lookup_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON ${audit_table};
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON ${audit_table}
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON ${audit_table} FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON ${audit_table} TO ${app_role};
|
||||
|
||||
DROP TRIGGER IF EXISTS worm_audit_row_guard ON ${audit_user};
|
||||
CREATE TRIGGER worm_audit_row_guard BEFORE UPDATE OR DELETE ON ${audit_user}
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_lookup_reject();
|
||||
DROP TRIGGER IF EXISTS worm_audit_truncate_guard ON ${audit_user};
|
||||
CREATE TRIGGER worm_audit_truncate_guard BEFORE TRUNCATE ON ${audit_user}
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.worm_audit_reject();
|
||||
REVOKE UPDATE, DELETE, TRUNCATE ON ${audit_user} FROM ${app_role};
|
||||
GRANT SELECT, INSERT ON ${audit_user} TO ${app_role};
|
||||
|
||||
-- Lookups may refresh display metadata, but never identities or names.
|
||||
GRANT UPDATE (label, create_date, modify_date) ON ${audit_table} TO ${app_role};
|
||||
GRANT UPDATE (create_date, modify_date) ON ${audit_user} TO ${app_role};
|
||||
|
||||
REVOKE ALL ON FUNCTION public.worm_audit_purge(timestamptz) FROM PUBLIC;
|
||||
REVOKE ALL ON FUNCTION public.worm_audit_purge(timestamptz) FROM ${app_role};
|
||||
GRANT EXECUTE ON FUNCTION public.worm_audit_purge(timestamptz) TO worm_audit_purger;
|
||||
|
||||
-- changeset worm-audit:7 splitStatements:false
|
||||
DO $$
|
||||
DECLARE
|
||||
table_name text;
|
||||
sequence_name text;
|
||||
BEGIN
|
||||
FOREACH table_name IN ARRAY ARRAY['${audit}', '${audit_detail}', '${audit_table}', '${audit_user}',
|
||||
'public.worm_audit_checkpoint', 'public.worm_audit_export', 'public.worm_audit_verification', 'public.worm_audit_purge'] LOOP
|
||||
sequence_name := pg_get_serial_sequence(table_name, 'id');
|
||||
IF sequence_name IS NOT NULL THEN
|
||||
EXECUTE format('GRANT USAGE ON SEQUENCE %s TO ${app_role}', sequence_name);
|
||||
END IF;
|
||||
END LOOP;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:8 splitStatements:false
|
||||
CREATE OR REPLACE FUNCTION public.worm_audit_detail_coverage() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public, pg_temp AS $$
|
||||
BEGIN
|
||||
-- Serialize even direct SQL appends; the GIN index makes overlap checks indexed.
|
||||
PERFORM pg_advisory_xact_lock(86753090421);
|
||||
IF NEW.detail_ids <> '' AND (
|
||||
EXISTS (SELECT FROM unnest(string_to_array(NEW.detail_ids, ',')) AS d(id) GROUP BY d.id HAVING count(*) > 1)
|
||||
OR EXISTS (SELECT FROM public.worm_audit_chain c
|
||||
WHERE string_to_array(c.detail_ids, ',') && string_to_array(NEW.detail_ids, ','))) THEN
|
||||
RAISE EXCEPTION USING ERRCODE = '23505', MESSAGE = 'worm_audit: audit detail is already covered by the chain';
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- changeset worm-audit:9
|
||||
CREATE INDEX IF NOT EXISTS worm_audit_chain_detail_coverage ON public.worm_audit_chain USING gin (string_to_array(detail_ids, ','));
|
||||
DROP TRIGGER IF EXISTS worm_audit_detail_coverage_guard ON public.worm_audit_chain;
|
||||
CREATE TRIGGER worm_audit_detail_coverage_guard BEFORE INSERT ON public.worm_audit_chain
|
||||
FOR EACH ROW EXECUTE FUNCTION public.worm_audit_detail_coverage();
|
||||
@@ -0,0 +1,204 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.KeyPairGenerator;
|
||||
import java.time.Instant;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.wormaudit.canonical.AuditCanonical;
|
||||
import com.kingsrook.qbits.wormaudit.canonical.Jcs;
|
||||
import com.kingsrook.qbits.wormaudit.chain.Hashes;
|
||||
import com.kingsrook.qbits.wormaudit.checkpoint.Checkpoints;
|
||||
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
|
||||
import com.kingsrook.qbits.wormaudit.guard.AppendOnlyGuard;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for canonical and crypto test.
|
||||
*******************************************************************************/
|
||||
class CanonicalAndCryptoTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Verify jcs ordering escaping and invalid types.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void jcsOrderingEscapingAndInvalidTypes()
|
||||
{
|
||||
var value = new LinkedHashMap<String, Object>();
|
||||
value.put("\u20ac", "Euro");
|
||||
value.put("\ud83d\ude00", "Emoji");
|
||||
value.put("\ufb33", "Hebrew");
|
||||
value.put("a", null);
|
||||
String result = Jcs.string(value);
|
||||
assertThat(result)
|
||||
.isEqualTo(
|
||||
"{\"a\":null,\"\u20ac\":\"Euro\",\"\ud83d\ude00\":\"Emoji\",\"\ufb33\":\"Hebrew\"}");
|
||||
assertThat(Jcs.string("\"\\\b\f\n\r\t\u0001/é"))
|
||||
.isEqualTo("\"\\\"\\\\\\b\\f\\n\\r\\t\\u0001/é\"");
|
||||
assertThat(Jcs.string(List.of(Long.MIN_VALUE, Long.MAX_VALUE)))
|
||||
.isEqualTo("[-9223372036854775808,9223372036854775807]");
|
||||
assertThatThrownBy(() -> Jcs.string(1.5)).isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatThrownBy(() -> Jcs.string("\ud800")).isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatThrownBy(() -> Jcs.string("\udc00")).isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatThrownBy(() -> Jcs.string("\ud800x"))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify exact canonical document.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void exactCanonicalDocument()
|
||||
{
|
||||
var audit =
|
||||
new QRecord()
|
||||
.withValue("id", 4211L)
|
||||
.withValue("recordId", 17)
|
||||
.withValue("message", "Record was Edited")
|
||||
.withValue("timestamp", Instant.parse("2026-10-13T21:04:05.123456Z"));
|
||||
var detail =
|
||||
new QRecord()
|
||||
.withValue("id", 9001L)
|
||||
.withValue("fieldName", "title")
|
||||
.withValue("message", "Changed Title")
|
||||
.withValue("oldValue", "hmac:v1:test:old")
|
||||
.withValue("newValue", "hmac:v1:test:new");
|
||||
assertThat(
|
||||
new String(
|
||||
AuditCanonical.bytes(audit, "cms_page", "james", List.of(detail)),
|
||||
StandardCharsets.UTF_8))
|
||||
.isEqualTo(
|
||||
"{\"audit\":{\"clientChannel\":null,\"id\":4211,\"message\":\"Record was"
|
||||
+ " Edited\",\"recordId\":\"17\",\"table\":\"cms_page\",\"timestamp\":\"2026-10-13T21:04:05.123456Z\",\"user\":\"james\"},\"details\":[{\"fieldName\":\"title\",\"id\":9001,\"message\":\"Changed"
|
||||
+ " Title\",\"newValue\":\"hmac:v1:test:new\",\"oldValue\":\"hmac:v1:test:old\"}],\"v\":1}");
|
||||
assertThat(
|
||||
new String(
|
||||
AuditCanonical.bytes(
|
||||
audit,
|
||||
null,
|
||||
null,
|
||||
List.of(
|
||||
new QRecord().withValue("id", 2L),
|
||||
new QRecord().withValue("id", 1L))),
|
||||
StandardCharsets.UTF_8))
|
||||
.containsSubsequence("\"id\":1,", "\"id\":2,");
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify independent hash and merkle vectors.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void independentHashAndMerkleVectors() throws Exception
|
||||
{
|
||||
assertThat(Hashes.link(Hashes.GENESIS, "{}".getBytes(StandardCharsets.UTF_8)))
|
||||
.isEqualTo("72f1d7c888d85ce3159bd8bbf38f3046725d3d4d6cef2b9449d9fb4b58cf7a9c");
|
||||
String a = "11".repeat(32), b = "22".repeat(32), c = "33".repeat(32);
|
||||
assertThat(Hashes.merkle(List.of(a))).isEqualTo(a);
|
||||
assertThat(Hashes.merkle(List.of(a, b, c)))
|
||||
.isEqualTo("2470b93c7954a79f41c0106caeeeae350559c2af06bd9a9ab203b163951f64f7");
|
||||
assertThatThrownBy(() -> Hashes.merkle(List.of()))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify checkpoint pinning and signing.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void checkpointPinningAndSigning() throws Exception
|
||||
{
|
||||
var keys = KeyPairGenerator.getInstance("Ed25519").generateKeyPair();
|
||||
var other = KeyPairGenerator.getInstance("Ed25519").generateKeyPair();
|
||||
var config =
|
||||
new WormAuditQBitConfig()
|
||||
.withSigningKey(
|
||||
() -> new WormAuditQBitConfig.SigningKey("a", keys.getPrivate()))
|
||||
.withTrustedPublicKeys(Map.of("a", keys.getPublic()));
|
||||
var links =
|
||||
List.of(new QRecord().withValue("id", 1L).withValue("link_hash", "11".repeat(32)));
|
||||
var checkpoint = Checkpoints.sign(links, config, Instant.now());
|
||||
assertThat(Checkpoints.verify(checkpoint, keys.getPublic())).isTrue();
|
||||
assertThat(Checkpoints.verify(checkpoint, other.getPublic())).isFalse();
|
||||
assertThat(Checkpoints.verify(checkpoint, null)).isFalse();
|
||||
checkpoint.setValue("signature", "bad");
|
||||
assertThat(Checkpoints.verify(checkpoint, keys.getPublic())).isFalse();
|
||||
checkpoint.setValue("algorithm", "RSA");
|
||||
assertThat(Checkpoints.verify(checkpoint, keys.getPublic())).isFalse();
|
||||
config.withTrustedPublicKeys(Map.of());
|
||||
assertThatThrownBy(() -> Checkpoints.sign(links, config, Instant.now()))
|
||||
.hasMessageContaining("not pinned");
|
||||
config.withTrustedPublicKeys(Map.of("a", other.getPublic()));
|
||||
assertThatThrownBy(() -> Checkpoints.sign(links, config, Instant.now()))
|
||||
.hasMessageContaining("does not match");
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify ddl is reviewed rendering and rejects injection.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void ddlIsReviewedRenderingAndRejectsInjection() throws Exception
|
||||
{
|
||||
String sql = WormAuditDdl.postgres(new WormAuditQBitConfig());
|
||||
assertThat(sql)
|
||||
.isEqualTo(
|
||||
java.nio.file.Files.readString(
|
||||
java.nio.file.Path.of("docs/ddl/worm-audit-postgres.sql")));
|
||||
assertThat(sql).doesNotContain("${");
|
||||
assertThat(sql).contains("GET DIAGNOSTICS stack = PG_CONTEXT", "function worm_audit_purge(timestamp with time zone)", "SET timezone = 'UTC'");
|
||||
// The documented migration contract deliberately uses physical snake_case columns.
|
||||
assertThat(sql).contains("OLD.audit_id", "OLD.timestamp", "NEW.name", "GRANT UPDATE (label, create_date, modify_date)");
|
||||
assertThat(java.nio.file.Files.readString(java.nio.file.Path.of("README.md")))
|
||||
.contains("renders table names only", "audit_detail.audit_id", "audit_user.id/name/create_date/modify_date");
|
||||
assertThat(
|
||||
WormAuditDdl.postgres(
|
||||
new WormAuditQBitConfig()
|
||||
.withRetentionYears(7)
|
||||
.withAppRole("runtime")
|
||||
.withCoreAuditTableNames(Map.of("audit", "history.audit"))))
|
||||
.contains("\"history\".\"audit\"", "7 years", "TO \"runtime\"");
|
||||
assertThatThrownBy(() -> WormAuditDdl.identifier("a; DROP TABLE audit"))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatThrownBy(
|
||||
() ->
|
||||
WormAuditDdl.postgres(
|
||||
new WormAuditQBitConfig().withRetentionYears(0)))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatThrownBy(
|
||||
() -> WormAuditDdl.postgres(new WormAuditQBitConfig().withAppRole("a.b")))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
var sqlError = new java.sql.SQLException("worm_audit: rejected", "P0001");
|
||||
assertThat(AppendOnlyGuard.map(new Exception(sqlError)))
|
||||
.hasMessage(AppendOnlyGuard.MESSAGE);
|
||||
assertThat(
|
||||
AppendOnlyGuard.map(
|
||||
new java.sql.SQLException("ERROR: worm_audit: rejected", "P0001")))
|
||||
.hasMessage(AppendOnlyGuard.MESSAGE);
|
||||
assertThat(AppendOnlyGuard.map(new java.sql.SQLException("other", "P0001")))
|
||||
.hasMessage("WORM audit operation failed");
|
||||
var original = new com.kingsrook.qqq.backend.core.exceptions.QException("original");
|
||||
assertThat(AppendOnlyGuard.map(original)).isSameAs(original);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,347 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import java.security.KeyPairGenerator;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.chain.Hashes;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.OLD;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.STORE;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.core;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.detail;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.remove;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.tamper;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for chain verification test.
|
||||
*******************************************************************************/
|
||||
class ChainVerificationTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Verify signed.
|
||||
*******************************************************************************/
|
||||
static WormAuditQBitConfig signed() throws Exception
|
||||
{
|
||||
var pair = KeyPairGenerator.getInstance("Ed25519").generateKeyPair();
|
||||
return (new WormAuditQBitConfig()
|
||||
.withCheckpointEveryLinks(1)
|
||||
.withSigningKey(() -> new WormAuditQBitConfig.SigningKey("test", pair.getPrivate()))
|
||||
.withTrustedPublicKeys(Map.of("test", pair.getPublic())));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify late headers details and chunk boundaries.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void lateHeadersDetailsAndChunkBoundaries() throws Exception
|
||||
{
|
||||
var config = signed().withBatchSize(1);
|
||||
init(config);
|
||||
audit(20);
|
||||
detail(1, 20);
|
||||
var writer = new WormAuditChainer(config);
|
||||
assertThat(writer.chainOnce().audits()).isEqualTo(1);
|
||||
audit(1);
|
||||
detail(2, 1);
|
||||
detail(3, 20);
|
||||
assertThat(writer.chainOnce().lateDetails()).isEqualTo(1);
|
||||
for(long i = 4; i <= 260; i++)
|
||||
{
|
||||
detail(i, 1);
|
||||
}
|
||||
assertThat(writer.chainOnce().lateDetails()).isEqualTo(257);
|
||||
var links = STORE.all(WormAuditTables.CHAIN);
|
||||
assertThat(links).extracting(r -> r.getValueLong("id")).containsExactly(1L, 2L, 3L, 4L, 5L);
|
||||
assertThat(links).allMatch(r -> AuditStore.detailIds(r).size() <= 250);
|
||||
var result = new WormAuditVerifier(config).verify();
|
||||
assertThat(result.getBreaks()).isEmpty();
|
||||
assertThat(result.getLinksChecked()).isEqualTo(5);
|
||||
assertThat(result.getFromLinkId()).isEqualTo(1);
|
||||
assertThat(result.getToLinkId()).isEqualTo(5);
|
||||
assertThat(writer.chainOnce().links()).isZero();
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify overflow at initial chain and settling and elapsed checkpoint.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void overflowAtInitialChainAndSettlingAndElapsedCheckpoint() throws Exception
|
||||
{
|
||||
var config = signed().withCheckpointEveryLinks(10000).withCheckpointEveryMinutes(1);
|
||||
init(config);
|
||||
audit(1);
|
||||
for(long i = 1; i <= 251; i++)
|
||||
{
|
||||
detail(i, 1);
|
||||
}
|
||||
var writer = new WormAuditChainer(config);
|
||||
assertThat(writer.chainOnce(OLD.plusSeconds(30)).links()).isZero();
|
||||
Instant now = Instant.now();
|
||||
assertThat(writer.chainOnce(now).links()).isEqualTo(2);
|
||||
assertThat(STORE.all(WormAuditTables.CHECKPOINT)).isEmpty();
|
||||
assertThat(writer.chainOnce(now.plusSeconds(61)).checkpoints()).isEqualTo(1);
|
||||
assertThat(new WormAuditVerifier(config).verify().passed()).isTrue();
|
||||
config.withSigningKey(null);
|
||||
assertThat(new WormAuditVerifier(config).verify().getWarnings())
|
||||
.contains("no signing key configured");
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify detects breaks.
|
||||
*******************************************************************************/
|
||||
@ParameterizedTest
|
||||
@ValueSource(
|
||||
strings =
|
||||
{
|
||||
"content",
|
||||
"headerDeleted",
|
||||
"detailDeleted",
|
||||
"inserted",
|
||||
"detailInserted",
|
||||
"gap",
|
||||
"previous",
|
||||
"reorder",
|
||||
"forged",
|
||||
"untrusted",
|
||||
"merkle",
|
||||
"head",
|
||||
"duplicate",
|
||||
"kind",
|
||||
"lookup",
|
||||
"range",
|
||||
"nullCheckpoint",
|
||||
"nullPrevious",
|
||||
"nullTimestamp"
|
||||
})
|
||||
void detectsBreaks(String scenario) throws Exception
|
||||
{
|
||||
var config = signed();
|
||||
init(config);
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
audit(2);
|
||||
detail(2, 2);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
String expected;
|
||||
switch(scenario)
|
||||
{
|
||||
case "content" ->
|
||||
{
|
||||
tamper("auditDetail", 1, "newValue", "tampered");
|
||||
expected =
|
||||
"Break at link 1 (audit 1, table cms_page, record 17): content differs from"
|
||||
+ " the chained hash (audit_detail 1)";
|
||||
}
|
||||
case "headerDeleted" ->
|
||||
{
|
||||
remove("audit", 1);
|
||||
expected = "audit header is missing";
|
||||
}
|
||||
case "detailDeleted" ->
|
||||
{
|
||||
remove("auditDetail", 1);
|
||||
expected = "audit_detail 1 is missing";
|
||||
}
|
||||
case "inserted" ->
|
||||
{
|
||||
audit(3);
|
||||
expected = "audit 3 is not covered";
|
||||
}
|
||||
case "detailInserted" ->
|
||||
{
|
||||
detail(3, 1);
|
||||
expected = "audit_detail 3 is not covered";
|
||||
}
|
||||
case "gap" ->
|
||||
{
|
||||
remove(WormAuditTables.CHAIN, 1);
|
||||
expected = "gap";
|
||||
}
|
||||
case "previous" ->
|
||||
{
|
||||
tamper(WormAuditTables.CHAIN, 2, "prev_hash", Hashes.GENESIS);
|
||||
expected = "prev-hash discontinuity";
|
||||
}
|
||||
case "reorder" ->
|
||||
{
|
||||
tamper(WormAuditTables.CHAIN, 1, "audit_id", 2L);
|
||||
expected = "another audit";
|
||||
}
|
||||
case "forged" ->
|
||||
{
|
||||
tamper(WormAuditTables.CHECKPOINT, 1, "signature", "AA==");
|
||||
expected = "checkpoint 1: signature does not verify";
|
||||
}
|
||||
case "untrusted" ->
|
||||
{
|
||||
config.withTrustedPublicKeys(Map.of());
|
||||
expected = "untrusted key";
|
||||
}
|
||||
case "merkle" ->
|
||||
{
|
||||
tamper(WormAuditTables.CHECKPOINT, 1, "merkle_root", Hashes.GENESIS);
|
||||
expected = "merkle mismatch";
|
||||
}
|
||||
case "head" ->
|
||||
{
|
||||
tamper(WormAuditTables.CHECKPOINT, 1, "head_hash", Hashes.GENESIS);
|
||||
expected = "head mismatch";
|
||||
}
|
||||
case "duplicate" ->
|
||||
{
|
||||
tamper(WormAuditTables.CHAIN, 1, "detail_ids", "1,1");
|
||||
expected = "audit_detail 1 is covered twice";
|
||||
}
|
||||
case "kind" ->
|
||||
{
|
||||
tamper(WormAuditTables.CHAIN, 1, "kind", "INVALID");
|
||||
expected = "invalid kind";
|
||||
}
|
||||
case "lookup" ->
|
||||
{
|
||||
tamper("auditTable", 1, "name", "renamed");
|
||||
expected = "content differs";
|
||||
}
|
||||
case "nullCheckpoint" ->
|
||||
{
|
||||
tamper(WormAuditTables.CHECKPOINT, 1, "signature", null);
|
||||
expected = "malformed checkpoint";
|
||||
}
|
||||
case "nullPrevious" ->
|
||||
{
|
||||
tamper(WormAuditTables.CHAIN, 1, "prev_hash", null);
|
||||
expected = "malformed link";
|
||||
}
|
||||
case "nullTimestamp" ->
|
||||
{
|
||||
tamper("audit", 1, "timestamp", null);
|
||||
expected = "missing audit timestamp";
|
||||
}
|
||||
default ->
|
||||
{
|
||||
tamper(WormAuditTables.CHECKPOINT, 1, "link_count", 22L);
|
||||
expected = "invalid link range";
|
||||
}
|
||||
}
|
||||
var result = new WormAuditVerifier(config).verify();
|
||||
assertThat(result.passed()).isFalse();
|
||||
assertThat(result.getBreaks()).anyMatch(b -> b.reason().contains(expected));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify selection does not run completeness and checks boundary.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void selectionDoesNotRunCompletenessAndChecksBoundary() throws Exception
|
||||
{
|
||||
var config = signed();
|
||||
init(config);
|
||||
audit(1);
|
||||
audit(2);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
audit(3);
|
||||
assertThat(new WormAuditVerifier(config).verify(Set.of(2L)).passed()).isTrue();
|
||||
assertThat(new WormAuditVerifier(config).verify(Set.of()).getLinksChecked()).isZero();
|
||||
assertThat(new WormAuditVerifier(config).verify(Set.of(99L)).getBreaks())
|
||||
.anyMatch(b -> b.reason().contains("Missing selected link 99"));
|
||||
tamper(WormAuditTables.CHAIN, 2, "prev_hash", Hashes.GENESIS);
|
||||
assertThat(new WormAuditVerifier(config).verify(Set.of(2L)).getBreaks())
|
||||
.anyMatch(b -> b.reason().contains("prev-hash"));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify continues after purge anchor including empty live chain.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void continuesAfterPurgeAnchorIncludingEmptyLiveChain() throws Exception
|
||||
{
|
||||
var config = signed();
|
||||
init(config);
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
audit(2);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
var first = STORE.get(WormAuditTables.CHAIN, 1);
|
||||
STORE.insert(
|
||||
WormAuditTables.PURGE,
|
||||
List.of(
|
||||
new com.kingsrook.qqq.backend.core.model.data.QRecord()
|
||||
.withValue("outcome", "PURGED")
|
||||
.withValue("anchor_link_id", 1L)
|
||||
.withValue("anchor_hash", first.getValueString("link_hash"))));
|
||||
remove("auditDetail", 1);
|
||||
remove("audit", 1);
|
||||
remove(WormAuditTables.CHAIN, 1);
|
||||
assertThat(new WormAuditVerifier(config).verify().passed()).isTrue();
|
||||
var last = STORE.get(WormAuditTables.CHAIN, 2);
|
||||
STORE.insert(
|
||||
WormAuditTables.PURGE,
|
||||
List.of(
|
||||
new com.kingsrook.qqq.backend.core.model.data.QRecord()
|
||||
.withValue("outcome", "PURGED")
|
||||
.withValue("anchor_link_id", 2L)
|
||||
.withValue("anchor_hash", last.getValueString("link_hash"))));
|
||||
remove("audit", 2);
|
||||
remove(WormAuditTables.CHAIN, 2);
|
||||
assertThat(new WormAuditVerifier(config).verify().passed()).isTrue();
|
||||
audit(3);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
assertThat(STORE.all(WormAuditTables.CHAIN))
|
||||
.extracting(r -> r.getValueLong("id"))
|
||||
.containsExactly(3L);
|
||||
assertThat(new WormAuditVerifier(config).verify().passed()).isTrue();
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Findings retain bounded diagnostics while recording the complete failure count.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void boundedDiagnosticsPreserveTotalCount() throws Exception
|
||||
{
|
||||
init(new WormAuditQBitConfig());
|
||||
var result = new com.kingsrook.qbits.wormaudit.verify.VerificationResult();
|
||||
for(long id = 1; id <= 1200; id++)
|
||||
{
|
||||
result.add(id, id, "gap");
|
||||
result.warn("warning " + id);
|
||||
}
|
||||
result.omitted(20);
|
||||
assertThat(result.getBreaks()).hasSize(1000);
|
||||
assertThat(result.getWarnings()).hasSize(100);
|
||||
assertThat(result.getBreakCount()).isEqualTo(1220);
|
||||
assertThat(result.passed()).isFalse();
|
||||
assertThat(com.kingsrook.qbits.wormaudit.processes.VerificationRuns.save(result, "USER").getValueLong("break_count")).isEqualTo(1220);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for docker condition test.
|
||||
*******************************************************************************/
|
||||
class DockerConditionTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Verify never skips in ci.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void neverSkipsInCi()
|
||||
{
|
||||
assertThat(RequiresDockerCondition.evaluate(false, null).isDisabled()).isTrue();
|
||||
assertThat(RequiresDockerCondition.evaluate(true, "true").isDisabled()).isFalse();
|
||||
assertThatThrownBy(() -> RequiresDockerCondition.evaluate(false, "false"))
|
||||
.hasMessageContaining("CI is set");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import javax.crypto.Mac;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Base64;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.UUID;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.actions.audits.AuditAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.audits.AuditActionCustomizerInterface;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QSupplementalFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.OLD;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.STORE;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.core;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.detail;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for phi canary test.
|
||||
*******************************************************************************/
|
||||
class PhiCanaryTest
|
||||
{
|
||||
// RC.5 has AiFieldMetaData but not DataClassification; use a test-local marker.
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for phi.
|
||||
*******************************************************************************/
|
||||
public static class Phi extends QSupplementalFieldMetaData
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Verify get type.
|
||||
*******************************************************************************/
|
||||
public String getType()
|
||||
{
|
||||
return ("test-phi");
|
||||
}
|
||||
}
|
||||
|
||||
// Real RC.5 AuditAction customization point: rewrite details BEFORE insertion.
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for token hook.
|
||||
*******************************************************************************/
|
||||
public static class TokenHook implements AuditActionCustomizerInterface
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Verify customize input.
|
||||
*******************************************************************************/
|
||||
public void customizeInput(AuditSingleInput input)
|
||||
{
|
||||
for(QRecord detail : input.getDetails())
|
||||
{
|
||||
var field =
|
||||
QContext.getQInstance()
|
||||
.getTable(input.getAuditTableName())
|
||||
.getField(detail.getValueString("fieldName"));
|
||||
if(field.getSupplementalMetaData("test-phi") != null)
|
||||
{
|
||||
for(String column : List.of("oldValue", "newValue"))
|
||||
{
|
||||
try
|
||||
{
|
||||
Mac mac = Mac.getInstance("HmacSHA256");
|
||||
mac.init(
|
||||
new SecretKeySpec(
|
||||
"fixture-key-not-production"
|
||||
.getBytes(StandardCharsets.UTF_8),
|
||||
"HmacSHA256"));
|
||||
detail.setValue(
|
||||
column,
|
||||
"hmac:v1:test:"
|
||||
+ Base64.getUrlEncoder()
|
||||
.withoutPadding()
|
||||
.encodeToString(
|
||||
mac.doFinal(
|
||||
detail.getValueString(column)
|
||||
.getBytes(
|
||||
StandardCharsets
|
||||
.UTF_8))));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
}
|
||||
detail.setValue("message", "Changed Diagnosis");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify token hook keeps every audit and chain value free of canary.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void tokenHookKeepsEveryAuditAndChainValueFreeOfCanary() throws Exception
|
||||
{
|
||||
var config = ChainVerificationTest.signed();
|
||||
var instance = init(config);
|
||||
instance.addTable(
|
||||
new QTableMetaData()
|
||||
.withName("patient")
|
||||
.withBackendName("memory")
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
|
||||
.withField(
|
||||
new QFieldMetaData("diagnosis", QFieldType.STRING)
|
||||
.withSupplementalMetaData(new Phi())));
|
||||
instance.addSupplementalCustomizer(
|
||||
AuditActionCustomizerInterface.CUSTOMIZER_TYPE,
|
||||
new QCodeReference(TokenHook.class));
|
||||
String canary = "CANARY-PHI-" + UUID.randomUUID();
|
||||
var detail =
|
||||
new QRecord()
|
||||
.withValue("fieldName", "diagnosis")
|
||||
.withValue("oldValue", canary)
|
||||
.withValue("newValue", canary)
|
||||
.withValue("message", "Changed Diagnosis from " + canary + " to " + canary);
|
||||
new AuditAction()
|
||||
.execute(
|
||||
new AuditInput()
|
||||
.withAuditSingleInput(
|
||||
new AuditSingleInput()
|
||||
.withAuditTableName("patient")
|
||||
.withAuditUserName("james")
|
||||
.withRecordId(17)
|
||||
.withTimestamp(OLD)
|
||||
.withMessage("Record was Edited")
|
||||
.withDetails(List.of(detail))));
|
||||
assertThat(STORE.all("audit")).hasSize(1);
|
||||
assertThat(STORE.all("auditDetail")).hasSize(1);
|
||||
assertThat(STORE.all("auditDetail").getFirst().getValueString("newValue"))
|
||||
.startsWith("hmac:v1:test:");
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
List<String> values = new ArrayList<>();
|
||||
for(String table :
|
||||
List.of(
|
||||
"audit",
|
||||
"auditDetail",
|
||||
WormAuditTables.CHAIN,
|
||||
WormAuditTables.CHECKPOINT))
|
||||
{
|
||||
STORE.all(table)
|
||||
.forEach(
|
||||
r ->
|
||||
r.getValues().values().stream()
|
||||
.filter(Objects::nonNull)
|
||||
.map(Object::toString)
|
||||
.forEach(values::add));
|
||||
}
|
||||
for(var link : STORE.all(WormAuditTables.CHAIN))
|
||||
{
|
||||
values.add(
|
||||
new String(
|
||||
STORE.canonical(
|
||||
STORE.get("audit", link.getValueLong("audit_id")),
|
||||
AuditStore.detailIds(link)),
|
||||
StandardCharsets.UTF_8));
|
||||
}
|
||||
for(String value : values)
|
||||
{
|
||||
assertThat(value).doesNotContain(canary);
|
||||
// Every longer substring contains a 5-character substring.
|
||||
for(int start = 0; start <= canary.length() - 5; start++)
|
||||
{
|
||||
assertThat(value).doesNotContain(canary.substring(start, start + 5));
|
||||
}
|
||||
}
|
||||
assertThat(new WormAuditVerifier(config).verify().passed()).isTrue();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,552 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.testcontainers.containers.PostgreSQLContainer;
|
||||
import org.testcontainers.junit.jupiter.Container;
|
||||
import org.testcontainers.junit.jupiter.Testcontainers;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.STORE;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.core;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.detail;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.lookups;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for postgres acceptance test.
|
||||
*******************************************************************************/
|
||||
@ExtendWith(RequiresDockerCondition.class)
|
||||
@Testcontainers
|
||||
class PostgresAcceptanceTest
|
||||
{
|
||||
@Container
|
||||
static final PostgreSQLContainer<?> POSTGRES = new PostgreSQLContainer<>("postgres:16-alpine");
|
||||
|
||||
WormAuditQBitConfig config;
|
||||
QInstance instance;
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify owner.
|
||||
*******************************************************************************/
|
||||
Connection owner() throws SQLException
|
||||
{
|
||||
return (DriverManager.getConnection(
|
||||
POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword()));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify app.
|
||||
*******************************************************************************/
|
||||
Connection app() throws SQLException
|
||||
{
|
||||
return (DriverManager.getConnection(POSTGRES.getJdbcUrl(), "app", "fixture"));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify sql.
|
||||
*******************************************************************************/
|
||||
void sql(Connection c, String sql) throws SQLException
|
||||
{
|
||||
try(var s = c.createStatement())
|
||||
{
|
||||
s.execute(sql);
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify count.
|
||||
*******************************************************************************/
|
||||
long count(Connection c, String sql) throws SQLException
|
||||
{
|
||||
try(var s = c.createStatement();
|
||||
var r = s.executeQuery(sql))
|
||||
{
|
||||
r.next();
|
||||
return (r.getLong(1));
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify database.
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void database() throws Exception
|
||||
{
|
||||
config = ChainVerificationTest.signed().withSettleHorizonSeconds(0);
|
||||
instance = RdbmsTestSupport.instance(POSTGRES.getJdbcUrl(), "app", "fixture", config, true);
|
||||
try(var c = owner())
|
||||
{
|
||||
sql(
|
||||
c,
|
||||
"DROP SCHEMA public CASCADE; CREATE SCHEMA public; GRANT USAGE ON SCHEMA public"
|
||||
+ " TO PUBLIC");
|
||||
sql(
|
||||
c,
|
||||
"DO $$ BEGIN IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='app') THEN"
|
||||
+ " CREATE ROLE app LOGIN PASSWORD 'fixture'; END IF; END $$");
|
||||
RdbmsTestSupport.createTables(c, instance, true);
|
||||
String ddl = WormAuditDdl.postgres(config);
|
||||
sql(c, ddl);
|
||||
sql(c, ddl);
|
||||
}
|
||||
QContext.init(instance, new QSession());
|
||||
lookups();
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify grants triggers lookup rules and spoofed flag.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void grantsTriggersLookupRulesAndSpoofedFlag() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
try(var c = owner();
|
||||
var runtime = app())
|
||||
{
|
||||
assertThatThrownBy(() -> sql(runtime,
|
||||
"INSERT INTO worm_audit_chain(id,kind,audit_id,detail_ids,prev_hash,link_hash,chained_at) "
|
||||
+ "SELECT 99,'LATE_DETAIL',audit_id,detail_ids,prev_hash,link_hash,chained_at FROM worm_audit_chain WHERE id=1"))
|
||||
.isInstanceOf(SQLException.class).satisfies(e -> assertThat(((SQLException) e).getSQLState()).isEqualTo("23505"));
|
||||
|
||||
for(String table :
|
||||
List.of("worm_audit_export", "worm_audit_verification", "worm_audit_purge"))
|
||||
{
|
||||
sql(c, "INSERT INTO " + table + " DEFAULT VALUES");
|
||||
}
|
||||
List<String> tables =
|
||||
List.of(
|
||||
"audit",
|
||||
"audit_detail",
|
||||
"worm_audit_chain",
|
||||
"worm_audit_checkpoint",
|
||||
"worm_audit_export",
|
||||
"worm_audit_verification",
|
||||
"worm_audit_purge");
|
||||
for(String table : tables)
|
||||
{
|
||||
assertThat(
|
||||
count(
|
||||
c,
|
||||
"SELECT has_table_privilege('app','"
|
||||
+ table
|
||||
+ "','UPDATE')::int"))
|
||||
.isZero();
|
||||
assertThat(
|
||||
count(
|
||||
c,
|
||||
"SELECT has_table_privilege('app','"
|
||||
+ table
|
||||
+ "','DELETE')::int"))
|
||||
.isZero();
|
||||
assertThat(
|
||||
count(
|
||||
c,
|
||||
"SELECT has_table_privilege('app','"
|
||||
+ table
|
||||
+ "','TRUNCATE')::int"))
|
||||
.isZero();
|
||||
// PostgreSQL checks grants before firing triggers: an unprivileged app
|
||||
// receives 42501. Temporarily grant DML to prove the trigger defense too.
|
||||
assertThatThrownBy(() -> sql(runtime, "UPDATE " + table + " SET id=id"))
|
||||
.isInstanceOf(SQLException.class)
|
||||
.satisfies(
|
||||
e ->
|
||||
assertThat(((SQLException) e).getSQLState())
|
||||
.isEqualTo("42501"));
|
||||
sql(c, "GRANT UPDATE, DELETE, TRUNCATE ON " + table + " TO app");
|
||||
for(String operation :
|
||||
List.of(
|
||||
"UPDATE " + table + " SET id=id",
|
||||
"DELETE FROM " + table,
|
||||
"TRUNCATE " + table))
|
||||
{
|
||||
assertThatThrownBy(() -> sql(runtime, operation))
|
||||
.isInstanceOf(SQLException.class)
|
||||
.hasMessageContaining("worm_audit:")
|
||||
.satisfies(
|
||||
e ->
|
||||
assertThat(((SQLException) e).getSQLState())
|
||||
.isEqualTo("P0001"));
|
||||
}
|
||||
}
|
||||
sql(runtime, "SELECT set_config('worm_audit.purging','on',false)");
|
||||
assertThatThrownBy(() -> sql(runtime, "DELETE FROM audit"))
|
||||
.hasMessageContaining("worm_audit:");
|
||||
sql(runtime, "UPDATE audit_table SET label='Allowed label'");
|
||||
sql(runtime, "UPDATE audit_user SET modify_date=now()");
|
||||
for(String table : List.of("audit_table", "audit_user"))
|
||||
{
|
||||
sql(c, "GRANT UPDATE, DELETE ON " + table + " TO app");
|
||||
assertThatThrownBy(() -> sql(runtime, "UPDATE " + table + " SET name='renamed'"))
|
||||
.hasMessageContaining("worm_audit:");
|
||||
assertThatThrownBy(() -> sql(runtime, "UPDATE " + table + " SET id=id+1"))
|
||||
.hasMessageContaining("worm_audit:");
|
||||
assertThatThrownBy(() -> sql(runtime, "DELETE FROM " + table))
|
||||
.hasMessageContaining("worm_audit:");
|
||||
assertThatThrownBy(() -> sql(c, "TRUNCATE " + table))
|
||||
.hasMessageContaining("worm_audit:");
|
||||
sql(runtime, "UPDATE " + table + " SET modify_date=now()");
|
||||
}
|
||||
sql(runtime, "UPDATE audit_table SET label='New label'");
|
||||
sql(c, WormAuditDdl.postgres(config));
|
||||
assertThat(
|
||||
count(
|
||||
c,
|
||||
"SELECT"
|
||||
+ " has_function_privilege('app','worm_audit_purge(timestamptz)','EXECUTE')::int"))
|
||||
.isZero();
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify retention refusal purge anchor and real backend tamper.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void retentionRefusalPurgeAnchorAndRealBackendTamper() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
audit(2);
|
||||
detail(2, 2);
|
||||
// A live suffix remains after the expired prefix is removed.
|
||||
try(var c = owner())
|
||||
{
|
||||
sql(
|
||||
c,
|
||||
"INSERT INTO audit(id,audit_table_id,audit_user_id,record_id,message,timestamp)"
|
||||
+ " VALUES(3,1,1,17,'Record was Edited',now()-interval '2 minutes')");
|
||||
}
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).isEmpty();
|
||||
String anchor = STORE.get(WormAuditTables.CHAIN, 2).getValueString("link_hash");
|
||||
try(var c = owner())
|
||||
{
|
||||
sql(
|
||||
c,
|
||||
"DO $$ BEGIN IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='purge_login')"
|
||||
+ " THEN CREATE ROLE purge_login LOGIN PASSWORD 'purger-fixture'; END IF;"
|
||||
+ " END $$; GRANT worm_audit_purger TO purge_login");
|
||||
}
|
||||
try(var purger =
|
||||
DriverManager.getConnection(
|
||||
POSTGRES.getJdbcUrl(), "purge_login", "purger-fixture"))
|
||||
{
|
||||
sql(purger, "SELECT worm_audit_purge(now())");
|
||||
assertThat(STORE.all(WormAuditTables.PURGE).getFirst().getValueString("outcome"))
|
||||
.isEqualTo("REFUSED");
|
||||
sql(purger, "SELECT worm_audit_purge(now()-interval '6 years')");
|
||||
}
|
||||
var purge = STORE.all(WormAuditTables.PURGE).getLast();
|
||||
assertThat(purge.getValueString("outcome")).isEqualTo("PURGED");
|
||||
assertThat(purge.getValueString("requested_by")).isEqualTo("purge_login");
|
||||
assertThat(purge.getValueLong("audits_deleted")).isEqualTo(2);
|
||||
assertThat(purge.getValueLong("details_deleted")).isEqualTo(2);
|
||||
assertThat(purge.getValueLong("links_deleted")).isEqualTo(2);
|
||||
assertThat(purge.getValueString("anchor_hash")).isEqualTo(anchor);
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).isEmpty();
|
||||
detail(3, 3);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
try(var c = owner())
|
||||
{
|
||||
sql(
|
||||
c,
|
||||
"ALTER TABLE audit_detail DISABLE TRIGGER ALL; UPDATE audit_detail SET"
|
||||
+ " new_value='tampered' WHERE id=3; ALTER TABLE audit_detail ENABLE"
|
||||
+ " TRIGGER ALL");
|
||||
}
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks())
|
||||
.anyMatch(
|
||||
b ->
|
||||
b.reason()
|
||||
.contains(
|
||||
"Break at link 4 (audit 3, table cms_page,"
|
||||
+ " record 17)")
|
||||
&& b.reason().contains("audit_detail 3"));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify prefix cannot split an audit and advisory lock skips tick.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void prefixCannotSplitAnAuditAndAdvisoryLockSkipsTick() throws Exception
|
||||
{
|
||||
// This retention fixture deliberately polls a detail for a six-year-old header.
|
||||
config.withLateDetailLookbackHours(24 * 365 * 10);
|
||||
audit(1);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
try(var c = owner())
|
||||
{
|
||||
sql(
|
||||
c,
|
||||
"INSERT INTO audit(id,audit_table_id,audit_user_id,record_id,message,timestamp)"
|
||||
+ " VALUES(2,1,1,17,'Record was Edited',now()-interval '2 minutes')");
|
||||
}
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
detail(1, 1);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
try(var c = owner())
|
||||
{
|
||||
sql(c, "SELECT worm_audit_purge(now()-interval '6 years')");
|
||||
assertThat(STORE.all(WormAuditTables.PURGE).getLast().getValueString("outcome"))
|
||||
.isEqualTo("REFUSED");
|
||||
assertThat(STORE.all(WormAuditTables.CHAIN)).hasSize(3);
|
||||
c.setAutoCommit(false);
|
||||
sql(c, "SELECT pg_advisory_xact_lock(" + WormAuditChainer.ADVISORY_LOCK + ")");
|
||||
assertThat(new WormAuditChainer(config).chainOnce().skipped()).isTrue();
|
||||
c.rollback();
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify eight writers and three chainers with late commits.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void eightWritersAndThreeChainersWithLateCommits() throws Exception
|
||||
{
|
||||
var done = new AtomicBoolean(false);
|
||||
var executor = Executors.newFixedThreadPool(11);
|
||||
List<Future<?>> writers = new ArrayList<>();
|
||||
List<Future<?>> chainers = new ArrayList<>();
|
||||
try
|
||||
{
|
||||
for(int thread = 0; thread < 8; thread++)
|
||||
{
|
||||
final int writer = thread;
|
||||
writers.add(
|
||||
executor.submit(
|
||||
() ->
|
||||
{
|
||||
try(var c = app())
|
||||
{
|
||||
for(int i = 0; i < 8; i++)
|
||||
{
|
||||
long id = 1000L + writer * 100 + i;
|
||||
c.setAutoCommit(false);
|
||||
sql(
|
||||
c,
|
||||
"INSERT INTO"
|
||||
+ " audit(id,audit_table_id,audit_user_id,record_id,message,timestamp)"
|
||||
+ " VALUES("
|
||||
+ id
|
||||
+ ",1,1,17,'Record was"
|
||||
+ " Edited',now()-interval '2"
|
||||
+ " minutes')");
|
||||
if(i % 2 == 0)
|
||||
{
|
||||
c.commit();
|
||||
}
|
||||
Thread.sleep(5L * (writer + 1));
|
||||
sql(
|
||||
c,
|
||||
"INSERT INTO"
|
||||
+ " audit_detail(id,audit_id,field_name,message,old_value,new_value)"
|
||||
+ " VALUES("
|
||||
+ id
|
||||
+ ","
|
||||
+ id
|
||||
+ ",'title','Changed"
|
||||
+ " Title','old','new')");
|
||||
c.commit();
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}));
|
||||
}
|
||||
for(int i = 0; i < 3; i++)
|
||||
{
|
||||
chainers.add(
|
||||
executor.submit(
|
||||
() ->
|
||||
{
|
||||
QContext.init(instance, new QSession());
|
||||
try
|
||||
{
|
||||
do
|
||||
{
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
Thread.sleep(10);
|
||||
} while(!done.get());
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}));
|
||||
}
|
||||
for(var writer : writers)
|
||||
{
|
||||
writer.get(90, TimeUnit.SECONDS);
|
||||
}
|
||||
done.set(true);
|
||||
for(var chainer : chainers)
|
||||
{
|
||||
chainer.get(90, TimeUnit.SECONDS);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
done.set(true);
|
||||
executor.shutdownNow();
|
||||
}
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
var links = STORE.all(WormAuditTables.CHAIN);
|
||||
for(int i = 0; i < links.size(); i++)
|
||||
{
|
||||
assertThat(links.get(i).getValueLong("id")).isEqualTo(i + 1L);
|
||||
}
|
||||
assertThat(links.stream().filter(r -> "AUDIT".equals(r.getValueString("kind"))).count())
|
||||
.isEqualTo(64);
|
||||
assertThat(links.stream().flatMap(r -> AuditStore.detailIds(r).stream()).toList())
|
||||
.hasSize(64)
|
||||
.doesNotHaveDuplicates();
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).isEmpty();
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Even the function owner cannot authorize direct deletes by setting a GUC.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void ownerCannotSpoofPurgeStack() throws Exception
|
||||
{
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
try(var connection = owner())
|
||||
{
|
||||
sql(connection, "ALTER TABLE audit ALTER COLUMN timestamp TYPE timestamp without time zone USING timestamp AT TIME ZONE 'UTC'");
|
||||
sql(connection, "INSERT INTO audit(id,audit_table_id,audit_user_id,timestamp) VALUES"
|
||||
+ "(2,1,1,(now() AT TIME ZONE 'UTC')-interval '6 years'-interval '1 hour')");
|
||||
}
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
try(var connection = owner())
|
||||
{
|
||||
sql(connection, "SELECT set_config('worm_audit.purging','on',false)");
|
||||
for(String table : List.of("audit", "audit_detail", "worm_audit_chain"))
|
||||
{
|
||||
assertThatThrownBy(() -> sql(connection, "DELETE FROM " + table + " WHERE id=1"))
|
||||
.isInstanceOf(SQLException.class).hasMessageContaining("worm_audit:");
|
||||
}
|
||||
assertThat(count(connection, "SELECT count(*) FROM worm_audit_purge")).isZero();
|
||||
// This succeeds only if the PG16 context text matches the stack guard.
|
||||
sql(connection, "SET timezone='America/Chicago'");
|
||||
sql(connection, "SELECT worm_audit_purge(now()-interval '6 years')");
|
||||
assertThat(count(connection, "SELECT count(*) FROM audit")).isZero();
|
||||
assertThat(count(connection, "SELECT count(*) FROM worm_audit_purge WHERE outcome='PURGED'")).isEqualTo(1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A UTC without-tz audit column settles identically in a Chicago JVM/session.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void chainerUsesUtcCutoffInChicago() throws Exception
|
||||
{
|
||||
var original = java.util.TimeZone.getDefault();
|
||||
config.withSettleHorizonSeconds(60);
|
||||
try(var connection = owner())
|
||||
{
|
||||
sql(connection, "ALTER TABLE audit ALTER COLUMN timestamp TYPE timestamp without time zone USING timestamp AT TIME ZONE 'UTC'");
|
||||
sql(connection, "INSERT INTO audit(id,audit_table_id,audit_user_id,timestamp) VALUES"
|
||||
+ "(1,1,1,TIMESTAMP '2026-10-13 12:00:00'),(2,1,1,TIMESTAMP '2026-10-13 12:01:30')");
|
||||
// New runtime connections inherit this non-UTC session setting.
|
||||
sql(connection, "ALTER ROLE app SET timezone='America/Chicago'");
|
||||
java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("America/Chicago"));
|
||||
assertThat(new WormAuditChainer(config).chainOnce(java.time.Instant.parse("2026-10-13T12:02:00Z")).audits()).isEqualTo(1);
|
||||
assertThat(STORE.all(WormAuditTables.CHAIN)).extracting(r -> r.getValueLong("audit_id")).containsExactly(1L);
|
||||
java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("UTC"));
|
||||
assertThat(new WormAuditVerifier(config).verify(java.util.Set.of(1L)).getBreaks()).isEmpty();
|
||||
}
|
||||
finally
|
||||
{
|
||||
java.util.TimeZone.setDefault(original);
|
||||
try(var connection = owner())
|
||||
{
|
||||
sql(connection, "ALTER ROLE app RESET timezone");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** PostgreSQL containment polling includes the lower window boundary only.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void postgresLateDetailWindowAndCompleteness() throws Exception
|
||||
{
|
||||
config.withSettleHorizonSeconds(60);
|
||||
var now = java.time.Instant.now().truncatedTo(java.time.temporal.ChronoUnit.SECONDS);
|
||||
var cutoff = now.minusSeconds(60);
|
||||
try(var connection = owner(); var statement = connection.prepareStatement(
|
||||
"INSERT INTO audit(id,audit_table_id,audit_user_id,timestamp) VALUES(?,1,1,?)"))
|
||||
{
|
||||
for(int id = 1; id <= 4; id++)
|
||||
{
|
||||
long seconds = switch(id)
|
||||
{
|
||||
case 1 -> 86401;
|
||||
case 2 -> 86400;
|
||||
case 3 -> 1;
|
||||
default -> 0;
|
||||
};
|
||||
statement.setInt(1, id);
|
||||
statement.setObject(2, cutoff.minusSeconds(seconds).atOffset(java.time.ZoneOffset.UTC));
|
||||
statement.executeUpdate();
|
||||
}
|
||||
}
|
||||
var writer = new WormAuditChainer(config);
|
||||
assertThat(writer.chainOnce(now).audits()).isEqualTo(3);
|
||||
for(long id = 1; id <= 4; id++)
|
||||
{
|
||||
detail(id, id);
|
||||
}
|
||||
assertThat(writer.chainOnce(now).lateDetails()).isEqualTo(2);
|
||||
assertThat(STORE.all(WormAuditTables.CHAIN).stream().flatMap(r -> AuditStore.detailIds(r).stream()).toList()).containsExactly(2L, 3L);
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).anyMatch(b -> b.reason().contains("audit_detail 1 is not covered"));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.VerificationRuns;
|
||||
import com.kingsrook.qbits.wormaudit.processes.WormAuditProcesses;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertResolution;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertSignal;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertSinkInterface;
|
||||
import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunInput;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunOutput;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.STORE;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.core;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.detail;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.tamper;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for processes and alerts test.
|
||||
*******************************************************************************/
|
||||
class ProcessesAndAlertsTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for sink.
|
||||
*******************************************************************************/
|
||||
public static class Sink implements AlertSinkInterface
|
||||
{
|
||||
static final List<AlertSignal> raised = new ArrayList<>();
|
||||
static final List<AlertResolution> resolved = new ArrayList<>();
|
||||
static final List<String> beats = new ArrayList<>();
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify raise.
|
||||
*******************************************************************************/
|
||||
public void raise(AlertSignal signal)
|
||||
{
|
||||
raised.add(signal);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify resolve.
|
||||
*******************************************************************************/
|
||||
public void resolve(AlertResolution resolution)
|
||||
{
|
||||
resolved.add(resolution);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify heartbeat.
|
||||
*******************************************************************************/
|
||||
public void heartbeat(String key, Instant at)
|
||||
{
|
||||
beats.add(key);
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify daily raise resolve heartbeat and user result.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void dailyRaiseResolveHeartbeatAndUserResult() throws Exception
|
||||
{
|
||||
var config = ChainVerificationTest.signed();
|
||||
var instance = init(config);
|
||||
Sink.raised.clear();
|
||||
Sink.resolved.clear();
|
||||
Sink.beats.clear();
|
||||
AlertsInstanceMetaData.ofOrWithNew(instance).withSink(new QCodeReference(Sink.class));
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
new WormAuditProcesses.ChainTick()
|
||||
.run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
tamper("auditDetail", 1, "newValue", "broken");
|
||||
var daily = new WormAuditProcesses.Daily();
|
||||
var output = new RunBackendStepOutput();
|
||||
daily.run(new RunBackendStepInput(), output);
|
||||
assertThat(output.getValueString("outcome")).isEqualTo("FAIL");
|
||||
assertThat(Sink.raised).hasSize(1);
|
||||
assertThat(Sink.raised.getFirst().getAlertTypeName()).isEqualTo("worm_audit.chain.break");
|
||||
assertThat(Sink.raised.getFirst().getDedupKey()).isEqualTo("chain");
|
||||
assertThat(Sink.raised.getFirst().getRecordRefs().getFirst().getTableName())
|
||||
.isEqualTo(WormAuditTables.VERIFICATION);
|
||||
assertThat(Sink.raised.getFirst().getTitle()).contains("Break at link 1");
|
||||
var first = STORE.all(WormAuditTables.VERIFICATION).getFirst();
|
||||
assertThat(first.getValueLong("first_break_link_id")).isEqualTo(1);
|
||||
assertThat(first.getValueLong("first_break_audit_id")).isEqualTo(1);
|
||||
assertThat(first.getValueString("trigger")).isEqualTo("SCHEDULE");
|
||||
tamper("auditDetail", 1, "newValue", "new");
|
||||
daily.run(new RunBackendStepInput(), new RunBackendStepOutput());
|
||||
assertThat(Sink.resolved).hasSize(1);
|
||||
assertThat(Sink.beats)
|
||||
.containsExactly("worm_audit.verify.daily", "worm_audit.verify.daily");
|
||||
assertThat(STORE.all(WormAuditTables.VERIFICATION)).hasSize(2);
|
||||
assertThat(VerificationRuns.run(null, "USER").passed()).isTrue();
|
||||
assertThat(Sink.beats).hasSize(2);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify review and load across pages including failures and empty scope.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void reviewAndLoadAcrossPagesIncludingFailuresAndEmptyScope() throws Exception
|
||||
{
|
||||
var config = new WormAuditQBitConfig();
|
||||
init(config);
|
||||
audit(1);
|
||||
audit(2);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
var links = STORE.all(WormAuditTables.CHAIN);
|
||||
var review = new WormAuditProcesses.Review();
|
||||
var load = new WormAuditProcesses.Load();
|
||||
var output = new RunBackendStepOutput();
|
||||
assertThat(load.getProcessSummary(output, true)).isEmpty();
|
||||
for(var link : links)
|
||||
{
|
||||
var input = new RunBackendStepInput().withRecords(List.of(link));
|
||||
review.runOnePage(input, output);
|
||||
load.runOnePage(input, output);
|
||||
}
|
||||
assertThat(review.getProcessSummary(output, false).toString())
|
||||
.contains("2 chain links will be verified", "0 checkpoints");
|
||||
load.postRun(
|
||||
new BackendStepPostRunInput(new RunBackendStepInput()),
|
||||
new BackendStepPostRunOutput(output));
|
||||
assertThat(STORE.all(WormAuditTables.VERIFICATION)).hasSize(1);
|
||||
assertThat(load.getProcessSummary(output, true).toString())
|
||||
.contains("2 links verified", "no signing key configured");
|
||||
audit(3);
|
||||
var fullWithEmptyFilter = new WormAuditProcesses.Load();
|
||||
fullWithEmptyFilter.runOnePage(new RunBackendStepInput().withRecords(links), output);
|
||||
fullWithEmptyFilter.postRun(new BackendStepPostRunInput(new RunBackendStepInput().addValue("queryFilterJson", "{}")),
|
||||
new BackendStepPostRunOutput(output));
|
||||
assertThat(fullWithEmptyFilter.getProcessSummary(output, true).toString()).contains("audit 3 is not covered by the chain");
|
||||
tamper("audit", 1, "message", "changed");
|
||||
var selected = new WormAuditProcesses.Load();
|
||||
selected.runOnePage(
|
||||
new RunBackendStepInput().withRecords(List.of(links.getFirst())), output);
|
||||
selected.postRun(
|
||||
new BackendStepPostRunInput(new RunBackendStepInput().addValue("recordIds", "1")),
|
||||
new BackendStepPostRunOutput(output));
|
||||
assertThat(selected.getProcessSummary(output, true).toString())
|
||||
.contains("ERROR", "content differs");
|
||||
init(config);
|
||||
var empty = new WormAuditProcesses.Load();
|
||||
empty.postRun(
|
||||
new BackendStepPostRunInput(new RunBackendStepInput()),
|
||||
new BackendStepPostRunOutput(output));
|
||||
assertThat(empty.getProcessSummary(output, true).toString()).contains("0 links verified");
|
||||
assertThat(new WormAuditProcesses.Review().getProcessSummary(output, false).toString())
|
||||
.contains("links none");
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Full loads retain no id collection; review summarizes disjoint streamed pages once.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void reviewRangesAndSelectionOnlyIds() throws Exception
|
||||
{
|
||||
var config = ChainVerificationTest.signed().withCheckpointEveryLinks(2);
|
||||
init(config);
|
||||
for(long id = 1; id <= 4; id++)
|
||||
{
|
||||
audit(id);
|
||||
}
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
var links = STORE.all(WormAuditTables.CHAIN);
|
||||
var review = new WormAuditProcesses.Review();
|
||||
var full = new WormAuditProcesses.Load();
|
||||
var selected = new WormAuditProcesses.Load();
|
||||
var output = new RunBackendStepOutput();
|
||||
var ids = WormAuditProcesses.Load.class.getDeclaredField("ids");
|
||||
ids.setAccessible(true);
|
||||
for(var link : links)
|
||||
{
|
||||
var input = new RunBackendStepInput().withRecords(List.of(link));
|
||||
review.runOnePage(input, output);
|
||||
full.runOnePage(input, output);
|
||||
assertThat(ids.get(full)).isNull();
|
||||
}
|
||||
String summary = review.getProcessSummary(output, false).toString();
|
||||
assertThat(summary).contains("4 chain links", "links 1–4", "2 checkpoints");
|
||||
// A second call reuses the count; later inserts cannot silently change Review.
|
||||
STORE.insert(WormAuditTables.CHECKPOINT, List.of(new com.kingsrook.qqq.backend.core.model.data.QRecord()
|
||||
.withValue("from_link_id", 1L).withValue("to_link_id", 4L)));
|
||||
assertThat(review.getProcessSummary(output, false).toString()).isEqualTo(summary);
|
||||
selected.runOnePage(new RunBackendStepInput().addValue("recordIds", "2").withRecords(List.of(links.get(1))), output);
|
||||
assertThat(ids.get(selected)).isEqualTo(java.util.Set.of(2L));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Docker dependencies must not enable debug wire logs during tests.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void dockerLoggersAreQuiet()
|
||||
{
|
||||
for(String name : List.of("org.testcontainers", "com.github.dockerjava", "org.apache.hc", "tc"))
|
||||
{
|
||||
assertThat(org.apache.logging.log4j.LogManager.getLogger(name).isDebugEnabled()).isFalse();
|
||||
assertThat(org.apache.logging.log4j.LogManager.getLogger(name).isWarnEnabled()).isTrue();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import java.sql.DriverManager;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.verify.WormAuditVerifier;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.STORE;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.core;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.detail;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.lookups;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for rdbms chain test.
|
||||
*******************************************************************************/
|
||||
class RdbmsChainTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Verify real qqq rdbms actions anti join rollback and tamper.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void realQqqRdbmsActionsAntiJoinRollbackAndTamper() throws Exception
|
||||
{
|
||||
String url =
|
||||
"jdbc:h2:mem:worm_"
|
||||
+ UUID.randomUUID()
|
||||
+ ";DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE";
|
||||
var config = ChainVerificationTest.signed();
|
||||
var instance = RdbmsTestSupport.instance(url, "sa", "", config, false);
|
||||
try(var connection = DriverManager.getConnection(url, "sa", ""))
|
||||
{
|
||||
RdbmsTestSupport.createTables(connection, instance, false);
|
||||
QContext.init(instance, new QSession());
|
||||
lookups();
|
||||
audit(1);
|
||||
detail(1, 1);
|
||||
var writer = new WormAuditChainer(config);
|
||||
assertThat(writer.chainOnce().links()).isEqualTo(1);
|
||||
assertThat(writer.chainOnce().links()).isZero();
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).isEmpty();
|
||||
// Fail signing after append, verifying the real RDBMS transaction rolls back.
|
||||
var pins = config.getTrustedPublicKeys();
|
||||
config.withTrustedPublicKeys(Map.of());
|
||||
audit(2);
|
||||
assertThatThrownBy(writer::chainOnce)
|
||||
.hasMessageContaining("WORM audit operation failed");
|
||||
assertThat(STORE.all(WormAuditTables.CHAIN)).hasSize(1);
|
||||
config.withTrustedPublicKeys(pins);
|
||||
assertThat(writer.chainOnce().links()).isEqualTo(1);
|
||||
assertThat(STORE.all(WormAuditTables.CHAIN))
|
||||
.extracting(r -> r.getValueLong("id"))
|
||||
.containsExactly(1L, 2L);
|
||||
connection
|
||||
.createStatement()
|
||||
.execute("UPDATE audit_detail SET new_value='changed' WHERE id=1");
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks())
|
||||
.anyMatch(b -> b.reason().contains("audit_detail 1"));
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise multiple keyset pages, capped checkpoints and bounded SQL query counts.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void batchesPagesAndCheckpointRanges() throws Exception
|
||||
{
|
||||
String url = "jdbc:h2:mem:pages_" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE";
|
||||
var config = ChainVerificationTest.signed().withBatchSize(1500).withCheckpointEveryLinks(400).withCheckpointEveryMinutes(1);
|
||||
var instance = RdbmsTestSupport.instance(url, "sa", "", config, false);
|
||||
try(var connection = DriverManager.getConnection(url, "sa", ""); var statement = connection.createStatement())
|
||||
{
|
||||
RdbmsTestSupport.createTables(connection, instance, false);
|
||||
QContext.init(instance, new QSession());
|
||||
lookups();
|
||||
statement.execute("INSERT INTO audit(id,audit_table_id,audit_user_id,record_id,timestamp) SELECT x,1,1,17,CURRENT_TIMESTAMP - INTERVAL '2' MINUTE FROM SYSTEM_RANGE(1,1005) series(x)");
|
||||
statement.execute("INSERT INTO audit_detail(id,audit_id,field_name,new_value) SELECT x,x,'title','new' FROM SYSTEM_RANGE(1,1005) series(x)");
|
||||
statement.execute("SET QUERY_STATISTICS TRUE");
|
||||
var writer = new WormAuditChainer(config);
|
||||
var now = java.time.Instant.now();
|
||||
var counts = writer.chainOnce(now);
|
||||
assertThat(counts.links()).isEqualTo(1005);
|
||||
assertThat(counts.checkpoints()).isEqualTo(2);
|
||||
assertSelectCount(connection, 100);
|
||||
assertThat(STORE.last(WormAuditTables.CHECKPOINT).getValueInstant("created_at")).isCloseTo(now, org.assertj.core.api.Assertions.within(1, java.time.temporal.ChronoUnit.SECONDS));
|
||||
statement.execute("SET QUERY_STATISTICS FALSE");
|
||||
statement.execute("SET QUERY_STATISTICS TRUE");
|
||||
assertThat(writer.chainOnce(now.plusSeconds(61)).checkpoints()).isEqualTo(1);
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).isEmpty();
|
||||
assertSelectCount(connection, 100);
|
||||
assertThat(STORE.all(WormAuditTables.CHECKPOINT)).extracting(r -> r.getValueLong("link_count")).containsExactly(400L, 400L, 205L);
|
||||
statement.execute("UPDATE audit_detail SET new_value='changed' WHERE id=1001");
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).anyMatch(b -> b.reason().contains("audit_detail 1,001"));
|
||||
statement.execute("DELETE FROM worm_audit_chain WHERE id=1000");
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).anyMatch(b -> b.reason().contains("gap"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Assert actual query executions stay proportional to batches, not row count.
|
||||
*******************************************************************************/
|
||||
private void assertSelectCount(java.sql.Connection connection, int limit) throws Exception
|
||||
{
|
||||
try(var statement = connection.createStatement(); var rows = statement.executeQuery(
|
||||
"SELECT COALESCE(SUM(EXECUTION_COUNT),0) FROM INFORMATION_SCHEMA.QUERY_STATISTICS WHERE UPPER(SQL_STATEMENT) LIKE 'SELECT%'"))
|
||||
{
|
||||
rows.next();
|
||||
assertThat(rows.getLong(1)).isLessThan(limit);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Poll only the lookback window and report older omissions with exact bounded ids.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void lateWindowAndCompletenessOffenderLimit() throws Exception
|
||||
{
|
||||
String url = "jdbc:h2:mem:window_" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE";
|
||||
var config = new WormAuditQBitConfig().withBatchSize(500);
|
||||
var instance = RdbmsTestSupport.instance(url, "sa", "", config, false);
|
||||
try(var connection = DriverManager.getConnection(url, "sa", ""); var statement = connection.createStatement())
|
||||
{
|
||||
RdbmsTestSupport.createTables(connection, instance, false);
|
||||
QContext.init(instance, new QSession());
|
||||
lookups();
|
||||
audit(1);
|
||||
statement.execute("INSERT INTO audit(id,audit_table_id,audit_user_id,timestamp) VALUES(2,1,1,CURRENT_TIMESTAMP - INTERVAL '2' MINUTE)");
|
||||
var writer = new WormAuditChainer(config);
|
||||
writer.chainOnce();
|
||||
detail(1, 1);
|
||||
detail(2, 2);
|
||||
assertThat(writer.chainOnce().lateDetails()).isEqualTo(1);
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).anyMatch(b -> b.reason().contains("audit_detail 1 is not covered"));
|
||||
statement.execute("INSERT INTO worm_audit_chain SELECT 99,kind,audit_id,detail_ids,prev_hash,link_hash,chained_at FROM worm_audit_chain WHERE id=3");
|
||||
assertThat(new WormAuditVerifier(config).verify().getBreaks()).anyMatch(b -> b.reason().contains("audit_detail 2 is covered twice"));
|
||||
statement.execute("DELETE FROM worm_audit_chain WHERE id=99");
|
||||
statement.execute("INSERT INTO audit_detail(id,audit_id) SELECT x,1 FROM SYSTEM_RANGE(3,107) series(x)");
|
||||
var result = new WormAuditVerifier(config).verify();
|
||||
assertThat(result.getBreakCount()).isEqualTo(106);
|
||||
assertThat(result.getBreaks()).hasSize(100);
|
||||
assertThat(result.getBreaks().getFirst().reason()).contains("audit_detail 1 is not covered");
|
||||
assertThat(result.getBreaks().getLast().reason()).contains("audit_detail 101 is not covered");
|
||||
assertThat(result.getWarnings()).anyMatch(w -> w.contains("106 coverage offenders"));
|
||||
statement.execute("INSERT INTO audit(id,timestamp) SELECT x,CURRENT_TIMESTAMP - INTERVAL '2' MINUTE FROM SYSTEM_RANGE(2000,2104) series(x)");
|
||||
result = new WormAuditVerifier(config).verify();
|
||||
assertThat(result.getBreakCount()).isEqualTo(211);
|
||||
assertThat(result.getBreaks()).hasSize(200);
|
||||
assertThat(result.getBreaks().getFirst().reason()).contains("audit 2,000 is not covered");
|
||||
assertThat(result.getBreaks().get(99).reason()).contains("audit 2,099 is not covered");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Cutoffs are UTC local values for timestamp columns, independent of JVM zone.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void withoutTimezoneCutoffInChicago() throws Exception
|
||||
{
|
||||
var original = java.util.TimeZone.getDefault();
|
||||
String url = "jdbc:h2:mem:timezone_" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE";
|
||||
var config = new WormAuditQBitConfig();
|
||||
var instance = RdbmsTestSupport.instance(url, "sa", "", config, false);
|
||||
try(var connection = DriverManager.getConnection(url, "sa", ""); var statement = connection.createStatement())
|
||||
{
|
||||
RdbmsTestSupport.createTables(connection, instance, false);
|
||||
statement.execute("ALTER TABLE audit ALTER COLUMN timestamp timestamp without time zone");
|
||||
QContext.init(instance, new QSession());
|
||||
lookups();
|
||||
statement.execute("INSERT INTO audit(id,audit_table_id,audit_user_id,timestamp) VALUES(1,1,1,TIMESTAMP '2026-10-13 12:00:00'),(2,1,1,TIMESTAMP '2026-10-13 12:01:30')");
|
||||
java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("America/Chicago"));
|
||||
assertThat(new WormAuditChainer(config).chainOnce(java.time.Instant.parse("2026-10-13T12:02:00Z")).audits()).isEqualTo(1);
|
||||
assertThat(STORE.all(WormAuditTables.CHAIN)).extracting(r -> r.getValueLong("audit_id")).containsExactly(1L);
|
||||
java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("UTC"));
|
||||
assertThat(new WormAuditVerifier(config).verify(java.util.Set.of(1L)).getBreaks()).isEmpty();
|
||||
}
|
||||
finally
|
||||
{
|
||||
java.util.TimeZone.setDefault(original);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.jdbc.SimpleConnectionProvider;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for rdbms test support.
|
||||
*******************************************************************************/
|
||||
class RdbmsTestSupport
|
||||
{
|
||||
// RC.5 supplies the RDBMS actions and a pluggable strategy. Hosts supply their
|
||||
// PostgreSQL quoting strategy; this fixture uses the core actions unchanged.
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for postgres strategy.
|
||||
*******************************************************************************/
|
||||
public static class PostgresStrategy extends BaseRDBMSActionStrategy
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Verify get identifier quote string.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getIdentifierQuoteString()
|
||||
{
|
||||
return ("\"");
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify get insert default values clause.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getInsertDefaultValuesClause()
|
||||
{
|
||||
return ("DEFAULT VALUES");
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify instance.
|
||||
*******************************************************************************/
|
||||
static QInstance instance(
|
||||
String url, String user, String password, WormAuditQBitConfig config, boolean postgres)
|
||||
throws Exception
|
||||
{
|
||||
com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager.resetConnectionProviders();
|
||||
var backend =
|
||||
new RDBMSBackendMetaData()
|
||||
.withName("db")
|
||||
.withVendor(postgres ? "postgresql" : "h2")
|
||||
.withJdbcUrl(url)
|
||||
.withJdbcDriverClassName(
|
||||
postgres ? "org.postgresql.Driver" : "org.h2.Driver")
|
||||
.withUsername(user)
|
||||
.withPassword(password)
|
||||
.withConnectionProvider(new QCodeReference(SimpleConnectionProvider.class));
|
||||
if(postgres)
|
||||
{
|
||||
backend.withActionStrategyCodeReference(new QCodeReference(PostgresStrategy.class));
|
||||
}
|
||||
QInstance instance = WormAuditTestSupport.core(backend);
|
||||
for(var table : instance.getTables().values())
|
||||
{
|
||||
table.withBackendDetails(
|
||||
new RDBMSTableBackendDetails().withTableName(snake(table.getName())));
|
||||
table.getFields().values().forEach(f -> f.withBackendName(snake(f.getName())));
|
||||
}
|
||||
new WormAuditQBitProducer().withQBitConfig(config.withBackendName("db")).produce(instance);
|
||||
return (instance);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify snake.
|
||||
*******************************************************************************/
|
||||
static String snake(String value)
|
||||
{
|
||||
return (value.replaceAll("([a-z])([A-Z])", "$1_$2").toLowerCase(Locale.ROOT));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify create tables.
|
||||
*******************************************************************************/
|
||||
static void createTables(Connection connection, QInstance instance, boolean coreOnly)
|
||||
throws Exception
|
||||
{
|
||||
for(var table : instance.getTables().values())
|
||||
{
|
||||
if(coreOnly && table.getName().startsWith("worm_audit_"))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
List<String> columns = new ArrayList<>();
|
||||
for(var field : table.getFields().values())
|
||||
{
|
||||
String type =
|
||||
switch(field.getType())
|
||||
{
|
||||
case INTEGER -> "integer";
|
||||
case LONG -> "bigint";
|
||||
case DATE_TIME -> "timestamp(6) with time zone";
|
||||
default -> "text";
|
||||
};
|
||||
if(field.getName().equals("id"))
|
||||
{
|
||||
type +=
|
||||
table.getName().equals("worm_audit_chain")
|
||||
? " PRIMARY KEY"
|
||||
: " GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY";
|
||||
}
|
||||
columns.add(
|
||||
WormAuditDdl.identifier(
|
||||
field.getBackendName() == null
|
||||
? field.getName()
|
||||
: field.getBackendName())
|
||||
+ " "
|
||||
+ type);
|
||||
}
|
||||
String name = ((RDBMSTableBackendDetails) table.getBackendDetails()).getTableName();
|
||||
try(var statement = connection.createStatement())
|
||||
{
|
||||
statement.execute(
|
||||
"CREATE TABLE "
|
||||
+ WormAuditDdl.identifier(name)
|
||||
+ " ("
|
||||
+ String.join(",", columns)
|
||||
+ ")");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.wormaudit.chain.WormAuditChainer;
|
||||
import com.kingsrook.qbits.wormaudit.model.LinkRangeWidget;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.VerificationRuns;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeEvent;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeListenerInterface;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeType;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
|
||||
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChildRecordListData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.audit;
|
||||
import static com.kingsrook.qbits.wormaudit.WormAuditTestSupport.init;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for record integration test.
|
||||
*******************************************************************************/
|
||||
class RecordIntegrationTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for event sink.
|
||||
*******************************************************************************/
|
||||
public static class EventSink implements RecordChangeListenerInterface
|
||||
{
|
||||
static List<String> tables = new ArrayList<>();
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify applies to.
|
||||
*******************************************************************************/
|
||||
public boolean appliesTo(String table, RecordChangeType type)
|
||||
{
|
||||
return (List.of(WormAuditTables.VERIFICATION, WormAuditTables.EXPORT).contains(table)
|
||||
&& type == RecordChangeType.INSERT);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify on records changed.
|
||||
*******************************************************************************/
|
||||
public void onRecordsChanged(RecordChangeEvent event)
|
||||
{
|
||||
tables.add(event.getTableName());
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify checkpoint child list uses range and read permissions.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void checkpointChildListUsesRangeAndReadPermissions() throws Exception
|
||||
{
|
||||
var config = ChainVerificationTest.signed().withCheckpointEveryLinks(2);
|
||||
var instance = init(config);
|
||||
audit(1);
|
||||
audit(2);
|
||||
new WormAuditChainer(config).chainOnce();
|
||||
var input =
|
||||
new RenderWidgetInput()
|
||||
.withWidgetMetaData(
|
||||
instance.getWidget(WormAuditTables.CHECKPOINT + "_links"));
|
||||
input.setQueryParams(Map.of("id", "1"));
|
||||
assertThatThrownBy(() -> new LinkRangeWidget().render(input)).isInstanceOf(Exception.class);
|
||||
QContext.getQSession()
|
||||
.withPermissions("worm_audit_checkpoint.read", "worm_audit_chain.read");
|
||||
var data = (ChildRecordListData) new LinkRangeWidget().render(input).getWidgetData();
|
||||
assertThat(data.getTotalRows()).isEqualTo(2);
|
||||
assertThat(data.getQueryOutput().getRecords()).hasSize(2);
|
||||
assertThat(data.getViewAllLink()).contains("filter=");
|
||||
input.setQueryParams(Map.of("id", "999"));
|
||||
assertThatThrownBy(() -> new LinkRangeWidget().render(input))
|
||||
.hasMessageContaining("was not found");
|
||||
assertThat(
|
||||
instance.getTable(WormAuditTables.EXPORT)
|
||||
.getField("from_checkpoint_id")
|
||||
.getPossibleValueSourceName())
|
||||
.isEqualTo(WormAuditTables.CHECKPOINT);
|
||||
assertThat(
|
||||
instance.getTable(WormAuditTables.VERIFICATION)
|
||||
.getField("export_id")
|
||||
.getPossibleValueSourceName())
|
||||
.isEqualTo(WormAuditTables.EXPORT);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify verification inserts reach core event spi for host esb publications.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void verificationInsertsReachCoreEventSpiForHostEsbPublications() throws Exception
|
||||
{
|
||||
var instance = init(new WormAuditQBitConfig());
|
||||
EventSink.tables.clear();
|
||||
instance.withRecordChangeListener(new QCodeReference(EventSink.class));
|
||||
audit(1);
|
||||
new WormAuditChainer(VerificationRuns.config()).chainOnce();
|
||||
VerificationRuns.run(null, "USER");
|
||||
assertThat(EventSink.tables).containsExactly(WormAuditTables.VERIFICATION);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import org.junit.jupiter.api.extension.ConditionEvaluationResult;
|
||||
import org.junit.jupiter.api.extension.ExecutionCondition;
|
||||
import org.junit.jupiter.api.extension.ExtensionContext;
|
||||
import org.testcontainers.DockerClientFactory;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for requires docker condition.
|
||||
*******************************************************************************/
|
||||
public class RequiresDockerCondition implements ExecutionCondition
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Verify evaluate execution condition.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context)
|
||||
{
|
||||
boolean available;
|
||||
try
|
||||
{
|
||||
available = DockerClientFactory.instance().isDockerAvailable();
|
||||
}
|
||||
catch(Exception | LinkageError e)
|
||||
{
|
||||
available = false;
|
||||
}
|
||||
return (evaluate(available, System.getenv("CI")));
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify evaluate.
|
||||
*******************************************************************************/
|
||||
static ConditionEvaluationResult evaluate(boolean available, String ci)
|
||||
{
|
||||
if(available)
|
||||
{
|
||||
return (ConditionEvaluationResult.enabled("Docker is available"));
|
||||
}
|
||||
if(ci != null)
|
||||
{
|
||||
throw new IllegalStateException("Docker is required when CI is set");
|
||||
}
|
||||
return (ConditionEvaluationResult.disabled(
|
||||
"Docker unavailable; coordinator must run PostgreSQL acceptance tests"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.wormaudit.ddl.WormAuditDdl;
|
||||
import com.kingsrook.qbits.wormaudit.model.WormAuditTables;
|
||||
import com.kingsrook.qbits.wormaudit.processes.WormAuditProcesses;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
/*******************************************************************************
|
||||
** Tests and fixtures for worm audit qbit producer test.
|
||||
*******************************************************************************/
|
||||
class WormAuditQBitProducerTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Verify metadata and defaults.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void metadataAndDefaults() throws Exception
|
||||
{
|
||||
var config = new WormAuditQBitConfig();
|
||||
QInstance instance = WormAuditTestSupport.init(config);
|
||||
assertThat(config.getRetentionYears()).isEqualTo(6);
|
||||
assertThat(config.getTableMetaDataCustomizer()).isNull();
|
||||
assertThat(config.getSigningKey()).isNull();
|
||||
assertThat(config.getTrustedPublicKeys()).isEmpty();
|
||||
assertThat(config.getChainIntervalSeconds()).isEqualTo(30);
|
||||
assertThat(config.getSettleHorizonSeconds()).isEqualTo(60);
|
||||
assertThat(config.getCheckpointEveryLinks()).isEqualTo(10000);
|
||||
assertThat(config.getCheckpointEveryMinutes()).isEqualTo(60);
|
||||
assertThat(config.getBatchSize()).isEqualTo(1000);
|
||||
assertThat(config.getLateDetailLookbackHours()).isEqualTo(24);
|
||||
assertThat(config.getVerifyCron()).isEqualTo("0 0 3 * * ?");
|
||||
assertThat(config.getAppRole()).isEqualTo("app");
|
||||
assertThat(instance.getQBits().get("com.kingsrook.qbits:qbit-worm-audit").getConfig())
|
||||
.isSameAs(config);
|
||||
assertThat(instance.getApp("worm_audit").getSections())
|
||||
.extracting(s -> s.getName())
|
||||
.containsExactlyElementsOf(WormAuditTables.NAV);
|
||||
assertThat(instance.getApp("worm_audit").getChildren())
|
||||
.allMatch(
|
||||
c ->
|
||||
c.getName().startsWith("worm_audit_")
|
||||
&& !c.getName().contains("verify"));
|
||||
assertThat(instance.getProcess(WormAuditProcesses.VERIFY).getTableName())
|
||||
.isEqualTo(WormAuditTables.CHAIN);
|
||||
assertThat(instance.getProcess(WormAuditProcesses.CHAIN).getSchedule().getRepeatSeconds())
|
||||
.isEqualTo(30);
|
||||
assertThat(instance.getProcess(WormAuditProcesses.DAILY).getSchedule().getCronExpression())
|
||||
.isEqualTo(config.getVerifyCron());
|
||||
for(String name :
|
||||
List.of(
|
||||
"audit",
|
||||
"auditDetail",
|
||||
"auditUser",
|
||||
"auditTable",
|
||||
WormAuditTables.CHAIN,
|
||||
WormAuditTables.CHECKPOINT,
|
||||
WormAuditTables.EXPORT,
|
||||
WormAuditTables.VERIFICATION,
|
||||
WormAuditTables.PURGE))
|
||||
{
|
||||
var table = instance.getTable(name);
|
||||
assertThat(
|
||||
table.isCapabilityEnabled(
|
||||
instance.getBackendForTable(name), Capability.TABLE_UPDATE))
|
||||
.isFalse();
|
||||
assertThat(
|
||||
table.isCapabilityEnabled(
|
||||
instance.getBackendForTable(name), Capability.TABLE_DELETE))
|
||||
.isFalse();
|
||||
if(name.startsWith("worm_audit_"))
|
||||
{
|
||||
assertThat(table.getPermissionRules().getPermissionBaseName()).isEqualTo(name);
|
||||
assertThat(
|
||||
table.isCapabilityEnabled(
|
||||
instance.getBackendForTable(name), Capability.TABLE_INSERT))
|
||||
.isFalse();
|
||||
assertThat(table.getSections()).isNotEmpty();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify customizations and validation.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void customizationsAndValidation() throws Exception
|
||||
{
|
||||
assertThat(new WormAuditQBitProducer().isEnabled()).isFalse();
|
||||
var config =
|
||||
new WormAuditQBitConfig()
|
||||
.withRetentionYears(7)
|
||||
.withChainIntervalSeconds(45)
|
||||
.withSettleHorizonSeconds(0)
|
||||
.withBatchSize(13)
|
||||
.withCheckpointEveryLinks(5)
|
||||
.withCheckpointEveryMinutes(2)
|
||||
.withVerifyCron("0 1 3 * * ?")
|
||||
.withAppRole("runtime")
|
||||
.withSchedulerName("jobs")
|
||||
.withCoreAuditTableNames(java.util.Map.of("audit", "custom_audit"))
|
||||
.withTableMetaDataCustomizer(
|
||||
(instance, table) ->
|
||||
table.withLabel("custom")
|
||||
.withCapabilities(Capability.TABLE_UPDATE));
|
||||
var instance =
|
||||
WormAuditTestSupport.core(
|
||||
new com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData()
|
||||
.withName("memory")
|
||||
.withBackendType(
|
||||
com.kingsrook.qqq.backend.core.modules.backend
|
||||
.implementations.memory.MemoryBackendModule.class));
|
||||
var producer =
|
||||
new WormAuditQBitProducer()
|
||||
.withNamespace("test")
|
||||
.withQBitConfig(config.withBackendName("memory"));
|
||||
producer.produce(instance);
|
||||
assertThat(producer.getNamespace()).isEqualTo("test");
|
||||
assertThat(producer.getQBitConfig()).isSameAs(config);
|
||||
assertThat(producer.isEnabled()).isTrue();
|
||||
assertThat(config.getCoreAuditTableNames()).containsEntry("audit", "custom_audit");
|
||||
assertThat(instance.getTable(WormAuditTables.CHAIN).getLabel()).isEqualTo("custom");
|
||||
assertThat(
|
||||
instance.getTable(WormAuditTables.CHAIN)
|
||||
.isCapabilityEnabled(
|
||||
instance.getBackend("memory"), Capability.TABLE_UPDATE))
|
||||
.isFalse();
|
||||
assertThatThrownBy(() -> new WormAuditQBitConfig().validate())
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
assertThat(config.withLateDetailLookbackHours(48).getLateDetailLookbackHours()).isEqualTo(48);
|
||||
assertThatThrownBy(() -> config.withLateDetailLookbackHours(0).validate()).isInstanceOf(IllegalArgumentException.class);
|
||||
config.withLateDetailLookbackHours(24);
|
||||
assertThatThrownBy(() -> config.withRetentionYears(0).validate())
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatThrownBy(
|
||||
() -> WormAuditDdl.resolveNames(new WormAuditQBitConfig(), new QInstance()))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.wormaudit;
|
||||
|
||||
import com.kingsrook.qbits.wormaudit.chain.AuditStore;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.audits.AuditsMetaDataProvider;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
|
||||
class WormAuditTestSupport {
|
||||
static final Instant OLD = Instant.parse("2020-01-02T03:04:05.123456Z");
|
||||
static final AuditStore STORE = new AuditStore(null);
|
||||
|
||||
static QInstance init(WormAuditQBitConfig config) throws Exception {
|
||||
MemoryRecordStore.fullReset();
|
||||
QInstance instance =
|
||||
core(
|
||||
new QBackendMetaData()
|
||||
.withName("memory")
|
||||
.withBackendType(MemoryBackendModule.class));
|
||||
new WormAuditQBitProducer()
|
||||
.withQBitConfig(config.withBackendName("memory"))
|
||||
.produce(instance);
|
||||
QContext.init(instance, new QSession());
|
||||
lookups();
|
||||
return (instance);
|
||||
}
|
||||
|
||||
static QInstance core(QBackendMetaData backend) throws Exception {
|
||||
QInstance instance = new QInstance();
|
||||
instance.withInstanceDefaultAuthentication(
|
||||
new com.kingsrook.qqq.backend.core.model.metadata.authentication
|
||||
.QAuthenticationMetaData()
|
||||
.withType(
|
||||
com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType
|
||||
.FULLY_ANONYMOUS)
|
||||
.withName("anon"));
|
||||
instance.addBackend(backend);
|
||||
new AuditsMetaDataProvider().defineAll(instance, backend.getName(), null);
|
||||
instance.getTable("audit")
|
||||
.withField(new QFieldMetaData("clientChannel", QFieldType.STRING));
|
||||
return (instance);
|
||||
}
|
||||
|
||||
static void lookups() throws Exception {
|
||||
STORE.insert(
|
||||
"auditTable",
|
||||
List.of(
|
||||
new QRecord()
|
||||
.withValue("id", 1)
|
||||
.withValue("name", "cms_page")
|
||||
.withValue("label", "Page")));
|
||||
STORE.insert(
|
||||
"auditUser", List.of(new QRecord().withValue("id", 1).withValue("name", "james")));
|
||||
}
|
||||
|
||||
static QRecord audit(long id) throws Exception {
|
||||
QRecord row =
|
||||
new QRecord()
|
||||
.withValue("id", id)
|
||||
.withValue("auditTableId", 1)
|
||||
.withValue("auditUserId", 1)
|
||||
.withValue("recordId", 17)
|
||||
.withValue("message", "Record was Edited")
|
||||
.withValue("timestamp", OLD.plusSeconds(id));
|
||||
return (STORE.insert("audit", List.of(row)).getFirst());
|
||||
}
|
||||
|
||||
static QRecord detail(long id, long auditId) throws Exception {
|
||||
return (STORE.insert(
|
||||
"auditDetail",
|
||||
List.of(
|
||||
new QRecord()
|
||||
.withValue("id", id)
|
||||
.withValue("auditId", auditId)
|
||||
.withValue("fieldName", "title")
|
||||
.withValue("message", "Changed Title")
|
||||
.withValue("oldValue", "old")
|
||||
.withValue("newValue", "new")))
|
||||
.getFirst());
|
||||
}
|
||||
|
||||
static void tamper(String table, long id, String field, java.io.Serializable value)
|
||||
throws Exception {
|
||||
MemoryRecordStore.getInstance()
|
||||
.update(
|
||||
new UpdateInput(table)
|
||||
.withRecords(
|
||||
List.of(
|
||||
new QRecord()
|
||||
.withValue(
|
||||
"id",
|
||||
table.equals("auditTable")
|
||||
|| table.equals(
|
||||
"auditUser")
|
||||
? (java.io.Serializable)
|
||||
(int) id
|
||||
: id)
|
||||
.withValue(field, value))),
|
||||
false);
|
||||
}
|
||||
|
||||
static void remove(String table, long id) throws Exception {
|
||||
MemoryRecordStore.getInstance().delete(new DeleteInput(table).withPrimaryKeys(List.of(id)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Configuration status="WARN">
|
||||
<Appenders>
|
||||
<Console name="Console" target="SYSTEM_OUT">
|
||||
<!-- Core can emit diagnostic events at OFF; exclude those as well. -->
|
||||
<LevelRangeFilter minLevel="FATAL" maxLevel="WARN" onMatch="NEUTRAL" onMismatch="DENY"/>
|
||||
<PatternLayout pattern="%d{HH:mm:ss} %-5level %logger{36} - %msg%n"/>
|
||||
</Console>
|
||||
</Appenders>
|
||||
<Loggers>
|
||||
<Logger name="org.testcontainers" level="WARN"/>
|
||||
<Logger name="com.github.dockerjava" level="WARN"/>
|
||||
<Logger name="org.apache.hc" level="WARN"/>
|
||||
<Logger name="tc" level="WARN"/>
|
||||
<Root level="WARN"><AppenderRef ref="Console"/></Root>
|
||||
</Loggers>
|
||||
</Configuration>
|
||||
Reference in new issue
Block a user