Compare commits

..
Author SHA1 Message Date
forge-recovery 61109316d5 ci: enroll shared Gitea lifecycle
QRun-IO/ci-policy: ci / fork-guard (push) Skipped
QRun-IO/ci-policy: ci / head-profile (push) Skipped
QRun-IO/ci-policy: ci / application-template-0 (push) Skipped
QRun-IO/ci-policy: ci / gradle-1 (push) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (push) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (push) Skipped
QRun-IO/ci-policy: ci / node-8 (push) Skipped
QRun-IO/ci-policy: ci / node-9 (push) Skipped
QRun-IO/ci-policy: ci / node-10 (push) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (push) Skipped
QRun-IO/ci-policy: ci / source-config-13 (push) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (push) Skipped
QRun-IO/ci-policy: ci / source-config-14 (push) Skipped
QRun-IO/ci-policy: ci / android-0 (push) Skipped
QRun-IO/ci-policy: ci / original-mcp (push) Skipped
QRun-IO/ci-policy: ci / not-enrolled (push) Skipped
QRun-IO/ci-policy: native / fork-deny (push) Skipped
QRun-IO/ci-policy: native / dotnet-0 (push) Skipped
Shared Java Maven CI with Docker / secrets (push) Successful in 23s
QRun-IO/ci-policy: native / native-result (push) Successful in 10s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (push) Successful in 25s
QRun-IO/ci-policy: ci / repository-policy (push) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (push) Skipped
Shared Java Maven CI with Docker / verify (push) Successful in 1m8s
Shared Java Maven CI with Docker / gate (push) Successful in 13s
qrun-release / release (push) Successful in 11m38s
qrun-release / postpublish (push) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (push) Successful in 53s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (push) Successful in 16s
QRun-IO/ci-policy: ci / java-docker-3 (push) Successful in 12m11s
QRun-IO/ci-policy: ci / ci-result (push) Successful in 5s
2026-11-02 22:10:19 +00:00
495bbb000c ci: add the isolated postpublish job to the QRun release caller
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: ci / head-profile (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-8 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-9 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-10 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-13 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-14 (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: native / fork-deny (pull_request) Skipped
QRun-IO/ci-policy: native / dotnet-0 (pull_request) Skipped
Shared Java Maven CI with Docker / secrets (push) Successful in 14s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (pull_request) Successful in 14s
QRun-IO/ci-policy: ci / repository-policy (pull_request) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (pull_request) Skipped
QRun-IO/ci-policy: native / native-result (pull_request) Successful in 4s
Shared Java Maven CI with Docker / verify (push) Successful in 50s
Shared Java Maven CI with Docker / gate (push) Successful in 10s
qrun-release / release (push) Successful in 2h27m55s
qrun-release / postpublish (push) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (pull_request) Successful in 46s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (pull_request) Successful in 11s
QRun-IO/ci-policy: ci / java-docker-3 (pull_request) Successful in 2h28m35s
QRun-IO/ci-policy: ci / ci-result (pull_request) Successful in 5s
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Co-Authored-By: Codex <noreply@openai.com>
2026-11-02 15:45:59 +00:00
bc5acbecb7 ci: move the QRun release caller to the thin v2 contract
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-3 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-6 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-7 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-8 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-9 (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: native / fork-deny (pull_request) Skipped
QRun-IO/ci-policy: native / dotnet-0 (pull_request) Skipped
Shared Java Maven CI with Docker / secrets (push) Successful in 18s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (pull_request) Successful in 18s
QRun-IO/ci-policy: native / native-result (pull_request) Successful in 6s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (pull_request) Canceled after 0s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (pull_request) Canceled after 0s
QRun-IO/ci-policy: ci / java-docker-2 (pull_request) Canceled after 1h24m53s
Shared Java Maven CI with Docker / gate (push) Canceled after 0s
Shared Java Maven CI with Docker / verify (push) Canceled after 0s
qrun-release / release (push) Canceled after 1h24m56s
QRun-IO/ci-policy: ci / ci-result (pull_request) Failing after 4s
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Co-Authored-By: Codex <noreply@openai.com>
2026-11-02 06:16:12 +00:00
forge-recoveryandCodex 45fc6ef44b ci: add the identical QRun release caller
QRun-IO/ci-policy: ci / gradle-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-2 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-3 (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: ci / node-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-5 (pull_request) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (pull_request) Successful in 19s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (pull_request) Successful in 53s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (pull_request) Successful in 10s
QRun-IO/ci-policy: ci / java-docker-1 (pull_request) Successful in 12m31s
QRun-IO/ci-policy: ci / ci-result (pull_request) Successful in 5s
Co-Authored-By: Codex <noreply@openai.com>
2026-11-02 03:25:41 +00:00
james.maesandCodex bdabdbebf3 ci: enroll in the shared QQQ orb (#46)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 19:50:07 -05:00
james.maesandCodex 9d11a7356d ci: use shared orb pipelines and source scanning (#41)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 10:43:29 -05:00
202 changed files with 78 additions and 30498 deletions

No files matched your search

+11 -57
View File
@@ -1,60 +1,14 @@
version: 2.1
setup: true
orbs:
qqq-orb: kingsrook/qqq-orb@0.6.8
commands:
configure-package-install:
steps:
- run:
name: Keep package installation unattended
command: |
# These build machines are discarded after the job.
sudo install -d /etc/needrestart/conf.d
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
qqq-orb: kingsrook/qqq-orb@0.8
workflows:
test_only:
jobs:
- qqq-orb/mvn_test_only:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
filters:
branches:
ignore: /(develop|release\/.*)/
publish_snapshot:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: snapshot
filters:
branches:
only: [develop]
publish_release_candidate:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: release_candidate
filters:
branches:
only: [/release\/.*/]
publish_release:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: release
filters:
branches:
ignore: /.*/
tags:
only: [/^v\d+\.\d+\.\d+$/]
setup:
jobs:
- qqq-orb/generate_pipeline:
orb_version: '0.8'
filters:
branches:
ignore: automation/qbit-rc-pending
tags:
only: /v.*/
+25
View File
@@ -0,0 +1,25 @@
# Standard QRun release caller (v2, thin contract): every push and manual dispatch delegates to the protected qrun-release-v1 channel. Do not edit per repository.
name: qrun-release
on:
push: {}
workflow_dispatch: {}
permissions:
contents: read
actions: read
jobs:
release:
uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/java-docker.yaml@qrun-release-v1
with:
runner: linux-small
docker-runner: container-build
browsers: true
browser-runner: browser-build
postpublish:
if: ${{ needs.release.outputs.capture == 'true' }}
needs: release
permissions:
contents: write
actions: write
uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/qrun-source-version.yaml@qrun-release-v1
with:
runner: linux-small
+23
View File
@@ -0,0 +1,23 @@
name: CodeQL
'on':
push:
branches:
- main
- develop
- release/**
- hotfix/**
pull_request: {}
schedule:
- cron: 41 16 * * 0
workflow_dispatch: {}
permissions:
contents: read
actions: read
security-events: write
packages: read
concurrency:
group: codeql-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
analyze:
uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22
-25
View File
@@ -1,25 +0,0 @@
# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
# the release line and the release/* push runs CircleCI publish_release_candidate.
name: Release candidate
on:
workflow_dispatch:
inputs:
dry_run:
description: Print the plan without merging or creating branches
type: boolean
default: false
pull_request:
types: [closed]
branches: [develop]
permissions:
contents: write
jobs:
release-rc:
if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
with:
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
+16
View File
@@ -0,0 +1,16 @@
profile: qbit
orb_version: '0.8'
default_branch: develop
java_version: '21'
browsers: true
api_checks: false
docs_only: false
release_on_main: false
release_tag_policy: v-prefix
test_branch_policy: non-release-no-hotfix
publish:
snapshot: true
release_candidate: true
release: true
hotfix: true
asciidoc: false
-7
View File
@@ -1,7 +0,0 @@
# Backups home and record screenshots
Taken 2026-10-12 against `BackupsDemoServer` (src/test/.../demo, memory backend, real engine history) and the Next UI 1.0.0-RC.14 jar.
Light and dark at 390, 600, 768, 1024, 1440, 1920 px (home) and 390, 1440 px (records).
- `.` : this branch (foundation base): home, plan, run, failed run, lists.
- `with-restore-verify/` : a scratch merge of PRs #5 (verify), #6 (restore), #7 (prune/schedule) onto this branch, so verifications, a restore run and retention classes exist. Not part of this PR.
Binary file not shown.

Before

Width:  |  Height:  |  Size: 163 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 160 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 165 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 138 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 143 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 167 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 163 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 161 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 166 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 138 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 144 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 168 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 170 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 55 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 146 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 55 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 147 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 54 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 171 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 54 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 58 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 45 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 58 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 44 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 161 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 56 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 140 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 58 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 141 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 57 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 163 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 55 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 134 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 221 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 135 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 216 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 171 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 166 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 169 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 144 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 151 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 176 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 172 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 167 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 170 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 144 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 151 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 177 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 177 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 61 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 178 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 60 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 88 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 46 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 89 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 46 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 55 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 28 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 54 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 28 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 94 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 169 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 56 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 171 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 54 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 92 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 60 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 92 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 59 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 69 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 67 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 69 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 66 KiB

-54
View File
@@ -72,49 +72,6 @@
<artifactId>qqq-backend-core</artifactId>
</dependency>
<!-- optional: the BackupsHealthIndicator (apps without qqq-middleware-health never load it) -->
<dependency>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-middleware-health</artifactId>
<optional>true</optional>
</dependency>
<!-- optional: ESB events (apps without qqq-esb never load it) -->
<dependency>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-esb</artifactId>
<optional>true</optional>
</dependency>
<!-- Optional: the production BackupStorage adapter. Apps that install this qbit -->
<!-- with object storage already have it; others use their own BackupStorage. -->
<dependency>
<groupId>com.kingsrook.qbits</groupId>
<artifactId>qbit-object-storage</artifactId>
<version>0.1.0-RC.2</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<version>2.0.3</version>
<scope>test</scope>
</dependency>
<!-- Demo server only (src/test/.../demo): never shipped. -->
<dependency>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-middleware-javalin</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-frontend-next</artifactId>
<version>1.0.0-RC.14</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
@@ -130,17 +87,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<!-- the MinIO test container default credentials, resolved through ${env.X} refs like production -->
<environmentVariables>
<STORAGE_TEST_ACCESS_KEY>minioadmin</STORAGE_TEST_ACCESS_KEY>
<STORAGE_TEST_SECRET_KEY>minioadmin</STORAGE_TEST_SECRET_KEY>
</environmentVariables>
</configuration>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
@@ -18,23 +18,16 @@ package com.kingsrook.qbits.backups;
import java.util.List;
import java.util.function.Function;
import java.util.function.Predicate;
import com.kingsrook.qbits.backups.codec.ArtifactTransform;
import com.kingsrook.qbits.backups.verify.ScratchTarget;
import com.kingsrook.qbits.backups.storage.BackupStorage;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
** Configuration of the Backups qbit.
** Host-facing configuration for the Backups QBit.
**
** Storage is resolved through a function from a bucket alias (the value of
** backups_plan.storageBucketAlias) to a BackupStorage. The production resolver
** is backed by qbit-object-storage; tests and local development use the
** in-memory implementation. No credential is ever held here or in a table.
** Tables are named backups_<name> (lowercase snake_case, backend name equal
** to the table name); permissions are backups.<action>.
*******************************************************************************/
public class BackupsQBitConfig implements QBitConfig
{
@@ -42,25 +35,6 @@ public class BackupsQBitConfig implements QBitConfig
public static final String PERMISSION_BASE_NAME = "backups";
private String backendName;
private String appName = "backups";
private String appLabel = "Backups";
private String parentAppName;
private String tablePathPrefix;
private String schedulerName;
private String instanceName;
private Integer maxConcurrentRuns = 1;
private Integer slaCheckIntervalSeconds = 300;
private Boolean openWhenAnonymous = false;
private Boolean prometheusEndpoint = false;
private Function<String, BackupStorage> storageResolver;
private Integer fileReverifyDays = 30;
private Function<String, ArtifactTransform> artifactTransformResolver;
private ScratchTarget scratchTarget;
private Predicate<String> planOwnerValidator;
private String esbDestinationName;
@@ -115,341 +89,4 @@ public class BackupsQBitConfig implements QBitConfig
return (this);
}
/*******************************************************************************
** Getter for appName
*******************************************************************************/
public String getAppName()
{
return (appName);
}
/*******************************************************************************
** Fluent setter for appName
*******************************************************************************/
public BackupsQBitConfig withAppName(String appName)
{
this.appName = appName;
return (this);
}
/*******************************************************************************
** Getter for appLabel
*******************************************************************************/
public String getAppLabel()
{
return (appLabel);
}
/*******************************************************************************
** Fluent setter for appLabel
*******************************************************************************/
public BackupsQBitConfig withAppLabel(String appLabel)
{
this.appLabel = appLabel;
return (this);
}
/*******************************************************************************
** Getter for parentAppName (for example "admin", so this is Admin, Backups).
*******************************************************************************/
public String getParentAppName()
{
return (parentAppName);
}
/*******************************************************************************
** Fluent setter for parentAppName
*******************************************************************************/
public BackupsQBitConfig withParentAppName(String parentAppName)
{
this.parentAppName = parentAppName;
return (this);
}
/*******************************************************************************
** Getter for tablePathPrefix (the front of the links the home makes to tables and
** records, "/app" for the Next dashboard, which routes /app/{table}/{id}).
*******************************************************************************/
public String getTablePathPrefix()
{
return (tablePathPrefix == null ? "/app" : tablePathPrefix);
}
/*******************************************************************************
** Fluent setter for tablePathPrefix
*******************************************************************************/
public BackupsQBitConfig withTablePathPrefix(String tablePathPrefix)
{
this.tablePathPrefix = tablePathPrefix;
return (this);
}
/*******************************************************************************
** Getter for schedulerName (the QQQ scheduler used for plan jobs and the SLA checker)
*******************************************************************************/
public String getSchedulerName()
{
return (schedulerName);
}
/*******************************************************************************
** Fluent setter for schedulerName
*******************************************************************************/
public BackupsQBitConfig withSchedulerName(String schedulerName)
{
this.schedulerName = schedulerName;
return (this);
}
/*******************************************************************************
** Getter for instanceName (used in object keys); defaults to the QInstance name
*******************************************************************************/
public String getInstanceName()
{
return (instanceName);
}
/*******************************************************************************
** Fluent setter for instanceName
*******************************************************************************/
public BackupsQBitConfig withInstanceName(String instanceName)
{
this.instanceName = instanceName;
return (this);
}
/*******************************************************************************
** Getter for maxConcurrentRuns
*******************************************************************************/
public Integer getMaxConcurrentRuns()
{
return (maxConcurrentRuns);
}
/*******************************************************************************
** Fluent setter for maxConcurrentRuns
*******************************************************************************/
public BackupsQBitConfig withMaxConcurrentRuns(Integer maxConcurrentRuns)
{
this.maxConcurrentRuns = maxConcurrentRuns;
return (this);
}
/*******************************************************************************
** Getter for slaCheckIntervalSeconds
*******************************************************************************/
public Integer getSlaCheckIntervalSeconds()
{
return (slaCheckIntervalSeconds);
}
/*******************************************************************************
** Fluent setter for slaCheckIntervalSeconds
*******************************************************************************/
public BackupsQBitConfig withSlaCheckIntervalSeconds(Integer slaCheckIntervalSeconds)
{
this.slaCheckIntervalSeconds = slaCheckIntervalSeconds;
return (this);
}
/*******************************************************************************
** Getter for openWhenAnonymous (local development only)
*******************************************************************************/
public Boolean getOpenWhenAnonymous()
{
return (openWhenAnonymous);
}
/*******************************************************************************
** Fluent setter for openWhenAnonymous
*******************************************************************************/
public BackupsQBitConfig withOpenWhenAnonymous(Boolean openWhenAnonymous)
{
this.openWhenAnonymous = openWhenAnonymous;
return (this);
}
/*******************************************************************************
** Getter for prometheusEndpoint
*******************************************************************************/
public Boolean getPrometheusEndpoint()
{
return (prometheusEndpoint);
}
/*******************************************************************************
** Fluent setter for prometheusEndpoint
*******************************************************************************/
public BackupsQBitConfig withPrometheusEndpoint(Boolean prometheusEndpoint)
{
this.prometheusEndpoint = prometheusEndpoint;
return (this);
}
/*******************************************************************************
** Getter for storageResolver
*******************************************************************************/
public Function<String, BackupStorage> getStorageResolver()
{
return (storageResolver);
}
/*******************************************************************************
** Fluent setter for storageResolver: bucket alias to storage
*******************************************************************************/
public BackupsQBitConfig withStorageResolver(Function<String, BackupStorage> storageResolver)
{
this.storageResolver = storageResolver;
return (this);
}
/*******************************************************************************
** Getter for fileReverifyDays: CONTENT verification skips file-store objects
** verified more recently than this.
*******************************************************************************/
public Integer getFileReverifyDays()
{
return (fileReverifyDays);
}
/*******************************************************************************
** Fluent setter for fileReverifyDays
*******************************************************************************/
public BackupsQBitConfig withFileReverifyDays(Integer fileReverifyDays)
{
this.fileReverifyDays = fileReverifyDays;
return (this);
}
/*******************************************************************************
** Getter for artifactTransformResolver (the decrypt hook for verify)
*******************************************************************************/
public Function<String, ArtifactTransform> getArtifactTransformResolver()
{
return (artifactTransformResolver);
}
/*******************************************************************************
** Fluent setter for artifactTransformResolver: encryption key reference (the name
** of a key in a key provider) to the transform that decrypts what that key wrote.
*******************************************************************************/
public BackupsQBitConfig withArtifactTransformResolver(Function<String, ArtifactTransform> artifactTransformResolver)
{
this.artifactTransformResolver = artifactTransformResolver;
return (this);
}
/*******************************************************************************
** Getter for scratchTarget (where TEST_RESTORE restores to; null means MEMORY)
*******************************************************************************/
public ScratchTarget getScratchTarget()
{
return (scratchTarget);
}
/*******************************************************************************
** Fluent setter for scratchTarget
*******************************************************************************/
public BackupsQBitConfig withScratchTarget(ScratchTarget scratchTarget)
{
this.scratchTarget = scratchTarget;
return (this);
}
/*******************************************************************************
** Getter for planOwnerValidator (true when the owner still exists and holds backups.admin)
*******************************************************************************/
public Predicate<String> getPlanOwnerValidator()
{
return (planOwnerValidator);
}
/*******************************************************************************
** Fluent setter for planOwnerValidator: with none, plan owners are not checked.
*******************************************************************************/
public BackupsQBitConfig withPlanOwnerValidator(Predicate<String> planOwnerValidator)
{
this.planOwnerValidator = planOwnerValidator;
return (this);
}
/*******************************************************************************
** Getter for esbDestinationName (ESB events are published here, when set)
*******************************************************************************/
public String getEsbDestinationName()
{
return (esbDestinationName);
}
/*******************************************************************************
** Fluent setter for esbDestinationName
*******************************************************************************/
public BackupsQBitConfig withEsbDestinationName(String esbDestinationName)
{
this.esbDestinationName = esbDestinationName;
return (this);
}
}
@@ -100,79 +100,4 @@ public class BackupsQBitProducer implements QBitMetaDataProducer<BackupsQBitConf
return (qBitConfig != null);
}
/*******************************************************************************
** Build everything and add it to the instance: possible-value sources, joins,
** tables, processes, widgets and the app.
*******************************************************************************/
@Override
public com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput produce(com.kingsrook.qqq.backend.core.model.metadata.QInstance qInstance) throws com.kingsrook.qqq.backend.core.exceptions.QException
{
java.util.List<String> errors = new java.util.ArrayList<>();
qBitConfig.validate(qInstance, errors);
if(!errors.isEmpty())
{
throw (new com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfigValidationException(qBitConfig, errors));
}
QBitMetaData qBit = getQBitMetaData();
qInstance.addQBit(qBit);
BackupsRuntime.install(qBitConfig);
///////////////////////////////////////////////////////////////////////////////
// bind the Alerts qbit producer SPI when it is on the class path (optional) //
///////////////////////////////////////////////////////////////////////////////
if(com.kingsrook.qbits.backups.alerts.AlertsQBitAlertSink.bindIfPresent())
{
new com.kingsrook.qbits.backups.alerts.AlertsQBitAlertSink().registerAlertTypes(qInstance);
}
com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput output = new com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput();
for(var possibleValueSource : com.kingsrook.qbits.backups.model.BackupsTables.possibleValueSources())
{
output.add(possibleValueSource);
}
for(var join : com.kingsrook.qbits.backups.model.BackupsTables.joins())
{
output.add(join);
}
java.util.List<com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData> tables = new java.util.ArrayList<>();
for(var table : com.kingsrook.qbits.backups.model.BackupsTables.tables(qBitConfig))
{
var customized = qBitConfig.getTableMetaDataCustomizer() == null ? table : qBitConfig.getTableMetaDataCustomizer().customizeMetaData(qInstance, table);
customized.setSourceQBitName(qBit.getName());
tables.add(customized);
output.add(customized);
}
for(var process : com.kingsrook.qbits.backups.processes.BackupsProcesses.processes())
{
process.setSourceQBitName(qBit.getName());
output.add(process);
}
for(var process : com.kingsrook.qbits.backups.processes.ExportTableProcess.exportProcesses(qInstance))
{
process.setSourceQBitName(qBit.getName());
output.add(process);
}
for(var widget : com.kingsrook.qbits.backups.app.BackupsApp.widgets())
{
output.add(widget);
}
for(var widget : com.kingsrook.qbits.backups.model.BackupsTables.widgets())
{
output.add(widget);
}
com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData app = com.kingsrook.qbits.backups.app.BackupsApp.app(qBitConfig, tables);
output.add(app);
output.addSelfToInstance(qInstance);
// A parent app only lists the apps that are its children, so attach ourselves (for example Admin, then Backups).
com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData parent = qBitConfig.getParentAppName() == null ? null : qInstance.getApp(qBitConfig.getParentAppName());
if(parent != null && (parent.getChildren() == null || parent.getChildren().stream().noneMatch(child -> app.getName().equals(child.getName()))))
{
parent.withChild(app);
}
return (new com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput());
}
}
@@ -1,195 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups;
import java.time.Clock;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Function;
import com.kingsrook.qbits.backups.alerts.BackupAlertSink;
import com.kingsrook.qbits.backups.alerts.NoOpBackupAlertSink;
import com.kingsrook.qbits.backups.storage.BackupStorage;
import com.kingsrook.qbits.backups.storage.BackupStorageException;
/*******************************************************************************
** Process-wide state of the installed qbit: the config, the clock, the alert
** sink and the per-process permission map. Static because QQQ instantiates
** steps and permission checkers by class reference.
*******************************************************************************/
public final class BackupsRuntime
{
private static final Map<String, String> PROCESS_PERMISSIONS = new ConcurrentHashMap<>();
private static volatile BackupsQBitConfig config;
private static volatile Clock clock = Clock.systemUTC();
private static volatile BackupAlertSink alertSink = new NoOpBackupAlertSink();
private static volatile Function<String, String> environment = System::getenv;
/*******************************************************************************
** Utility class.
*******************************************************************************/
private BackupsRuntime()
{
}
/*******************************************************************************
** Install the config (called by the producer).
*******************************************************************************/
public static void install(BackupsQBitConfig newConfig)
{
config = newConfig;
}
/*******************************************************************************
** Forget everything (tests).
*******************************************************************************/
public static void reset()
{
config = null;
clock = Clock.systemUTC();
alertSink = new NoOpBackupAlertSink();
environment = System::getenv;
PROCESS_PERMISSIONS.clear();
}
/*******************************************************************************
** The installed config, or null.
*******************************************************************************/
public static BackupsQBitConfig getConfig()
{
return (config);
}
/*******************************************************************************
** Whether the qbit is installed.
*******************************************************************************/
public static boolean isInstalled()
{
return (config != null);
}
/*******************************************************************************
** Whether permissions are open on fully anonymous instances (local development).
*******************************************************************************/
public static boolean isOpenWhenAnonymous()
{
return (config != null && Boolean.TRUE.equals(config.getOpenWhenAnonymous()));
}
/*******************************************************************************
** The clock (replaceable in tests).
*******************************************************************************/
public static Clock getClock()
{
return (clock);
}
/*******************************************************************************
** Replace the clock.
*******************************************************************************/
public static void setClock(Clock newClock)
{
clock = newClock;
}
/*******************************************************************************
** The alert sink.
*******************************************************************************/
public static BackupAlertSink getAlertSink()
{
return (alertSink);
}
/*******************************************************************************
** Bind an alert sink (the Alerts qbit adapter, or a test double).
*******************************************************************************/
public static void setAlertSink(BackupAlertSink newSink)
{
alertSink = newSink == null ? new NoOpBackupAlertSink() : newSink;
}
/*******************************************************************************
** Environment lookup (replaceable in tests).
*******************************************************************************/
public static Function<String, String> getEnvironment()
{
return (environment);
}
/*******************************************************************************
** Replace the environment lookup.
*******************************************************************************/
public static void setEnvironment(Function<String, String> newEnvironment)
{
environment = newEnvironment;
}
/*******************************************************************************
** The permission each qbit process needs, by process name.
*******************************************************************************/
public static Map<String, String> processPermissions()
{
return (PROCESS_PERMISSIONS);
}
/*******************************************************************************
** Resolve the storage for a bucket alias.
*******************************************************************************/
public static BackupStorage storageFor(String bucketAlias)
{
if(config == null || config.getStorageResolver() == null)
{
throw (new BackupStorageException(BackupStorageException.Reason.UNSUPPORTED, "No storage resolver is configured for the Backups qbit"));
}
BackupStorage storage = config.getStorageResolver().apply(bucketAlias);
if(storage == null)
{
throw (new BackupStorageException(BackupStorageException.Reason.NOT_FOUND, "No storage bucket is configured with the alias: " + bucketAlias));
}
return (storage);
}
}
@@ -1,310 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.alerts;
import java.lang.reflect.Method;
import java.util.Map;
import java.util.stream.Collectors;
import com.kingsrook.qbits.backups.BackupsRuntime;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
** The BackupAlertSink adapter for the Alerts qbit producer SPI (QRun-IO/qqq
** PR #1108, com.kingsrook.qqq.backend.core.alerts: QAlerts, AlertSignal,
** AlertResolution, AlertSeverity, QAlertTypeMetaData; design spec
** docs/specs/2026-10-12-qbit-alerts-design.md).
**
** STAND-IN: the SPI is an open core PR and is not in QQQ 4.1.0-RC.4, so this
** adapter is bound REFLECTIVELY and optionally. It compiles against nothing in
** the SPI; it looks the classes up by name, and every call is guarded:
** <ul>
** <li>no SPI classes on the class path: {@link #bindIfPresent()} binds nothing
** and the sink stays the no-op;</li>
** <li>the SPI is present but the instance has no alerts meta-data (the Alerts
** qbit is not installed): every call is a no-op;</li>
** <li>any reflective or SPI failure is logged once and swallowed, because an
** alerting problem must never fail a backup.</li>
** </ul>
** Replace with a direct binding (QAlerts.raise and friends) when RC.5 ships.
**
** Mapping: raise to QAlerts.raise(AlertSignal), resolve to QAlerts.resolve(type,
** dedupKey, note), heartbeat to QAlerts.heartbeat(key). The SPI has no per-plan
** heartbeat registration (monitors are declared on the alert type), so the
** expected interval and grace are not passed on; the SLA checker raises
** backups.plan.missed itself, independent of any heartbeat (spec 10.4).
*******************************************************************************/
public class AlertsQBitAlertSink implements BackupAlertSink
{
private static final QLogger LOG = QLogger.getLogger(AlertsQBitAlertSink.class);
/** The package of the Alerts producer SPI in qqq-backend-core. */
public static final String DEFAULT_API_PACKAGE = "com.kingsrook.qqq.backend.core.alerts";
private final String apiPackage;
private boolean warned = false;
private Class<?> qAlerts;
private Class<?> alertSignal;
private Class<?> alertSeverity;
private Class<?> instanceMetaData;
/*******************************************************************************
** Constructor for the real SPI package.
*******************************************************************************/
public AlertsQBitAlertSink()
{
this(DEFAULT_API_PACKAGE);
}
/*******************************************************************************
** Constructor for another package (a test double of the SPI shape).
*******************************************************************************/
public AlertsQBitAlertSink(String apiPackage)
{
this.apiPackage = apiPackage;
}
/*******************************************************************************
** Whether the SPI classes are on the class path.
*******************************************************************************/
public static boolean isPresent(String apiPackage)
{
try
{
Class.forName(apiPackage + ".QAlerts");
Class.forName(apiPackage + ".AlertSignal");
return (true);
}
catch(ClassNotFoundException | LinkageError e)
{
return (false);
}
}
/*******************************************************************************
** Bind this adapter as the runtime's alert sink when the SPI is present and no
** other sink was bound already. Returns whether it bound.
*******************************************************************************/
public static boolean bindIfPresent()
{
if(!(BackupsRuntime.getAlertSink() instanceof NoOpBackupAlertSink) || !isPresent(DEFAULT_API_PACKAGE))
{
return (false);
}
BackupsRuntime.setAlertSink(new AlertsQBitAlertSink());
return (true);
}
/*******************************************************************************
** Declare the backup alert types with the Alerts qbit (QAlertTypeMetaData), when
** the SPI is present. Returns how many were added.
*******************************************************************************/
public int registerAlertTypes(QInstance qInstance)
{
int added = 0;
try
{
Class<?> typeClass = Class.forName(apiPackage + ".QAlertTypeMetaData");
load();
for(BackupAlerts.AlertType alertType : BackupAlerts.alertTypes())
{
try
{
Object type = typeClass.getConstructor().newInstance();
typeClass.getMethod("withName", String.class).invoke(type, alertType.name());
typeClass.getMethod("withLabel", String.class).invoke(type, alertType.label());
typeClass.getMethod("withGroupName", String.class).invoke(type, "Backups");
typeClass.getMethod("withHowToFix", String.class).invoke(type, alertType.howToFix());
typeClass.getMethod("withDefaultSeverity", alertSeverity).invoke(type, severity(alertType.defaultSeverity()));
typeClass.getMethod("addSelfToInstance", QInstance.class).invoke(type, qInstance);
added++;
}
catch(ReflectiveOperationException | RuntimeException e)
{
warnOnce("Could not declare the alert type " + alertType.name(), e);
}
}
}
catch(ReflectiveOperationException | LinkageError e)
{
warnOnce("The alerts SPI is not available", e);
}
return (added);
}
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public void raise(String type, String dedupeKey, Severity severity, String title, Map<String, String> detail)
{
if(!isAlertsQBitInstalled())
{
return;
}
try
{
Object signal = alertSignal.getConstructor(String.class).newInstance(type);
alertSignal.getMethod("setDedupKey", String.class).invoke(signal, dedupeKey);
alertSignal.getMethod("setSeverity", alertSeverity).invoke(signal, severity(severity));
alertSignal.getMethod("setTitle", String.class).invoke(signal, title);
if(detail != null && !detail.isEmpty())
{
alertSignal.getMethod("setSummary", String.class).invoke(signal, detail.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(", ")));
Method withAttribute = alertSignal.getMethod("withAttribute", String.class, String.class);
for(Map.Entry<String, String> entry : detail.entrySet())
{
withAttribute.invoke(signal, entry.getKey(), entry.getValue());
}
}
qAlerts.getMethod("raise", alertSignal).invoke(null, signal);
}
catch(ReflectiveOperationException | RuntimeException e)
{
warnOnce("Could not raise an alert", e);
}
}
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public void resolve(String type, String dedupeKey)
{
if(!isAlertsQBitInstalled())
{
return;
}
try
{
qAlerts.getMethod("resolve", String.class, String.class, String.class).invoke(null, type, dedupeKey, "Cleared by the Backups qbit");
}
catch(ReflectiveOperationException | RuntimeException e)
{
warnOnce("Could not resolve an alert", e);
}
}
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public void heartbeat(String monitorKey, long expectedEverySeconds, long graceSeconds)
{
if(!isAlertsQBitInstalled())
{
return;
}
try
{
qAlerts.getMethod("heartbeat", String.class).invoke(null, monitorKey);
}
catch(ReflectiveOperationException | RuntimeException e)
{
warnOnce("Could not check in a heartbeat", e);
}
}
/*******************************************************************************
** The SPI classes load, and the instance has alerts meta-data (the Alerts qbit
** is installed). Checked on every call so the order qbits are produced in
** does not matter.
*******************************************************************************/
boolean isAlertsQBitInstalled()
{
try
{
load();
QInstance qInstance = QContext.getQInstance();
if(qInstance == null)
{
return (false);
}
return (instanceMetaData.getMethod("of", QInstance.class).invoke(null, qInstance) != null);
}
catch(ReflectiveOperationException | RuntimeException | LinkageError e)
{
warnOnce("The alerts SPI is not usable", e);
return (false);
}
}
/*******************************************************************************
**
*******************************************************************************/
private synchronized void load() throws ClassNotFoundException
{
if(qAlerts == null)
{
qAlerts = Class.forName(apiPackage + ".QAlerts");
alertSignal = Class.forName(apiPackage + ".AlertSignal");
alertSeverity = Class.forName(apiPackage + ".AlertSeverity");
instanceMetaData = Class.forName(apiPackage + ".AlertsInstanceMetaData");
}
}
/*******************************************************************************
** Map our severity to the SPI's enum constant of the same name.
*******************************************************************************/
@SuppressWarnings({ "unchecked", "rawtypes" })
private Object severity(Severity severity)
{
return (Enum.valueOf((Class<Enum>) alertSeverity, severity.name()));
}
/*******************************************************************************
**
*******************************************************************************/
private void warnOnce(String message, Throwable t)
{
if(!warned)
{
warned = true;
LOG.warn(message + " (the Alerts qbit adapter will keep failing quietly)", t, logPair("apiPackage", apiPackage));
}
}
}
@@ -1,65 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.alerts;
import java.util.Map;
/*******************************************************************************
** Where backups raises and resolves alerts. The adapter for the Alerts qbit
** producer SPI binds here when that qbit is installed; otherwise the no-op
** sink is used and every condition is still visible on the Backups home and in
** the tables. An alert has a stable dedupe key; counts and links only, never
** values, keys or presigned URLs.
*******************************************************************************/
public interface BackupAlertSink
{
/***************************************************************************
** Raise (or refresh) an alert.
**
** @param type alert type, e.g. backups.plan.missed
** @param dedupeKey stable key, type plus plan or run
** @param severity severity
** @param title short title
** @param detail counts and context only
***************************************************************************/
void raise(String type, String dedupeKey, Severity severity, String title, Map<String, String> detail);
/***************************************************************************
** Resolve an alert that was raised under the dedupe key.
***************************************************************************/
void resolve(String type, String dedupeKey);
/***************************************************************************
** Check in a heartbeat monitor: the plan ran successfully.
***************************************************************************/
void heartbeat(String monitorKey, long expectedEverySeconds, long graceSeconds);
/***************************************************************************
** Severities.
***************************************************************************/
enum Severity
{
INFO, WARNING, ERROR, CRITICAL
}
}
@@ -1,203 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.alerts;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qbits.backups.BackupsRuntime;
/*******************************************************************************
** Every alert the Backups qbit raises (design spec section 10.4), with stable
** dedupe keys, and helpers that send them through the bound BackupAlertSink.
**
** A dedupe key is the alert type plus a plan key (or a run key, or a plan key
** and a verify level), so a repeated condition refreshes one alert and a cleared
** condition resolves exactly that alert. Details carry counts and names only:
** never row values, object keys with secrets in them, or presigned URLs.
*******************************************************************************/
public final class BackupAlerts
{
public static final String PLAN_MISSED = "backups.plan.missed";
public static final String RUN_FAILED = "backups.run.failed";
public static final String RUN_MISSING = "backups.run.missing_in_storage";
public static final String FILES_MISSING = "backups.files.missing";
public static final String VERIFY_FAILED = "backups.verify.failed";
public static final String VERIFY_OVERDUE = "backups.verify.overdue";
public static final String PLAN_OWNER_INVALID = "backups.plan.owner_invalid";
public static final String STORAGE_DEBRIS = "backups.storage.debris";
public static final String RESTORE_COMPLETED = "backups.restore.completed";
/** The prefix of the heartbeat monitor key of a plan. */
public static final String HEARTBEAT_PREFIX = "backups.plan:";
/***************************************************************************
** A declared alert type, for the Alerts qbit's type registry.
**
** @param name the alert type name
** @param label a short label
** @param defaultSeverity the severity when a signal does not say
** @param howToFix one sentence for the person looking at the alert
***************************************************************************/
public record AlertType(String name, String label, BackupAlertSink.Severity defaultSeverity, String howToFix)
{
}
/*******************************************************************************
** Utility class.
*******************************************************************************/
private BackupAlerts()
{
}
/*******************************************************************************
** The alert types of the spec table (section 10.4), in table order.
*******************************************************************************/
public static List<AlertType> alertTypes()
{
return (List.of(
new AlertType(PLAN_MISSED, "Backup did not run", BackupAlertSink.Severity.ERROR,
"Open the plan and its latest runs, check the schedule and the scheduler, then run the backup by hand."),
new AlertType(RUN_FAILED, "Backup run failed", BackupAlertSink.Severity.WARNING,
"Open the failed run for the error, fix the cause, and run the plan again."),
new AlertType(RUN_MISSING, "Backup is missing from storage", BackupAlertSink.Severity.CRITICAL,
"The catalog says the run succeeded but its manifest is gone from the bucket. Check the bucket, its lifecycle rules and the credentials, then run a new backup."),
new AlertType(FILES_MISSING, "Backed up files were missing at the source", BackupAlertSink.Severity.WARNING,
"Open the run's file index: records point at files that no longer exist."),
new AlertType(VERIFY_FAILED, "Backup verification failed", BackupAlertSink.Severity.ERROR,
"Open the verification for the mismatches. Treat the run as unusable and take a new backup."),
new AlertType(VERIFY_OVERDUE, "Backup verification is overdue", BackupAlertSink.Severity.WARNING,
"Run Verify backup on a recent run of the plan, or set a verify schedule on the plan."),
new AlertType(PLAN_OWNER_INVALID, "Backup plan owner is not valid", BackupAlertSink.Severity.WARNING,
"Set an owner who exists and holds backups.admin on the plan."),
new AlertType(STORAGE_DEBRIS, "Backup storage has debris or orphans", BackupAlertSink.Severity.INFO,
"Prune clears debris. Orphan prefixes can be imported into the catalog or deleted."),
new AlertType(RESTORE_COMPLETED, "A restore finished", BackupAlertSink.Severity.INFO,
"Nothing to fix unless the restore failed. Open the restore run.")));
}
/*******************************************************************************
** The stable dedupe key for a type and an id (plan key, run key...).
*******************************************************************************/
public static String key(String type, String id)
{
return (type + ":" + id);
}
/*******************************************************************************
** The dedupe key of a verify failure: it includes the level so a pass at a
** higher level clears the lower failures, and a cheap pass never hides a
** content failure.
*******************************************************************************/
public static String verifyKey(String planKey, String level)
{
return (VERIFY_FAILED + ":" + planKey + ":" + level);
}
/*******************************************************************************
** Raise (or refresh) an alert. Never throws: an alerting problem must not fail
** a backup or a check.
*******************************************************************************/
public static void raise(String type, String id, BackupAlertSink.Severity severity, String title, String... detailPairs)
{
try
{
Map<String, String> detail = new LinkedHashMap<>();
for(int i = 0; i + 1 < detailPairs.length; i += 2)
{
detail.put(detailPairs[i], detailPairs[i + 1]);
}
BackupsRuntime.getAlertSink().raise(type, key(type, id), severity, title, detail);
}
catch(RuntimeException e)
{
// fire and forget
}
}
/*******************************************************************************
** Raise with an explicit dedupe key.
*******************************************************************************/
public static void raiseWithKey(String type, String dedupeKey, BackupAlertSink.Severity severity, String title, Map<String, String> detail)
{
try
{
BackupsRuntime.getAlertSink().raise(type, dedupeKey, severity, title, detail);
}
catch(RuntimeException e)
{
// fire and forget
}
}
/*******************************************************************************
** Resolve the alert of a type and id. Never throws.
*******************************************************************************/
public static void resolve(String type, String id)
{
resolveKey(type, key(type, id));
}
/*******************************************************************************
** Resolve an alert by its dedupe key. Never throws.
*******************************************************************************/
public static void resolveKey(String type, String dedupeKey)
{
try
{
BackupsRuntime.getAlertSink().resolve(type, dedupeKey);
}
catch(RuntimeException e)
{
// fire and forget
}
}
/*******************************************************************************
** A restore finished: an info alert, so restores are never silent (error when
** it failed). The Alerts qbit clears it after 7 days; nothing resolves it here.
*******************************************************************************/
public static void restoreCompleted(String restoreRunKey, String status, String planKey, long rowsRestored)
{
boolean failed = "FAILED".equals(status);
raise(RESTORE_COMPLETED, restoreRunKey, failed ? BackupAlertSink.Severity.ERROR : BackupAlertSink.Severity.INFO,
failed ? "A restore failed" : "A restore finished",
"restoreRun", restoreRunKey, "status", String.valueOf(status), "plan", String.valueOf(planKey), "rows", String.valueOf(rowsRestored));
}
}
@@ -1,56 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.alerts;
import java.util.Map;
/*******************************************************************************
** The sink used when the Alerts qbit is not installed.
*******************************************************************************/
public class NoOpBackupAlertSink implements BackupAlertSink
{
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public void raise(String type, String dedupeKey, Severity severity, String title, Map<String, String> detail)
{
}
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public void resolve(String type, String dedupeKey)
{
}
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public void heartbeat(String monitorKey, long expectedEverySeconds, long graceSeconds)
{
}
}
@@ -1,141 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.app;
import java.util.List;
import com.kingsrook.qbits.backups.BackupsQBitConfig;
import com.kingsrook.qbits.backups.model.BackupsTables;
import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
/*******************************************************************************
** The Backups app (a sub-app under Admin when configured) and its widgets. The
** home is the single owner of backup data: every card is built from the qbit
** tables, and each has an honest empty state. The nav lists the main tables;
** sub-tables appear as child lists on their parents.
*******************************************************************************/
public final class BackupsApp
{
public static final String HOME_WIDGET = "backupsHome";
public static final String HEALTH_CARD_WIDGET = "backupsHealthCard";
/*******************************************************************************
** Utility class.
*******************************************************************************/
private BackupsApp()
{
}
/*******************************************************************************
** The widgets.
*******************************************************************************/
public static List<QWidgetMetaData> widgets()
{
return (List.of(
new QWidgetMetaData().withName(HOME_WIDGET).withLabel("Overview").withType(WidgetType.HTML.getType())
.withCodeReference(new QCodeReference(HomeRenderer.class)).withGridColumns(12).withIsCard(false).withPermissionRules(BackupsTables.permissionRules()),
new QWidgetMetaData().withName(HEALTH_CARD_WIDGET).withLabel("Backups").withType(WidgetType.HTML.getType())
.withCodeReference(new QCodeReference(HealthCardRenderer.class)).withGridColumns(12).withIsCard(true).withPermissionRules(BackupsTables.permissionRules())));
}
/*******************************************************************************
** The app: the main tables only; sub-tables are relations on their parents.
*******************************************************************************/
public static QAppMetaData app(BackupsQBitConfig config, List<QTableMetaData> tables)
{
QAppMetaData app = new QAppMetaData()
.withName(config.getAppName())
.withLabel(config.getAppLabel())
.withIcon(new QIcon("backup"))
.withPermissionRules(BackupsTables.permissionRules())
.withWidgets(List.of(HOME_WIDGET));
if(config.getParentAppName() != null)
{
app.setParentAppName(config.getParentAppName());
}
for(QTableMetaData table : tables)
{
if(BackupsTables.NAV_TABLES.contains(table.getName()))
{
app.withChild(table);
}
}
return (app);
}
/*******************************************************************************
** The home.
*******************************************************************************/
public static class HomeRenderer extends AbstractWidgetRenderer
{
/*******************************************************************************
** Render.
*******************************************************************************/
@Override
public RenderWidgetOutput render(RenderWidgetInput input) throws QException
{
return (new RenderWidgetOutput(new RawHTML(null, BackupsHomeView.render(BackupsHomeData.load(), pathPrefix()))));
}
}
/*******************************************************************************
** The status card for Admin, Health.
*******************************************************************************/
public static class HealthCardRenderer extends AbstractWidgetRenderer
{
/*******************************************************************************
** Render.
*******************************************************************************/
@Override
public RenderWidgetOutput render(RenderWidgetInput input) throws QException
{
return (new RenderWidgetOutput(new RawHTML("Backups", BackupsHomeView.renderHealthCard(BackupsHomeData.load(), pathPrefix()))));
}
}
/*******************************************************************************
** The path prefix of table and record links.
*******************************************************************************/
static String pathPrefix()
{
var config = com.kingsrook.qbits.backups.BackupsRuntime.getConfig();
return (config == null ? "/app" : config.getTablePathPrefix());
}
}
@@ -1,304 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.app;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qbits.backups.BackupsRuntime;
import com.kingsrook.qbits.backups.model.BackupsTables;
import com.kingsrook.qbits.backups.run.BackupCatalog;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
/*******************************************************************************
** The data behind the Backups home: plan status, what needs attention, recent
** activity and the size trend. Everything is read from the qbit tables; nothing
** is invented. Home cards read the denormalized plan fields (slaStatus, last
** success) that the processes and the SLA checker maintain.
*******************************************************************************/
public class BackupsHomeData
{
/** Plan statuses that need attention (the plan or its backups have a problem). */
public static final List<String> ATTENTION = List.of("LATE", "FAILING", "MISSING_IN_STORAGE", "UNVERIFIED", "ABANDONED", "OWNER_INVALID");
private final List<QRecord> plans;
private final List<QRecord> recentRuns;
private final List<QRecord> failedRuns;
private final List<QRecord> verifications;
private final List<QRecord> restores;
private final Instant now;
/*******************************************************************************
** Constructor.
*******************************************************************************/
BackupsHomeData(List<QRecord> plans, List<QRecord> recentRuns, List<QRecord> failedRuns, List<QRecord> verifications, List<QRecord> restores, Instant now)
{
this.plans = plans;
this.recentRuns = recentRuns;
this.failedRuns = failedRuns;
this.verifications = verifications;
this.restores = restores;
this.now = now;
}
/*******************************************************************************
** Load from the tables.
*******************************************************************************/
public static BackupsHomeData load() throws QException
{
BackupCatalog catalog = new BackupCatalog();
Instant now = BackupsRuntime.getClock().instant();
List<QRecord> plans = catalog.query(BackupsTables.PLAN, new QQueryFilter().withOrderBy(new QFilterOrderBy("name", true)));
QQueryFilter recent = new QQueryFilter().withOrderBy(new QFilterOrderBy("id", false));
recent.setLimit(200);
List<QRecord> runs = catalog.query(BackupsTables.RUN, recent);
List<QRecord> failed = new ArrayList<>();
Instant weekAgo = now.minus(Duration.ofDays(7));
for(QRecord run : runs)
{
Instant start = run.getValueInstant("startDate");
if(List.of("FAILED", "TIMED_OUT", "ABANDONED").contains(run.getValueString("status")) && (start == null || start.isAfter(weekAgo)))
{
failed.add(run);
}
}
QQueryFilter verificationFilter = new QQueryFilter().withOrderBy(new QFilterOrderBy("id", false));
verificationFilter.setLimit(100);
QQueryFilter restoreFilter = new QQueryFilter().withOrderBy(new QFilterOrderBy("id", false));
restoreFilter.setLimit(20);
return (new BackupsHomeData(plans, runs, failed, catalog.query(BackupsTables.VERIFICATION, verificationFilter),
catalog.query(BackupsTables.RESTORE_RUN, restoreFilter), now));
}
/*******************************************************************************
** Plans.
*******************************************************************************/
public List<QRecord> plans()
{
return (plans);
}
/*******************************************************************************
** Recent runs (newest first).
*******************************************************************************/
public List<QRecord> recentRuns()
{
return (recentRuns);
}
/*******************************************************************************
** Runs that failed in the last 7 days.
*******************************************************************************/
public List<QRecord> failedRuns()
{
return (failedRuns);
}
/*******************************************************************************
** Verifications (newest first).
*******************************************************************************/
public List<QRecord> verifications()
{
return (verifications);
}
/*******************************************************************************
** Restores (newest first).
*******************************************************************************/
public List<QRecord> restores()
{
return (restores);
}
/*******************************************************************************
** The time the data was loaded.
*******************************************************************************/
public Instant now()
{
return (now);
}
/*******************************************************************************
** The status to show for a plan: the stored slaStatus when it is a problem state,
** otherwise LATE when the age of the last success is past the SLA (the SLA
** checker may not have run yet), otherwise the stored status (UNKNOWN when none).
*******************************************************************************/
public String effectiveStatus(QRecord plan)
{
String stored = plan.getValueString("slaStatus");
if(stored != null && ATTENTION.contains(stored))
{
return (stored);
}
Long age = ageHours(plan);
Integer sla = plan.getValueInteger("slaHours");
if(age != null && sla != null && age > sla)
{
return ("LATE");
}
return (stored == null ? "UNKNOWN" : stored);
}
/*******************************************************************************
** Plans that need attention.
*******************************************************************************/
public List<QRecord> plansNeedingAttention()
{
return (plans.stream().filter(p -> Boolean.TRUE.equals(p.getValueBoolean("isActive")) && ATTENTION.contains(effectiveStatus(p))).toList());
}
/*******************************************************************************
** The last run of a plan (from the recent runs), or null.
*******************************************************************************/
public QRecord lastRun(QRecord plan)
{
Integer lastRunId = plan.getValueInteger("lastRunId");
for(QRecord run : recentRuns)
{
if(lastRunId != null && lastRunId.equals(run.getValueInteger("id")))
{
return (run);
}
}
return (null);
}
/*******************************************************************************
** The overall state: NO_PLANS, ATTENTION or OK.
*******************************************************************************/
public String overallState()
{
if(plans.isEmpty())
{
return ("NO_PLANS");
}
return (plansNeedingAttention().isEmpty() && failedRuns.isEmpty() ? "OK" : "ATTENTION");
}
/*******************************************************************************
** Hours since the last successful backup of a plan, or null when it has none.
*******************************************************************************/
public Long ageHours(QRecord plan)
{
Instant last = plan.getValueInstant("lastSuccessDate");
return (last == null ? null : Duration.between(last, now).toHours());
}
/*******************************************************************************
** The latest test restore of a plan, or null.
*******************************************************************************/
public QRecord latestTestRestore(Integer planId)
{
for(QRecord verification : verifications)
{
if("TEST_RESTORE".equals(verification.getValueString("level")))
{
Integer runId = verification.getValueInteger("backupRunId");
for(QRecord run : recentRuns)
{
if(runId != null && planId != null && runId.equals(run.getValueInteger("id")) && planId.equals(run.getValueInteger("backupPlanId")))
{
return (verification);
}
}
}
}
return (null);
}
/*******************************************************************************
** Successful runs of a plan, oldest first, for the trend (bytes stored, rows, duration).
*******************************************************************************/
public List<QRecord> successfulRuns(Integer planId)
{
List<QRecord> rs = new ArrayList<>();
if(planId == null)
{
return (rs);
}
for(QRecord run : recentRuns)
{
if(planId.equals(run.getValueInteger("backupPlanId")) && run.getValueString("status") != null && run.getValueString("status").startsWith("SUCCEEDED"))
{
rs.add(0, run);
}
}
return (rs);
}
/*******************************************************************************
** Runs kept per retention class for a plan (as classified by the last prune).
*******************************************************************************/
public Map<String, Integer> retentionCounts(Integer planId)
{
Map<String, Integer> counts = new LinkedHashMap<>();
if(planId == null)
{
return (counts);
}
for(QRecord run : recentRuns)
{
if(planId.equals(run.getValueInteger("backupPlanId")) && run.getValueString("status") != null && run.getValueString("status").startsWith("SUCCEEDED")
&& run.getValue("prunedDate") == null)
{
counts.merge(run.getValueString("retentionClass") == null ? "UNCLASSIFIED" : run.getValueString("retentionClass"), 1, Integer::sum);
}
}
return (counts);
}
}
@@ -1,504 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.app;
import java.time.Instant;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.Map;
import com.kingsrook.qbits.backups.model.BackupsTables;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
/*******************************************************************************
** Renders the Backups home: a plan status row per plan across the full width,
** then cards for what needs attention, the latest test restore, recent activity,
** the size trend and retention. Every card has an honest empty state. Layout
** is flow and grid only (no tables, no fixed widths, no inner scrolling): plan
** rows wrap their cells and the cards stack to one column on narrow screens,
** long names wrap, and colors are inherited (only the status dot is colored), so
** it reads at every width in light and dark mode.
*******************************************************************************/
public final class BackupsHomeView
{
private static final DateTimeFormatter WHEN = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm 'UTC'").withZone(ZoneOffset.UTC);
private static final String ROOT = "display:grid;gap:1rem;min-width:0;";
private static final String GRID = "display:grid;grid-template-columns:repeat(auto-fit,minmax(min(100%,24rem),1fr));gap:1rem;align-items:start;min-width:0;";
private static final String CARD = "border:1px solid rgba(127,127,127,.3);border-radius:8px;padding:1rem;min-width:0;overflow-wrap:anywhere;";
private static final String ROW = "border:1px solid rgba(127,127,127,.25);border-radius:6px;padding:.75rem;margin:0 0 .75rem;min-width:0;";
private static final String CELLS = "display:grid;grid-template-columns:repeat(auto-fit,minmax(9.5rem,1fr));gap:.6rem 1rem;margin:.6rem 0 0;font-size:.9rem;";
private static final String LIST = "display:grid;grid-template-columns:minmax(7rem,auto) 1fr;gap:.25rem .75rem;margin:0 0 .75rem;font-size:.9rem;";
private static final String ITEMS = "margin:0 0 .75rem;padding-left:1.25rem;list-style:disc;font-size:.9rem;display:grid;gap:.35rem;";
private static final String MUTED = "opacity:.75;";
private static final String LINK = "color:inherit;text-decoration:underline;text-underline-offset:2px;";
private static final String CHIP = "display:inline-flex;align-items:center;gap:.4rem;padding:.05rem .6rem;border-radius:999px;border:1px solid rgba(127,127,127,.4);font-size:.8rem;white-space:nowrap;";
private static final int TREND_BARS = 30;
/*******************************************************************************
** Utility class.
*******************************************************************************/
private BackupsHomeView()
{
}
/*******************************************************************************
** The whole home; pathPrefix is the prefix of table and record links.
*******************************************************************************/
public static String render(BackupsHomeData data, String pathPrefix)
{
StringBuilder html = new StringBuilder("<div class=\"backups-home\" style=\"" + ROOT + "\">");
planStatus(html, data, pathPrefix);
html.append("<div style=\"").append(GRID).append("\">");
needsAttention(html, data, pathPrefix);
latestTestRestore(html, data, pathPrefix);
recentActivity(html, data, pathPrefix);
trend(html, data);
retention(html, data);
return (html.append("</div></div>").toString());
}
/*******************************************************************************
** The one status card for Admin, Health.
*******************************************************************************/
public static String renderHealthCard(BackupsHomeData data, String pathPrefix)
{
String text = switch(data.overallState())
{
case "NO_PLANS" -> "No backup plans yet.";
case "ATTENTION" -> "Needs attention: see the Backups home.";
default -> "All backups are within their SLA.";
};
return ("<div style=\"" + CARD + "\"><strong>Backups</strong><p>" + esc(text) + "</p><a style=\"" + LINK + "\" href=\"" + esc(pathPrefix) + "\">Open the Backups home</a></div>");
}
/*******************************************************************************
** Plan status: one row per plan, the cells wrap to the width.
*******************************************************************************/
private static void planStatus(StringBuilder html, BackupsHomeData data, String prefix)
{
open(html, "Plan status");
if(data.plans().isEmpty())
{
empty(html, "No backup plans yet. Create one from Backup plans, New.");
}
for(QRecord plan : data.plans())
{
Long age = data.ageHours(plan);
Integer sla = plan.getValueInteger("slaHours");
String status = data.effectiveStatus(plan);
QRecord last = data.lastRun(plan);
html.append("<div style=\"").append(ROW).append("\"><div style=\"display:flex;flex-wrap:wrap;align-items:center;gap:.5rem .75rem;\">");
html.append("<strong>").append(recordLink(prefix, BackupsTables.PLAN, plan.getValue("id"), plan.getValueString("name"))).append("</strong>");
html.append(chip(status));
if(!Boolean.TRUE.equals(plan.getValueBoolean("isActive")))
{
html.append("<span style=\"").append(MUTED).append("\">Not active</span>");
}
html.append("</div><div style=\"").append(CELLS).append("\">");
cell(html, "Last backup", plan.getValueInstant("lastSuccessDate") == null ? "No successful backup yet" : WHEN.format(plan.getValueInstant("lastSuccessDate")));
cell(html, "Age against SLA", age != null && sla != null ? age + " h of " + sla + " h" : (sla == null ? "No SLA set" : "Not run yet"));
cell(html, "Last run", last == null ? "None yet" : label(last.getValueString("status")) + (last.getValueInstant("startDate") == null ? "" : ", " + WHEN.format(last.getValueInstant("startDate"))));
cell(html, "Schedule", plan.getValueString("cronExpression") == null ? "Manual only" : plan.getValueString("cronExpression"));
cell(html, "Last verified", plan.getValueInstant("lastVerifiedDate") == null ? "Not verified yet" : WHEN.format(plan.getValueInstant("lastVerifiedDate")));
if(plan.getValueString("objectLockMode") != null && !"NONE".equals(plan.getValueString("objectLockMode")))
{
cell(html, "Object lock", plan.getValueString("objectLockMode") + (plan.getValueInteger("objectLockDays") == null ? "" : ", " + plan.getValueInteger("objectLockDays") + " days"));
}
html.append("</div></div>");
}
link(html, prefix, BackupsTables.PLAN, "Backup plans");
close(html);
}
/*******************************************************************************
** Needs attention.
*******************************************************************************/
private static void needsAttention(StringBuilder html, BackupsHomeData data, String prefix)
{
open(html, "Needs attention");
List<QRecord> plans = data.plansNeedingAttention();
if(data.plans().isEmpty())
{
empty(html, "Nothing to check yet: no plans.");
}
else if(plans.isEmpty() && data.failedRuns().isEmpty())
{
empty(html, "Nothing needs attention.");
}
else
{
html.append("<ul style=\"").append(ITEMS).append("\">");
for(QRecord plan : plans)
{
html.append("<li>").append(recordLink(prefix, BackupsTables.PLAN, plan.getValue("id"), plan.getValueString("name")))
.append(": ").append(esc(label(data.effectiveStatus(plan)))).append("</li>");
}
for(QRecord run : data.failedRuns())
{
html.append("<li>Run ").append(recordLink(prefix, BackupsTables.RUN, run.getValue("id"), run.getValueString("runKey")))
.append(": ").append(esc(label(run.getValueString("status")))).append("</li>");
}
html.append("</ul>");
}
link(html, prefix, BackupsTables.RUN, "All runs");
close(html);
}
/*******************************************************************************
** Latest test restore per plan.
*******************************************************************************/
private static void latestTestRestore(StringBuilder html, BackupsHomeData data, String prefix)
{
open(html, "Latest test restore");
boolean any = false;
for(QRecord plan : data.plans())
{
QRecord v = data.latestTestRestore(plan.getValueInteger("id"));
if(v == null)
{
continue;
}
any = true;
html.append("<dl style=\"").append(LIST).append("\">");
row(html, "Plan", plan.getValueString("name"));
row(html, "Result", label(v.getValueString("status")));
row(html, "Compared", number(v.getValueLong("rowsChecked")) + " rows in " + number(toLong(v.getValueInteger("tablesChecked"))) + " tables, "
+ number(toLong(v.getValueInteger("mismatches"))) + " mismatches");
row(html, "Scratch target", v.getValueString("scratchTarget") == null ? "Not recorded" : v.getValueString("scratchTarget"));
html.append("</dl>");
}
if(!any)
{
empty(html, "No test restore yet. Run Verify backup, Test restore on a run, or set a verify schedule on the plan.");
}
link(html, prefix, BackupsTables.VERIFICATION, "All verifications");
close(html);
}
/*******************************************************************************
** Size trend per plan: bars for the stored size of each successful run (oldest
** left), then the numbers.
*******************************************************************************/
private static void trend(StringBuilder html, BackupsHomeData data)
{
open(html, "Size and duration trend");
boolean any = false;
for(QRecord plan : data.plans())
{
List<QRecord> runs = data.successfulRuns(plan.getValueInteger("id"));
if(runs.isEmpty())
{
continue;
}
any = true;
QRecord first = runs.get(0);
QRecord last = runs.get(runs.size() - 1);
html.append("<div style=\"margin:0 0 .75rem;\"><strong style=\"font-size:.9rem;\">").append(esc(plan.getValueString("name"))).append("</strong>");
bars(html, runs);
html.append("<dl style=\"").append(LIST).append("\">");
row(html, "Successful runs", String.valueOf(runs.size()));
row(html, "Stored", bytes(first.getValueLong("bytesStored")) + " at first, " + bytes(last.getValueLong("bytesStored")) + " latest");
row(html, "Rows", number(first.getValueLong("rowsTotal")) + " at first, " + number(last.getValueLong("rowsTotal")) + " latest");
row(html, "Duration", last.getValueLong("durationMs") == null ? "Not recorded" : duration(last.getValueLong("durationMs")) + " latest");
html.append("</dl></div>");
}
if(!any)
{
empty(html, "Trends appear after the first successful run.");
}
close(html);
}
/*******************************************************************************
** A row of bars, one per run, tall in proportion to the stored bytes.
*******************************************************************************/
private static void bars(StringBuilder html, List<QRecord> runs)
{
List<QRecord> shown = runs.size() > TREND_BARS ? runs.subList(runs.size() - TREND_BARS, runs.size()) : runs;
long max = 1;
for(QRecord run : shown)
{
max = Math.max(max, run.getValueLong("bytesStored") == null ? 0 : run.getValueLong("bytesStored"));
}
html.append("<div role=\"img\" aria-label=\"Stored size of the last ").append(shown.size()).append(" successful runs\" style=\"display:flex;align-items:flex-end;gap:2px;height:2.5rem;margin:.4rem 0 .6rem;\">");
for(QRecord run : shown)
{
long value = run.getValueLong("bytesStored") == null ? 0 : run.getValueLong("bytesStored");
long pct = Math.max(4, Math.round(100.0 * value / max));
String title = (run.getValueInstant("startDate") == null ? "" : WHEN.format(run.getValueInstant("startDate")) + ": ") + bytes(run.getValueLong("bytesStored"));
html.append("<span title=\"").append(esc(title)).append("\" style=\"flex:1 1 0;min-width:2px;max-width:1.5rem;height:").append(pct).append("%;background:currentColor;opacity:.55;border-radius:2px 2px 0 0;\"></span>");
}
html.append("</div>");
}
/*******************************************************************************
** Recent activity: the last runs, restores and verifications.
*******************************************************************************/
private static void recentActivity(StringBuilder html, BackupsHomeData data, String prefix)
{
open(html, "Recent activity");
if(data.recentRuns().isEmpty() && data.restores().isEmpty() && data.verifications().isEmpty())
{
empty(html, "Nothing has run yet.");
}
else
{
html.append("<ul style=\"").append(ITEMS).append("\">");
data.recentRuns().stream().limit(6).forEach(run ->
{
Instant start = run.getValueInstant("startDate");
html.append("<li>Backup ").append(recordLink(prefix, BackupsTables.RUN, run.getValue("id"), run.getValueString("runKey")))
.append(": ").append(esc(label(run.getValueString("status")))).append(start == null ? "" : ", " + esc(WHEN.format(start))).append("</li>");
});
data.restores().stream().limit(2).forEach(restore -> html.append("<li>Restore ").append(recordLink(prefix, BackupsTables.RESTORE_RUN, restore.getValue("id"), "#" + restore.getValue("id")))
.append(": ").append(esc(label(restore.getValueString("status")))).append("</li>"));
data.verifications().stream().limit(2).forEach(v -> html.append("<li>Verify ").append(recordLink(prefix, BackupsTables.VERIFICATION, v.getValue("id"), label(v.getValueString("level"))))
.append(": ").append(esc(label(v.getValueString("status")))).append("</li>"));
html.append("</ul>");
}
link(html, prefix, BackupsTables.RUN, "All runs");
close(html);
}
/*******************************************************************************
** Retention: runs kept per class and the oldest restorable point.
*******************************************************************************/
private static void retention(StringBuilder html, BackupsHomeData data)
{
open(html, "Retention");
boolean any = false;
for(QRecord plan : data.plans())
{
Map<String, Integer> counts = data.retentionCounts(plan.getValueInteger("id"));
if(counts.isEmpty())
{
continue;
}
any = true;
html.append("<dl style=\"").append(LIST).append("\">");
row(html, "Plan", plan.getValueString("name"));
row(html, "Kept", counts.entrySet().stream().map(e -> e.getValue() + " " + ("UNCLASSIFIED".equals(e.getKey()) ? "not classified yet (no prune has run)" : label(e.getKey()).toLowerCase()))
.reduce((a, b) -> a + ", " + b).orElse(""));
List<QRecord> runs = data.successfulRuns(plan.getValueInteger("id"));
Instant oldest = runs.isEmpty() ? null : runs.get(0).getValueInstant("startDate");
row(html, "Oldest restorable", oldest == null ? "Not known" : WHEN.format(oldest));
html.append("</dl>");
}
if(!any)
{
empty(html, "Retention starts once runs exist.");
}
close(html);
}
/*******************************************************************************
** Open a card.
*******************************************************************************/
private static void open(StringBuilder html, String title)
{
html.append("<section style=\"").append(CARD).append("\"><h3 style=\"margin:0 0 .75rem;font-size:1rem;font-weight:600;\">").append(esc(title)).append("</h3>");
}
/*******************************************************************************
** Close a card.
*******************************************************************************/
private static void close(StringBuilder html)
{
html.append("</section>");
}
/*******************************************************************************
** A muted empty-state line.
*******************************************************************************/
private static void empty(StringBuilder html, String text)
{
html.append("<p style=\"").append(MUTED).append("margin:0 0 .75rem;font-size:.9rem;\">").append(esc(text)).append("</p>");
}
/*******************************************************************************
** A name and value pair in a definition list.
*******************************************************************************/
private static void row(StringBuilder html, String name, String value)
{
html.append("<dt style=\"").append(MUTED).append("\">").append(esc(name)).append("</dt><dd style=\"margin:0;\">").append(esc(value == null ? "" : value)).append("</dd>");
}
/*******************************************************************************
** A small label over its value (a plan row cell).
*******************************************************************************/
private static void cell(StringBuilder html, String name, String value)
{
html.append("<div style=\"min-width:0;\"><div style=\"").append(MUTED).append("font-size:.75rem;\">").append(esc(name)).append("</div><div>").append(esc(value)).append("</div></div>");
}
/*******************************************************************************
** A status chip: a colored dot (the only color) and the status label.
*******************************************************************************/
private static String chip(String status)
{
String color = switch(status == null ? "UNKNOWN" : status)
{
case "OK" -> "#16a34a";
case "LATE", "UNVERIFIED", "UNKNOWN" -> "#d97706";
default -> "#dc2626";
};
return ("<span style=\"" + CHIP + "\"><span aria-hidden=\"true\" style=\"width:.55rem;height:.55rem;border-radius:50%;background:" + color + ";display:inline-block;\"></span>" + esc(label(status)) + "</span>");
}
/*******************************************************************************
** A link to the table list at the end of a card.
*******************************************************************************/
private static void link(StringBuilder html, String prefix, String table, String text)
{
html.append("<a style=\"").append(LINK).append("font-size:.9rem;\" href=\"").append(esc(prefix + "/" + table)).append("\">").append(esc(text)).append("</a>");
}
/*******************************************************************************
** A link to a record.
*******************************************************************************/
private static String recordLink(String prefix, String table, Object id, String text)
{
return ("<a style=\"" + LINK + "\" href=\"" + esc(prefix + "/" + table + "/" + id) + "\">" + esc(text == null ? String.valueOf(id) : text) + "</a>");
}
/*******************************************************************************
** SOME_STATUS to Some status (OK stays OK).
*******************************************************************************/
static String label(String status)
{
if(status == null || status.isEmpty())
{
return ("Unknown");
}
if("OK".equals(status))
{
return ("OK");
}
String s = status.replace('_', ' ').toLowerCase();
return (Character.toUpperCase(s.charAt(0)) + s.substring(1));
}
/*******************************************************************************
** An Integer as a Long, null-safe.
*******************************************************************************/
private static Long toLong(Integer value)
{
return (value == null ? null : value.longValue());
}
/*******************************************************************************
** Milliseconds as a short duration.
*******************************************************************************/
static String duration(long millis)
{
if(millis < 1000)
{
return (millis + " ms");
}
long seconds = millis / 1000;
return (seconds < 60 ? seconds + " s" : seconds / 60 + " min " + seconds % 60 + " s");
}
/*******************************************************************************
** Bytes as a short size.
*******************************************************************************/
static String bytes(Long value)
{
if(value == null)
{
return ("Not recorded");
}
double v = value;
String[] units = { "B", "KB", "MB", "GB", "TB" };
int i = 0;
while(v >= 1024 && i < units.length - 1)
{
v /= 1024;
i++;
}
return (i == 0 ? String.format("%d B", value) : String.format("%.1f %s", v, units[i]));
}
/*******************************************************************************
** A number with separators.
*******************************************************************************/
static String number(Long value)
{
return (value == null ? "Not recorded" : String.format("%,d", value));
}
/*******************************************************************************
** HTML-escape.
*******************************************************************************/
static String esc(String s)
{
return (s == null ? "" : s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace("\"", "&quot;"));
}
}
@@ -1,86 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.codec;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
/*******************************************************************************
** A reversible transform applied to the bytes of an artifact after compression,
** before they reach storage. The only implementation is client-side encryption
** (the streaming cipher of the QQQ crypto package); IDENTITY is the default.
*******************************************************************************/
public interface ArtifactTransform
{
/** No transform. */
ArtifactTransform IDENTITY = new ArtifactTransform()
{
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public OutputStream wrap(OutputStream out)
{
return (out);
}
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public InputStream unwrap(InputStream in)
{
return (in);
}
};
/***************************************************************************
** Wrap the stream going to storage.
***************************************************************************/
OutputStream wrap(OutputStream out) throws IOException;
/***************************************************************************
** Wrap a stream read from storage, to undo the transform.
***************************************************************************/
InputStream unwrap(InputStream in) throws IOException;
/***************************************************************************
** Whether this transform encrypts.
***************************************************************************/
default boolean encrypts()
{
return (false);
}
/***************************************************************************
** File name suffix added to artifact keys, e.g. ".tink".
***************************************************************************/
default String keySuffix()
{
return ("");
}
}
@@ -1,277 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.codec;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeSet;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
/*******************************************************************************
** The CSV row format (spec 8.3): RFC 4180, UTF-8 without a BOM, a header of
** field names, and the same canonical value text as JSONL. Null is written as
** \N (unquoted) so it can be told from the empty string; a string that is
** literally \N is quoted. A multi-line value is one record in a quoted field,
** so the codec works on records (a record may span lines): encode() returns a
** text that may contain embedded newlines, and the part writer appends one
** newline after each.
*******************************************************************************/
public class CsvRowCodec implements RowCodec
{
public static final String NULL_TOKEN = "\\N";
private final QTableMetaData table;
private final List<String> fieldNames;
/*******************************************************************************
** Constructor.
*******************************************************************************/
public CsvRowCodec(QTableMetaData table, java.util.Collection<String> excludedFields)
{
this.table = table;
TreeSet<String> names = new TreeSet<>(table.getFields().keySet());
if(excludedFields != null)
{
names.removeAll(excludedFields);
}
this.fieldNames = new ArrayList<>(names);
}
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public List<String> getFieldNames()
{
return (fieldNames);
}
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public String header()
{
return (String.join(",", fieldNames.stream().map(CsvRowCodec::quoteIfNeeded).toList()));
}
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public String encode(QRecord record)
{
StringBuilder sb = new StringBuilder();
for(int i = 0; i < fieldNames.size(); i++)
{
if(i > 0)
{
sb.append(',');
}
String name = fieldNames.get(i);
Serializable value = record.getValue(name);
if(value == null)
{
sb.append(NULL_TOKEN);
}
else
{
String text = FieldValueCodec.toCanonicalText(table.getField(name).getType(), value);
sb.append(NULL_TOKEN.equals(text) ? "\"" + text + "\"" : quoteIfNeeded(text));
}
}
return (sb.toString());
}
/*******************************************************************************
** Parse one CSV record (as returned by CsvRecordReader, header excluded) to a record.
** The line may contain embedded newlines inside quoted fields.
*******************************************************************************/
@Override
public QRecord decode(String line) throws QException
{
List<String> cells = splitCells(line);
return (toRecord(cells, fieldNames));
}
/*******************************************************************************
** Build a record from already-split cells and the header names.
*******************************************************************************/
public QRecord toRecord(List<String> cells, List<String> header) throws QException
{
if(cells.size() != header.size())
{
throw (new QException("CSV row has " + cells.size() + " cells but the header has " + header.size() + " for table " + table.getName()));
}
Map<String, Serializable> values = new LinkedHashMap<>();
for(int i = 0; i < header.size(); i++)
{
String cell = cells.get(i);
QFieldMetaData field = table.getFields().get(header.get(i));
if(cell == null)
{
values.put(header.get(i), null);
}
else if(field == null)
{
values.put(header.get(i), cell);
}
else
{
values.put(header.get(i), FieldValueCodec.fromCanonicalText(field.getType(), cell));
}
}
QRecord record = new QRecord();
record.setTableName(table.getName());
record.setValues(values);
return (record);
}
/*******************************************************************************
** Quote a cell when it contains a comma, quote, CR or LF (or is empty after
** being a header, which never happens), doubling embedded quotes.
*******************************************************************************/
static String quoteIfNeeded(String s)
{
if(s.indexOf(',') >= 0 || s.indexOf('"') >= 0 || s.indexOf('\n') >= 0 || s.indexOf('\r') >= 0)
{
return ("\"" + s.replace("\"", "\"\"") + "\"");
}
return (s);
}
/*******************************************************************************
** Split one CSV record into cells. Returns null for an unquoted \N cell (a null).
*******************************************************************************/
public static List<String> splitCells(String line)
{
List<String> cells = new ArrayList<>();
StringBuilder cell = new StringBuilder();
boolean quoted = false;
boolean wasQuoted = false;
for(int i = 0; i < line.length(); i++)
{
char c = line.charAt(i);
if(quoted)
{
if(c == '"')
{
if(i + 1 < line.length() && line.charAt(i + 1) == '"')
{
cell.append('"');
i++;
}
else
{
quoted = false;
}
}
else
{
cell.append(c);
}
}
else if(c == '"')
{
quoted = true;
wasQuoted = true;
}
else if(c == ',')
{
cells.add(finishCell(cell, wasQuoted));
cell = new StringBuilder();
wasQuoted = false;
}
else
{
cell.append(c);
}
}
cells.add(finishCell(cell, wasQuoted));
return (cells);
}
/*******************************************************************************
**
*******************************************************************************/
private static String finishCell(StringBuilder cell, boolean wasQuoted)
{
String s = cell.toString();
return (!wasQuoted && NULL_TOKEN.equals(s) ? null : s);
}
/*******************************************************************************
** Split a whole CSV text into records, honoring newlines inside quoted fields.
*******************************************************************************/
public static List<String> splitRecords(String text)
{
List<String> records = new ArrayList<>();
StringBuilder current = new StringBuilder();
boolean quoted = false;
for(int i = 0; i < text.length(); i++)
{
char c = text.charAt(i);
if(c == '"')
{
quoted = !quoted;
current.append(c);
}
else if(c == '\n' && !quoted)
{
records.add(current.toString());
current = new StringBuilder();
}
else
{
current.append(c);
}
}
if(current.length() > 0)
{
records.add(current.toString());
}
return (records);
}
}
@@ -1,172 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.codec;
import java.io.Serializable;
import java.math.BigDecimal;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.Base64;
import com.fasterxml.jackson.databind.JsonNode;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
** The canonical, lossless text encoding of a field value, by QFieldType
** (spec 8.2). The same canonical text is used by JSONL and CSV, so a value
** means the same thing in both, and by the content hash.
**
** STRING, TEXT, HTML, PASSWORD: string
** INTEGER, LONG: number
** DECIMAL: string (BigDecimal.toPlainString), so there is no float rounding
** BOOLEAN: true or false
** DATE: yyyy-MM-dd
** TIME: HH:mm:ss, with .SSSSSS only when there are fractional seconds
** DATE_TIME: an ISO-8601 instant in UTC, with .SSSSSS only when there are fractional seconds
** BLOB: base64
*******************************************************************************/
public final class FieldValueCodec
{
private static final DateTimeFormatter TIME_FORMAT = DateTimeFormatter.ofPattern("HH:mm:ss");
private static final DateTimeFormatter TIME_FORMAT_MICROS = DateTimeFormatter.ofPattern("HH:mm:ss.SSSSSS");
private static final DateTimeFormatter INSTANT_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'").withZone(ZoneOffset.UTC);
private static final DateTimeFormatter INSTANT_FORMAT_MICRO = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'").withZone(ZoneOffset.UTC);
/*******************************************************************************
** Utility class.
*******************************************************************************/
private FieldValueCodec()
{
}
/*******************************************************************************
** Whether the canonical JSON form of a type is a bare number or boolean (not quoted).
*******************************************************************************/
public static boolean isBareJson(QFieldType type)
{
return (type == QFieldType.INTEGER || type == QFieldType.LONG || type == QFieldType.BOOLEAN);
}
/*******************************************************************************
** The canonical text of a non-null value.
*******************************************************************************/
public static String toCanonicalText(QFieldType type, Serializable value)
{
switch(type)
{
case INTEGER, LONG:
return (String.valueOf(ValueUtils.getValueAsLong(value)));
case DECIMAL:
return (ValueUtils.getValueAsBigDecimal(value).toPlainString());
case BOOLEAN:
return (String.valueOf(ValueUtils.getValueAsBoolean(value)));
case DATE:
return (ValueUtils.getValueAsLocalDate(value).toString());
case TIME:
{
LocalTime time = ValueUtils.getValueAsLocalTime(value);
return (time.getNano() == 0 ? TIME_FORMAT.format(time) : TIME_FORMAT_MICROS.format(time));
}
case DATE_TIME:
{
Instant instant = ValueUtils.getValueAsInstant(value);
return (instant.getNano() == 0 ? INSTANT_FORMAT.format(instant) : INSTANT_FORMAT_MICRO.format(instant));
}
case BLOB:
return (Base64.getEncoder().encodeToString(blobBytes(value)));
default:
return (String.valueOf(value));
}
}
/*******************************************************************************
**
*******************************************************************************/
private static byte[] blobBytes(Serializable value)
{
if(value instanceof byte[] bytes)
{
return (bytes);
}
return (String.valueOf(value).getBytes(java.nio.charset.StandardCharsets.UTF_8));
}
/*******************************************************************************
** Parse canonical text back to a Java value.
*******************************************************************************/
public static Serializable fromCanonicalText(QFieldType type, String text)
{
if(text == null)
{
return (null);
}
switch(type)
{
case INTEGER:
return (Integer.valueOf(text));
case LONG:
return (Long.valueOf(text));
case DECIMAL:
return (new BigDecimal(text));
case BOOLEAN:
return (Boolean.valueOf(text));
case DATE:
return (LocalDate.parse(text));
case TIME:
return (LocalTime.parse(text));
case DATE_TIME:
return (Instant.parse(text));
case BLOB:
return (Base64.getDecoder().decode(text));
default:
return (text);
}
}
/*******************************************************************************
** Convert a parsed JSON value to a Java value.
*******************************************************************************/
public static Serializable fromJson(QFieldType type, JsonNode node)
{
if(node == null || node.isNull())
{
return (null);
}
if(type == QFieldType.BOOLEAN)
{
return (node.asBoolean());
}
return (fromCanonicalText(type, node.isNumber() ? node.asText() : node.textValue()));
}
}
@@ -1,147 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.codec;
import java.io.FilterOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.HexFormat;
/*******************************************************************************
** An output stream that counts and SHA-256 hashes the bytes passing through.
** Closing it closes the stream underneath. The hash is available after close()
** (or any time, as the hash of the bytes so far).
*******************************************************************************/
public class HashingOutputStream extends FilterOutputStream
{
private final MessageDigest digest;
private long count = 0;
private String hex;
/*******************************************************************************
** Constructor.
*******************************************************************************/
public HashingOutputStream(OutputStream out)
{
super(out);
this.digest = newSha256();
}
/*******************************************************************************
** A new SHA-256 digest.
*******************************************************************************/
public static MessageDigest newSha256()
{
try
{
return (MessageDigest.getInstance("SHA-256"));
}
catch(NoSuchAlgorithmException e)
{
throw (new IllegalStateException("SHA-256 is not available", e));
}
}
/*******************************************************************************
** Hex of a digest result.
*******************************************************************************/
public static String hex(byte[] bytes)
{
return (HexFormat.of().formatHex(bytes));
}
/*******************************************************************************
** SHA-256 of a byte array, as hex.
*******************************************************************************/
public static String sha256Hex(byte[] bytes)
{
return (hex(newSha256().digest(bytes)));
}
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public void write(int b) throws IOException
{
out.write(b);
digest.update((byte) b);
count++;
}
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public void write(byte[] b, int off, int len) throws IOException
{
out.write(b, off, len);
digest.update(b, off, len);
count += len;
}
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public void close() throws IOException
{
super.close();
if(hex == null)
{
hex = hex(digest.digest());
}
}
/*******************************************************************************
** The SHA-256 (hex) of everything written; valid after close().
*******************************************************************************/
public String getSha256()
{
return (hex);
}
/*******************************************************************************
** The number of bytes written.
*******************************************************************************/
public long getCount()
{
return (count);
}
}
@@ -1,200 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.codec;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeSet;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
/*******************************************************************************
** The JSON Lines row format (spec 8.2): one JSON object per line, keys are field
** names (sorted), values are the raw canonical encodings, never display values.
** The text of a line is canonical, so the same rows always give the same bytes
** (which is what the rowsHash is computed over).
*******************************************************************************/
public class JsonlRowCodec implements RowCodec
{
private static final ObjectMapper MAPPER = new ObjectMapper();
private final QTableMetaData table;
private final List<String> fieldNames;
/*******************************************************************************
** Constructor. The fields are the table fields, minus any excluded, sorted by name.
*******************************************************************************/
public JsonlRowCodec(QTableMetaData table, java.util.Collection<String> excludedFields)
{
this.table = table;
TreeSet<String> names = new TreeSet<>(table.getFields().keySet());
if(excludedFields != null)
{
names.removeAll(excludedFields);
}
this.fieldNames = new ArrayList<>(names);
}
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public List<String> getFieldNames()
{
return (fieldNames);
}
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public String header()
{
return (null);
}
/*******************************************************************************
** The canonical line (no trailing newline) for a record.
*******************************************************************************/
@Override
public String encode(QRecord record)
{
StringBuilder sb = new StringBuilder("{");
boolean first = true;
for(String name : fieldNames)
{
QFieldType type = table.getField(name).getType();
Serializable value = record.getValue(name);
if(!first)
{
sb.append(',');
}
first = false;
appendString(sb, name);
sb.append(':');
if(value == null)
{
sb.append("null");
}
else
{
String text = FieldValueCodec.toCanonicalText(type, value);
if(FieldValueCodec.isBareJson(type))
{
sb.append(text);
}
else
{
appendString(sb, text);
}
}
}
return (sb.append('}').toString());
}
/*******************************************************************************
** Parse one line into a record. Unknown keys are kept as strings so that
** restore can report them as unmapped.
*******************************************************************************/
@Override
public QRecord decode(String line) throws QException
{
try
{
JsonNode node = MAPPER.readTree(line);
Map<String, Serializable> values = new LinkedHashMap<>();
for(var it = node.fields(); it.hasNext(); )
{
var entry = it.next();
QFieldMetaData field = table.getFields().get(entry.getKey());
if(field == null)
{
values.put(entry.getKey(), entry.getValue().isNull() ? null : entry.getValue().asText());
}
else
{
values.put(entry.getKey(), FieldValueCodec.fromJson(field.getType(), entry.getValue()));
}
}
QRecord record = new QRecord();
record.setTableName(table.getName());
record.setValues(values);
return (record);
}
catch(Exception e)
{
throw (new QException("Could not read a backup row for table " + table.getName() + ": " + e.getMessage()));
}
}
/*******************************************************************************
** Append a JSON string with escapes. Control characters, quotes and backslashes
** are escaped; everything else (including non-ASCII) is written as is, in UTF-8.
*******************************************************************************/
static void appendString(StringBuilder sb, String s)
{
sb.append('"');
for(int i = 0; i < s.length(); i++)
{
char c = s.charAt(i);
switch(c)
{
case '"' -> sb.append("\\\"");
case '\\' -> sb.append("\\\\");
case '\n' -> sb.append("\\n");
case '\r' -> sb.append("\\r");
case '\t' -> sb.append("\\t");
case '\b' -> sb.append("\\b");
case '\f' -> sb.append("\\f");
default ->
{
if(c < 0x20)
{
sb.append(String.format("\\u%04x", (int) c));
}
else
{
sb.append(c);
}
}
}
}
sb.append('"');
}
}
@@ -1,53 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.codec;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
/*******************************************************************************
** Encodes a record to one line of text, and decodes it back.
*******************************************************************************/
public interface RowCodec
{
/***************************************************************************
** The field names written, in order.
***************************************************************************/
List<String> getFieldNames();
/***************************************************************************
** A header line to write at the start of each part, or null.
***************************************************************************/
String header();
/***************************************************************************
** One line, without the trailing newline.
***************************************************************************/
String encode(QRecord record);
/***************************************************************************
** Read one line back into a record.
***************************************************************************/
QRecord decode(String line) throws QException;
}
@@ -1,346 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.codec;
import java.io.Closeable;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.function.Consumer;
import java.util.zip.GZIPOutputStream;
import com.kingsrook.qbits.backups.storage.BackupStorage;
import com.kingsrook.qbits.backups.storage.WriteOptions;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
/*******************************************************************************
** Streams the rows of one table into numbered parts in storage. Each row goes
** through the row codec, then (per part) into a hash of the raw bytes, gzip,
** the artifact transform (encryption) and a hash of the stored bytes, and out
** through BackupStorage.openWrite. Memory is O(row); nothing touches a temp disk.
**
** Whatever the output format, a canonical JSONL hash of every row (in the order
** written, which is primary key order) is kept: that is the rowsHash a verifier
** recomputes (spec 8.2), independent of compression, encryption and part sizes.
*******************************************************************************/
public class TableArtifactWriter implements Closeable
{
private final BackupStorage storage;
private final String keyPrefix;
private final QTableMetaData table;
private final RowCodec codec;
private final JsonlRowCodec canonical;
private final boolean csv;
private final boolean gzip;
private final long targetPartBytes;
private final long maxRowsPerPart;
private final ArtifactTransform transform;
private final String runKey;
private final MessageDigest rowsDigest = HashingOutputStream.newSha256();
private final List<PartResult> parts = new ArrayList<>();
private int partNumber = 0;
private PartState current;
private long totalRows = 0;
private String firstKeyOfTable;
private String lastKeyOfTable;
private Consumer<PartResult> partListener;
private String objectLockMode;
private Instant objectLockUntil;
/***************************************************************************
** The outcome of one written part.
***************************************************************************/
public record PartResult(int partNumber, String key, long rows, long bytesRaw, long bytesStored, String sha256Raw, String sha256Stored,
String firstKey, String lastKey, boolean encrypted, String format, String compression)
{
}
/***************************************************************************
**
***************************************************************************/
private static class PartState
{
String key;
HashingOutputStream raw;
HashingOutputStream stored;
long rows;
String firstKey;
String lastKey;
}
/*******************************************************************************
** Constructor.
**
** @param keyPrefix the key prefix of the table directory, ending in a slash
** @param format JSONL or CSV
** @param compression GZIP or NONE
*******************************************************************************/
public TableArtifactWriter(BackupStorage storage, String keyPrefix, QTableMetaData table, Collection<String> excludedFields, String format,
String compression, long targetPartBytes, long maxRowsPerPart, ArtifactTransform transform, String runKey)
{
this.storage = storage;
this.keyPrefix = keyPrefix;
this.table = table;
this.csv = "CSV".equals(format);
this.canonical = new JsonlRowCodec(table, excludedFields);
this.codec = csv ? new CsvRowCodec(table, excludedFields) : canonical;
this.gzip = "GZIP".equals(compression);
this.targetPartBytes = targetPartBytes;
this.maxRowsPerPart = maxRowsPerPart;
this.transform = transform == null ? ArtifactTransform.IDENTITY : transform;
this.runKey = runKey;
if(!gzip && !"NONE".equals(compression))
{
throw (new IllegalArgumentException("Unsupported compression: " + compression));
}
}
/*******************************************************************************
** Be told about each part as it is committed to storage (so the catalog records
** a part the moment it is complete, which is what a resume starts from).
*******************************************************************************/
public TableArtifactWriter withPartListener(Consumer<PartResult> partListener)
{
this.partListener = partListener;
return (this);
}
/*******************************************************************************
** Apply an object lock to every part written.
*******************************************************************************/
public TableArtifactWriter withObjectLock(String mode, Instant until)
{
this.objectLockMode = mode;
this.objectLockUntil = until;
return (this);
}
/*******************************************************************************
** Continue after parts that were already written (resume): numbering carries on
** after them and the table totals include them. The caller then feeds the rows
** hash with seedRowsHash.
*******************************************************************************/
public void resumeFrom(List<PartResult> priorParts)
{
if(priorParts.isEmpty())
{
return;
}
parts.addAll(priorParts);
partNumber = priorParts.get(priorParts.size() - 1).partNumber();
for(PartResult part : priorParts)
{
totalRows += part.rows();
}
firstKeyOfTable = priorParts.get(0).firstKey();
lastKeyOfTable = priorParts.get(priorParts.size() - 1).lastKey();
}
/*******************************************************************************
** Feed already-written canonical JSONL bytes into the rows hash (resume).
*******************************************************************************/
public void seedRowsHash(byte[] bytes, int offset, int length)
{
rowsDigest.update(bytes, offset, length);
}
/*******************************************************************************
** Write one record.
*******************************************************************************/
public void write(QRecord record) throws IOException
{
if(current == null)
{
openPart();
}
String pk = String.valueOf(record.getValue(table.getPrimaryKeyField()));
byte[] line = (codec.encode(record) + "\n").getBytes(StandardCharsets.UTF_8);
current.raw.write(line);
if(current.firstKey == null)
{
current.firstKey = pk;
}
current.lastKey = pk;
current.rows++;
totalRows++;
if(firstKeyOfTable == null)
{
firstKeyOfTable = pk;
}
lastKeyOfTable = pk;
rowsDigest.update((canonical.encode(record) + "\n").getBytes(StandardCharsets.UTF_8));
if(current.raw.getCount() >= targetPartBytes || current.rows >= maxRowsPerPart)
{
closePart();
}
}
/*******************************************************************************
** Finish: close the last part (writing an empty part if there were no rows at all).
*******************************************************************************/
public List<PartResult> finish() throws IOException
{
if(current == null && parts.isEmpty())
{
openPart();
}
if(current != null)
{
closePart();
}
return (parts);
}
/*******************************************************************************
** The canonical content hash of all rows written, as sha256:hex. Valid after finish().
*******************************************************************************/
public String rowsHash()
{
return ("sha256:" + HashingOutputStream.hex(rowsDigest.digest()));
}
/*******************************************************************************
** Total rows written.
*******************************************************************************/
public long getTotalRows()
{
return (totalRows);
}
/*******************************************************************************
** First primary key written.
*******************************************************************************/
public String getFirstKey()
{
return (firstKeyOfTable);
}
/*******************************************************************************
** Last primary key written.
*******************************************************************************/
public String getLastKey()
{
return (lastKeyOfTable);
}
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public void close() throws IOException
{
if(current != null)
{
// an abandoned part is never committed to storage
current.raw.close();
current = null;
}
}
/*******************************************************************************
**
*******************************************************************************/
private void openPart() throws IOException
{
partNumber++;
current = new PartState();
String extension = csv ? "csv" : "jsonl";
current.key = keyPrefix + String.format("part-%05d.%s%s%s", partNumber, extension, gzip ? ".gz" : "", transform.keySuffix());
WriteOptions options = WriteOptions.create().withContentType("application/octet-stream")
.withMetadata("qqq-backup-run", runKey)
.withMetadata("qqq-backup-table", table.getName());
if(objectLockMode != null)
{
options.withObjectLock(objectLockMode, objectLockUntil);
}
OutputStream toStorage = storage.openWrite(current.key, options);
current.stored = new HashingOutputStream(toStorage);
OutputStream transformed = transform.wrap(current.stored);
OutputStream compressed = gzip ? new GZIPOutputStream(transformed) : transformed;
current.raw = new HashingOutputStream(compressed);
String header = codec.header();
if(header != null)
{
current.raw.write((header + "\n").getBytes(StandardCharsets.UTF_8));
}
}
/*******************************************************************************
**
*******************************************************************************/
private void closePart() throws IOException
{
current.raw.close();
PartResult result = new PartResult(partNumber, current.key, current.rows, current.raw.getCount(), current.stored.getCount(),
current.raw.getSha256(), current.stored.getSha256(), current.firstKey, current.lastKey, transform.encrypts(),
csv ? "CSV" : "JSONL", gzip ? "GZIP" : "NONE");
parts.add(result);
current = null;
if(partListener != null)
{
partListener.accept(result);
}
}
}
@@ -1,146 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.events;
import java.io.Serializable;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.function.BiConsumer;
import com.kingsrook.qbits.backups.BackupsRuntime;
import com.kingsrook.qqq.backend.core.logging.QLogger;
/*******************************************************************************
** ESB events of the Backups qbit (design spec section 10.4): qqq.backups.run.succeeded,
** qqq.backups.run.failed and qqq.backups.restore.completed. They are for
** integrations, not alerting, and are FIRE AND FORGET: a failed publish is logged
** at most once and never fails a backup or a restore.
**
** Events go out only when BackupsQBitConfig.esbDestinationName is set and the ESB
** (qqq-esb, an optional dependency) is on the class path; otherwise publish() does
** nothing. A test, or an app with its own bus, can bind a publisher with
** {@link #setPublisher}.
*******************************************************************************/
public final class BackupEvents
{
private static final QLogger LOG = QLogger.getLogger(BackupEvents.class);
public static final String RUN_SUCCEEDED = "qqq.backups.run.succeeded";
public static final String RUN_FAILED = "qqq.backups.run.failed";
public static final String RESTORE_COMPLETED = "qqq.backups.restore.completed";
private static volatile BiConsumer<String, Map<String, Serializable>> publisher;
private static volatile boolean warned = false;
/*******************************************************************************
** Utility class.
*******************************************************************************/
private BackupEvents()
{
}
/*******************************************************************************
** Bind a publisher (event type, data), or null to go back to the ESB default.
*******************************************************************************/
public static void setPublisher(BiConsumer<String, Map<String, Serializable>> newPublisher)
{
publisher = newPublisher;
warned = false;
}
/*******************************************************************************
** A run finished. Succeeded and warnings publish run.succeeded; failed, timed
** out and abandoned publish run.failed; cancelled publishes nothing.
*******************************************************************************/
public static void runFinished(String planKey, String runKey, String status, long rows, long bytesStored)
{
String type;
if("SUCCEEDED".equals(status) || "SUCCEEDED_WITH_WARNINGS".equals(status))
{
type = RUN_SUCCEEDED;
}
else if("CANCELLED".equals(status))
{
return;
}
else
{
type = RUN_FAILED;
}
Map<String, Serializable> data = new LinkedHashMap<>();
data.put("plan", planKey);
data.put("run", runKey);
data.put("status", status);
data.put("rows", rows);
data.put("bytesStored", bytesStored);
publish(type, data);
}
/*******************************************************************************
** A restore finished (any status).
*******************************************************************************/
public static void restoreCompleted(String restoreRunKey, String status, long rowsRestored)
{
Map<String, Serializable> data = new LinkedHashMap<>();
data.put("restoreRun", restoreRunKey);
data.put("status", status);
data.put("rows", rowsRestored);
publish(RESTORE_COMPLETED, data);
}
/*******************************************************************************
** Publish an event. Never throws.
*******************************************************************************/
public static void publish(String type, Map<String, Serializable> data)
{
try
{
BiConsumer<String, Map<String, Serializable>> bound = publisher;
if(bound != null)
{
bound.accept(type, data);
return;
}
String destination = BackupsRuntime.getConfig() == null ? null : BackupsRuntime.getConfig().getEsbDestinationName();
if(destination == null)
{
return;
}
EsbBackupEventPublisher.publish(destination, type, data);
}
catch(Exception | LinkageError e)
{
if(!warned)
{
warned = true;
LOG.warn("A backup event could not be published (events are best effort)", e);
}
}
}
}
@@ -1,54 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.events;
import java.io.Serializable;
import java.util.Map;
import com.kingsrook.qqq.esb.publish.EsbPublishAction;
import com.kingsrook.qqq.esb.publish.EsbPublishInput;
/*******************************************************************************
** Publishes a backup event on the QQQ ESB. This is the only class that touches
** qqq-esb (an optional dependency); BackupEvents calls it inside a guard that
** swallows a missing class (NoClassDefFoundError), so apps without the ESB never
** notice.
*******************************************************************************/
final class EsbBackupEventPublisher
{
/*******************************************************************************
** Utility class.
*******************************************************************************/
private EsbBackupEventPublisher()
{
}
/*******************************************************************************
** Publish one event to a destination.
*******************************************************************************/
static void publish(String destinationName, String type, Map<String, Serializable> data)
{
new EsbPublishAction().execute(new EsbPublishInput()
.withDestinationName(destinationName)
.withType(type)
.withSourcePath("/backups")
.withData(data));
}
}
@@ -1,242 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.files;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Optional;
import java.util.function.Function;
import com.kingsrook.qbits.backups.codec.ArtifactTransform;
import com.kingsrook.qbits.backups.codec.HashingOutputStream;
import com.kingsrook.qbits.backups.storage.BackupStorage;
import com.kingsrook.qbits.backups.storage.StoredObjectInfo;
import com.kingsrook.qbits.backups.storage.WriteOptions;
/*******************************************************************************
** Makes sure each file that a record references is in the content-addressed
** store (spec 8.5), and says what it did.
**
** For each reference: head the source (missing is reported, never silently
** skipped); find its SHA-256 (known from the reference provider, else the object
** is read once and hashed); if the store already has that hash the file is
** DEDUPLICATED and nothing is copied; otherwise it is copied (server-side when
** the source and the destination are the same storage and the plan does not
** encrypt, else streamed through the transform) and the hash observed while
** copying must equal the hash the key was derived from. If the source changed
** in between, the file is hashed again and copied once more.
*******************************************************************************/
public class FileCopier
{
public static final String COPIED = "COPIED";
public static final String DEDUPLICATED = "DEDUPLICATED";
public static final String MISSING = "MISSING";
public static final String REFERENCE_ONLY = "REFERENCE_ONLY";
private final Function<String, BackupStorage> sourceResolver;
private final String destinationAlias;
private final BackupStorage destination;
private final String casPrefix;
private final ArtifactTransform transform;
private final FileStoreRegistry registry;
private final boolean referenceOnly;
private final String runKey;
private String objectLockMode;
private java.time.Instant objectLockUntil;
/***************************************************************************
** The catalog side of the file store: which hashes are already stored.
***************************************************************************/
public interface FileStoreRegistry
{
/***************************************************************************
** Whether the store knows this hash (and the object is expected to be there).
***************************************************************************/
boolean isKnown(String sha256);
/***************************************************************************
** Record a newly stored file.
***************************************************************************/
void register(String sha256, long sizeBytes, String contentType, String casKey, boolean encrypted);
}
/***************************************************************************
** One line of the per-run file index.
**
** @param tableName the referencing table
** @param recordKey the referencing record primary key
** @param field the referencing field
** @param sourceBucketAlias where the file was
** @param sourceKey the source key
** @param sha256 content hash (null when missing or reference-only without a known hash)
** @param size size in bytes (-1 when unknown)
** @param status COPIED, DEDUPLICATED, MISSING or REFERENCE_ONLY
** @param casKey the content-addressed key in the destination, if stored
***************************************************************************/
public record IndexEntry(String tableName, String recordKey, String field, String sourceBucketAlias, String sourceKey,
String sha256, long size, String status, String casKey)
{
}
/*******************************************************************************
** Constructor.
**
** @param casPrefix prefix of the store, ending in a slash, e.g. qqq-backups/app/_files/sha256/
*******************************************************************************/
public FileCopier(Function<String, BackupStorage> sourceResolver, String destinationAlias, BackupStorage destination, String casPrefix,
ArtifactTransform transform, FileStoreRegistry registry, boolean referenceOnly, String runKey)
{
this.sourceResolver = sourceResolver;
this.destinationAlias = destinationAlias;
this.destination = destination;
this.casPrefix = casPrefix;
this.transform = transform == null ? ArtifactTransform.IDENTITY : transform;
this.registry = registry;
this.referenceOnly = referenceOnly;
this.runKey = runKey;
}
/*******************************************************************************
** Lock every object this copier writes (a server-side copy cannot set a lock,
** so with a lock the bytes are streamed instead).
*******************************************************************************/
public FileCopier withObjectLock(String mode, java.time.Instant until)
{
this.objectLockMode = mode;
this.objectLockUntil = until;
return (this);
}
/*******************************************************************************
** The content-addressed key for a hash: prefix/ab/cd/sha256.
*******************************************************************************/
public static String casKey(String casPrefix, String sha256)
{
return (casPrefix + sha256.substring(0, 2) + "/" + sha256.substring(2, 4) + "/" + sha256);
}
/*******************************************************************************
** Process one reference.
*******************************************************************************/
public IndexEntry process(FileReference ref) throws IOException
{
BackupStorage source = sourceResolver.apply(ref.bucketAlias());
Optional<StoredObjectInfo> head = source == null ? Optional.empty() : source.head(ref.key());
if(head.isEmpty())
{
return (entry(ref, null, -1, MISSING, null));
}
long size = head.get().size();
if(referenceOnly)
{
return (entry(ref, ref.knownSha256(), size, REFERENCE_ONLY, null));
}
String sha = ref.knownSha256() != null ? ref.knownSha256() : hashOf(source, ref.key());
for(int attempt = 0; attempt < 2; attempt++)
{
String key = casKey(casPrefix, sha);
if(registry.isKnown(sha) && destination.head(key).isPresent())
{
return (entry(ref, sha, size, DEDUPLICATED, key));
}
String observed = copy(source, ref, key);
if(observed == null || observed.equals(sha))
{
registry.register(sha, size, ref.contentType(), key, transform.encrypts());
return (entry(ref, sha, size, COPIED, key));
}
// the source changed while it was being copied: the object we wrote is under the wrong key
destination.deletePermanent(key);
sha = observed;
}
throw (new IOException("The file " + ref.key() + " kept changing while it was being copied"));
}
/*******************************************************************************
** Copy the file to the store; returns the hash observed while streaming, or null
** for a server-side copy (which cannot observe the bytes).
*******************************************************************************/
private String copy(BackupStorage source, FileReference ref, String key) throws IOException
{
if(source == destination && !transform.encrypts() && objectLockMode == null)
{
destination.copy(ref.key(), key);
return (null);
}
WriteOptions options = WriteOptions.create().withContentType(ref.contentType() == null ? "application/octet-stream" : ref.contentType())
.withMetadata("qqq-backup-run", runKey);
if(objectLockMode != null)
{
options.withObjectLock(objectLockMode, objectLockUntil);
}
try(InputStream in = source.openRead(ref.key());
HashingOutputStream plain = new HashingOutputStream(transform.wrap(destination.openWrite(key, options))))
{
in.transferTo(plain);
plain.flush();
plain.close();
return (plain.getSha256());
}
}
/*******************************************************************************
** Read an object once, to find its SHA-256.
*******************************************************************************/
private String hashOf(BackupStorage source, String key) throws IOException
{
try(InputStream in = source.openRead(key);
HashingOutputStream sink = new HashingOutputStream(OutputStream.nullOutputStream()))
{
in.transferTo(sink);
sink.close();
return (sink.getSha256());
}
}
/*******************************************************************************
**
*******************************************************************************/
private IndexEntry entry(FileReference ref, String sha, long size, String status, String casKey)
{
return (new IndexEntry(ref.tableName(), ref.recordKey(), ref.field(), ref.bucketAlias(), ref.key(), sha, size, status, casKey));
}
}
Loaded 100 of 202 files, more files were not shown because too many files have changed in this diff. Show more