ci: centralize QBit source scanning

Co-Authored-By: Codex <noreply@openai.com>

.circleci/config.yml

@@ -1,71 +1,14 @@
 version: 2.1
-
+setup: true
 orbs:
-   qqq-orb: kingsrook/qqq-orb@0.6.8
-
-commands:
-   configure-package-install:
-      steps:
-         - run:
-              name: Keep package installation unattended
-              command: |
-                 # These build machines are discarded after the job.
-                 sudo install -d /etc/needrestart/conf.d
-                 printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
-
+  qqq-orb: kingsrook/qqq-orb@0.7.12
 workflows:
-   test_only:
-      jobs:
-         - qqq-orb/mvn_test_only:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              filters:
-                 branches:
-                    ignore: /(develop|release\/.*|hotfix\/.*)/
-
-   publish_snapshot:
-      jobs:
-         - qqq-orb/mvn_publish:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              branch_type: snapshot
-              filters:
-                 branches:
-                    only: [develop]
-
-   publish_release_candidate:
-      jobs:
-         - qqq-orb/mvn_publish:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              branch_type: release_candidate
-              filters:
-                 branches:
-                    only: [/release\/.*/]
-
-   publish_release:
-      jobs:
-         - qqq-orb/mvn_publish:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              branch_type: release
-              filters:
-                 branches:
-                    ignore: /.*/
-                 tags:
-                    only: [/^v\d+\.\d+\.\d+$/]
-
-   publish_hotfix_release:
-      jobs:
-         - qqq-orb/mvn_publish:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              branch_type: hotfix
-              filters:
-                 branches:
-                    only: [/hotfix\/.*/]
+  setup:
+    jobs:
+    - qqq-orb/generate_pipeline:
+        orb_version: 0.7.12
+        filters:
+          branches:
+            ignore: automation/qbit-rc-pending
+          tags:
+            only: /^v\d+\.\d+\.\d+$/

.github/workflows/codeql.yml

@@ -0,0 +1,23 @@
+name: CodeQL
+'on':
+  push:
+    branches:
+    - main
+    - develop
+    - release/**
+    - hotfix/**
+  pull_request: {}
+  schedule:
+  - cron: 41 16 * * 0
+  workflow_dispatch: {}
+permissions:
+  contents: read
+  actions: read
+  security-events: write
+  packages: read
+concurrency:
+  group: codeql-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+jobs:
+  analyze:
+    uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22

.github/workflows/release-rc.yml

@@ -0,0 +1,74 @@
+name: Release candidate
+'on':
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: Plan a cut without changing release state
+        type: boolean
+        default: false
+      finalize:
+        description: Retry finalization of an existing cut; do not cut another RC
+        type: boolean
+        default: false
+      recover:
+        description: Recover the same published RC after a deployment timeout; never
+          reupload
+        type: boolean
+        default: false
+      circle_build:
+        description: Original Circle publisher build number (recovery only)
+        type: string
+        default: ''
+      merge_sha:
+        description: Existing cut merge_sha (finalize retry only)
+        type: string
+        default: ''
+      bump_sha:
+        description: Existing cut bump_sha (finalize retry only)
+        type: string
+        default: ''
+      branch:
+        description: Existing cut branch (finalize retry only)
+        type: string
+        default: ''
+      version:
+        description: Existing cut version (finalize retry only)
+        type: string
+        default: ''
+  pull_request:
+    types:
+    - closed
+    branches:
+    - develop
+  repository_dispatch:
+    types:
+    - qbit-rc-published
+permissions:
+  contents: write
+  actions: read
+  checks: read
+jobs:
+  release-rc:
+    if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
+      || (github.event_name == 'pull_request' && github.event.pull_request.merged
+      == true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
+      'release:rc'))
+    uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
+    with:
+      dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
+        }}
+      finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
+        == 'workflow_dispatch' && inputs.finalize) }}
+      recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
+        }}
+      circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
+        || '' }}
+      merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
+        || inputs.merge_sha || '' }}
+      bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
+        || inputs.bump_sha || '' }}
+      branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
+        || inputs.branch || '' }}
+      version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
+        || inputs.version || '' }}
+    secrets: inherit

.qqq-ci.yml

@@ -0,0 +1,16 @@
+profile: qbit
+orb_version: 0.7.12
+default_branch: develop
+java_version: '21'
+browsers: true
+api_checks: false
+docs_only: false
+release_on_main: false
+release_tag_policy: stable
+test_branch_policy: non-release
+publish:
+  snapshot: true
+  release_candidate: true
+  release: true
+  asciidoc: false
+release_completion: true

AGENTS.md

@@ -35,7 +35,7 @@ This file provides guidance to Codex when working with code in this repository.
 ## Dependencies
 
 - Java 21.
-- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). The default-active `qqq-snapshot` profile imports `qqq-bom-pom:4.1.0-RC.4` ahead of it, as in qbit-quick-search (ADR-0007). Remove that activation when the parent moves to 4.1.
+- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). The default-active `qqq-snapshot` profile imports `qqq-bom-pom:4.1.0-RC.6` ahead of it, as in qbit-quick-search (ADR-0007). Remove that activation when the parent moves to 4.1.
 - Tests: JUnit 5, AssertJ.
 
 ## Release

CHANGELOG.md

@@ -5,4 +5,33 @@ All notable changes to this project are documented here. The format follows Keep
 ## [0.1.0] - Unreleased
 
 ### Added
+- Central `backups.sla` and per-active-plan `backups.destination.<planId>` health checks, with bounded read-only destination diagnostics and secret-free status details.
+- Always-durable backup, resume, scheduled backup, restore and verify processes with conservative `NONE` idempotency and stable backup identities on explicit job retry.
+- Weighted backup phases, table/file counts, stored bytes, safe per-table/artifact logs, warnings, failures and resume notices through QQQ `JobProgress`.
+- Cooperative job cancellation during backup pages, media copying, restore pages and verification scans; local media-reference spooling keeps phase separation from growing heap usage across tables.
+- Complete host-neutral in-app help bundle with live references and a help coverage gate. Hosts can override entries with a HOST bundle.
+- Guided empty states: help bundle `help/qbit-backups.yaml` (app and all nine tables, with "How rows get created") applied by `BackupsHelp`, and a Setup card on the Backups home (plan, scope rules, storage, schedule, verification, prune bucket).
+- "Create default plan" table action on Backup plans: adds "Nightly full" for a storage bucket alias; insert-only.
+- `backupVerifyPlan`: the scheduled verification process. A plan verify cron created a scheduled job for a process that did not exist, so scheduled verification never ran.
 - Repository scaffold: `BackupsQBitConfig` and `BackupsQBitProducer` (table prefix `backups_`, permissions `backups.*`), build, CI and repository standard files.
+
+### Changed
+- Tables whose backend module has no query interface (storage-only modules such as qbit-edi `edi_preview_download`) are classified EXTERNAL and always left out of a backup with the reason `NOT_QUERYABLE`, even when selected by name or role; the Run backup review lists them. Before, such a table was read as DATA and the whole run FAILED with "Query is not implemented in this module".
+- Failed runs persist an actionable `error_message` ("Backup failed: Class: message") and fill `error_detail` with the exception class, the cause chain and the top root-cause frames (bounded to 8 KB). Both go through `FailureText.sanitize`, which redacts URL user-info, presigned query parameters, key=value secrets, Authorization/Bearer values, AWS access key ids, PEM blocks and Postgres row values while keeping object keys and bucket names.
+- Multi-node deployments: the cached Prometheus text (`BackupsMetrics`) is registered with QQQ `ClusterCaches` as `qbit-backups.metrics` and cleared on every node when a run finishes or a plan is inserted, updated or deleted; the 60 s TTL remains the safety net. `SlaChecker`'s manifest-head and bucket-audit throttles and `BackupsRuntime`'s process-permission map are registered as node-local, with reasons. `BackupsClusterCachesTest` audits the package for unregistered static caches and covers a remote invalidation.
+- Builds against QQQ `4.1.0-SNAPSHOT` (`qqq.snapshot.version`) for `ClusterCaches` until the next QQQ RC ships; a release must pin that RC.
+- Read the QBit metadata version from the Maven artifact version so release builds report their released version.
+- Target QQQ `4.1.0-RC.8` (job progress).
+- Persist controlled backup failure messages instead of untrusted provider exception text.
+- Pin QQQ to released 4.1.0-RC.7 and replace the temporary help loader with core HelpBundles registration.
+- Saving a plan with Use read replica on is refused (it was never read); the not-yet-available settings (read replica, native dump, incremental, PARQUET, ZSTD) say so in their field help; scheduledJobId is documented as unused.
+- Build against the released QQQ BOM 4.1.0-RC.5; replace reflective alerts binding with core QAlerts and real sink integration tests.
+- Register backup alert types for every instance while preserving custom backup sinks and no-op behavior without a configured core sink.
+- Use RC.5-valid alert names `backups.run.missing-in-storage` and `backups.plan.owner-invalid` (and matching deduplication keys). Hosts using the old underscore names must update routing rules.
+- Target QQQ 4.1.0-RC.6 and replace the fingerprint, keyset pager and reflective restore bridges with typed core APIs.
+- New manifests identify `qqq-table-schema-v1` and carry core schema descriptors; restore and content verification still accept pre-RC.6 manifests with the original fingerprint algorithm.
+- Preserve primary-key ordering, exclusive resume cursors, bounded pages and caller-owned transactions; reject non-orderable primary keys and scope filters with skip/limit.
+- Require the core `backups.restore` permission during Review when using a core restore writer, including anonymous-mode hosts; preserve supplied timestamps through `RestoreWriteFlag`.
+
+### Removed
+- The legacy `BackupsHealthIndicator` and its optional `qqq-middleware-health` dependency; central health registration replaces manual indicator configuration.

README.md

@@ -10,7 +10,7 @@
 ## Requirements
 
 - Java 21.
-- QQQ 4.1 (built against `4.1.0-RC.4`).
+- QQQ `4.1.0-RC.8` or later.
 
 ## Install
 
@@ -22,6 +22,10 @@ new BackupsQBitProducer()
    .produce(qInstance);
 ```
 
+## Setup
+
+This qbit ships no DDL and seeds nothing from a changelog: its one setup value, the storage bucket alias, belongs to your application. On a fresh install open Backups, read the Setup card, and run **Create default plan** on Backup plans (it asks only for the alias of a bucket your storage resolver provides). It creates the insert-only "Nightly full" plan: whole instance, data and audit tables, JSONL and gzip, 02:00 UTC, verified Sundays 04:00 UTC. Edit any setting afterwards. Help for every table, including how its rows get created, ships in `help/qbit-backups.yaml`.
+
 ## Naming
 
 Tables are `backups_<name>` (lowercase snake_case, backend name equal to the table name). Permissions are `backups.<action>`.
@@ -36,8 +40,83 @@ Follow the QQQ [adoption checklist](https://github.com/QRun-IO/qqq/blob/develop/
 ./mvnw verify
 ```
 
+See [the RC.6 migration notes](docs/RC6-MIGRATION.md) for fingerprint compatibility,
+paging guarantees and restore permissions.
+
+Health checks use the core health API released in QQQ `4.1.0-RC.8`.
+The producer registers `backups.sla` and a read-only destination check for each
+active plan (`backups.destination.<planId>`). With dependencies cached locally,
+run `./mvnw -o verify`; no enforcer skip flags are required.
+
+## In-app help
+
+The producer registers [help/qbit-backups.yaml](src/main/resources/help/qbit-backups.yaml)
+as a QBIT bundle. Hosts can register a `layer: HOST` bundle with matching entry keys
+and roles to override its text. Core's help startup sync applies the bundle when the
+host installs the help content table; registration also works without that table.
+
+`BackupPlanScheduleTest` checks coverage with `HelpBundleAssertions`, including
+live references and optional scheduler processes. `BackupsTablesGuardrailTest`
+checks empty-state, widget and possible-value-source coverage. Run it with
+`./mvnw verify`. Keep the bundle's literal version aligned with the POM version.
+
 Always use the Maven wrapper (Maven 3.9.11); Maven 3.10 cannot read the parent POM.
 
+## Durable jobs and progress
+
+Run backup now, Resume, scheduled backups, both Restore actions, and Verify backup
+are always durable jobs. Hosts must configure QQQ durable jobs and its worker.
+Review remains a preview; execution runs in the background and reports through
+the job page and jobs tray.
+
+Backups report Resolve scope, Back up tables, Copy media files, and Write catalog
+and manifest. Separate verification jobs report a single Verify phase.
+The job includes table/file counts, stored bytes, per-table
+rows and duration, artifact names, warnings and failures. Logs exclude row values,
+source file keys, storage credentials and endpoints. Provider errors use controlled
+failure messages; detailed exceptions stay in server diagnostics.
+
+These processes use `NONE` idempotency: a worker interruption requires review before
+retry because object writes and catalog updates are not atomic. Explicitly retrying
+a backup job reuses its run and resumes completed parts; it does not create another
+backup. Snapshot runs that cannot resume must be started as a new backup. See
+[the progress and retry contract](docs/DESIGN.md#job-progress-implementation) for limits and cancellation.
+
 ## License
 
 Apache License 2.0. See [LICENSE](LICENSE) and [NOTICE](NOTICE).
+
+### Alerts integration
+
+Backups binds directly to core QAlerts in QQQ 4.1.0-RC.5 and declares its nine
+alert types on each instance. Hosts install an `AlertSinkInterface` through
+`AlertsInstanceMetaData.withSink(QCodeReference)`. Without a configured sink,
+backup alert calls remain no-ops (they do not use core's logging fallback).
+Custom `BackupAlertSink` bindings are preserved. Core handles dispatch failures;
+adapter failures are caught and logged once, so alerting cannot fail a backup.
+Raise and resolve follow core's transaction-aware dispatch; heartbeats run immediately.
+The backup SLA checker still owns expected intervals and grace periods.
+
+RC.5 requires hyphenated alert names: `backups.run.missing-in-storage` and
+`backups.plan.owner-invalid` replace the former underscore spellings, including
+their deduplication keys. Update host routing rules that used those spellings.
+
+## Health checks
+
+The producer registers these central checks under component `backups`, category `STORAGE`:
+
+| Name | Kind | Interval | Timeout |
+| --- | --- | --- | --- |
+| `backups.sla` | DEPENDENCY | 900s (15 minutes) | 5000ms |
+| `backups.destination.<planId>` | DEPENDENCY | 86400s (24 hours) | 30000ms |
+
+The SLA check reads persisted `slaStatus`: critical active plans that are LATE,
+FAILING or MISSING_IN_STORAGE yield DOWN; non-critical failures yield DEGRADED.
+Only UNVERIFIED/UNKNOWN plans yield UNKNOWN; no active plans yield UP. Details
+contain counts per SLA status. The scheduled `backupCheckSla` still computes SLA state.
+
+Destination checks are supplied for active plans and reuse `DestinationProbe` in
+read-only list/head mode, including on unlocked buckets. They never write, delete,
+or resolve prune credentials. The Test destination action retains its write tests.
+Storage calls are bounded by the central runner, with deadline checks between calls.
+Override intervals in `health_check` and disable checks through `health_check.enabled`.

UPGRADE.md

@@ -0,0 +1,7 @@
+# Upgrading
+
+## Unreleased
+
+- No schema change and no new include line. Hosts get the help text and the Setup card automatically.
+- Plans with `use_read_replica` true are now refused on the next save (the setting was never used). Turn it off.
+- Hosts that set a plan `verifyCronExpression` and a scheduler now get working scheduled verification: the `backupVerifyPlan` process exists. It verifies the plan's latest successful run through the plan's `verifyLevel` (CONTENT when unset). Expect verification rows and, on failure, alerts.

docs/DESIGN.md

@@ -1,3 +1,18 @@
 # QBit Backups: design
 
 The design for this qbit is tracked in [QRun-IO/qqq#1091](https://github.com/QRun-IO/qqq/pull/1091). This repository implements it; the implementation is in progress.
+
+
+## Job progress implementation
+
+Backup, resume, scheduled backup, restore, and verify execution use always-durable
+jobs with `NONE` idempotency against the QQQ job-progress API targeted for RC.8.
+Backup phases are `prepare`, `tables`, `media`, and `upload`, weighted 2, 75, 18,
+and 5. Verification runs separately with a single `verify` phase.
+
+Progress reports table/file counts, stored bytes, and safe artifact names without
+row data, source keys, credentials, or endpoints. Cancellation is cooperative at
+page and media-copy boundaries. Media references spool locally between phases.
+Object writes and catalog updates are not atomic, so worker interruptions require
+review before retry. Explicit backup retries reuse the run and resume completed
+parts; snapshot runs that cannot resume require a new backup.

docs/RC6-MIGRATION.md

@@ -0,0 +1,100 @@
+# RC.6 core migration (issue #19, F3)
+
+The POM targets `4.1.0-RC.6`. Until that release is available, local verification
+uses `./mvnw -B verify -Dqqq.snapshot.version=4.1.0-ARCHRC6-SNAPSHOT` against the
+installed build from `qqq-wt-archrc6`. No core source changes are needed.
+
+## Fingerprint compatibility
+
+The algorithms are **not byte-compatible**, even for identical table metadata:
+
+| Descriptor property | Pre-RC.6 backup algorithm | Core algorithm |
+| --- | --- | --- |
+| Table and key | `name`, `primaryKey` | `table`, `primaryKeyField` |
+| Fields | Map keyed by name | Sorted list with explicit `name` |
+| Field flags | `required`, `pvs`, `heavy`, `sensitive` | `isRequired`, `possibleValueSource`, `isHeavy`, `isSensitive` |
+| Unique keys | Sorted `+`-joined strings | Sorted lists of field names |
+| Associations | Sorted `name->table` strings | Objects with name, associatedTable and joinName |
+| Null backend type | JSON null | Empty string |
+| Escaping | Historical hand-written escaping | Jackson JSON escaping |
+
+New table entries contain `fingerprintAlgorithm: "qqq-table-schema-v1"`, the core
+`schema` descriptor, and `fingerprint: "sha256:" + core.getHash()`. The outer
+manifest remains version 1; row parts, checksums, compression and storage paths
+are unchanged. The instance/catalog hash still combines the ordered table
+fingerprint strings, so its value changes too. New manifests require this RC.6
+reader; older readers do not understand the new descriptor shape.
+
+An absent algorithm identifies the old `qbit-backups-table-v1` algorithm. The
+reader also accepts that explicit identifier. `LegacyTableFingerprint` freezes
+the original descriptor and hash implementation solely for comparison against
+existing backups; no new backup uses it. Restore and content verification select
+the declared algorithm. Unknown identifiers never match a known fingerprint.
+Restore normalizes either field shape for its existing compatibility analysis
+when a table has changed. Fingerprint equality is not required for a compatible
+schema migration, as before.
+
+`src/test/resources/manifests/legacy-schema.json` freezes descriptors and hashes
+produced by the original stand-in for the blog fixture. Tests place these into a
+real backup manifest without the algorithm field, verify its content, mutate the
+live data, then restore and compare the complete content hash including dates.
+They also cover optional additions and incompatible type changes in old schemas.
+
+## Paging and consistency
+
+`TableReader.Keyset` delegates to core `KeysetPager`. Every query uses
+`(scope filter) AND pk > lastKey ORDER BY pk ASC LIMIT pageSize`, with no offset.
+The resume key is exclusive. Page size is a row bound, **not a JDBC fetch-size
+setting**. Core's normal bounded QueryAction path retains backend driver defaults;
+this adapter does not opt into the large-result streaming query hint or change
+JDBC fetch size. Heavy/hidden/password fields and untranslated stored values keep
+the same query settings and existing backup permission boundary.
+
+Every page receives the caller's transaction. The pager does not open, commit,
+close, or replace that transaction. A test opens a real core memory snapshot,
+changes a later page's data, and proves every page still sees the original state.
+The existing BackupEngine continues to report BEST_EFFORT and reject
+SNAPSHOT_REQUIRED: adopting the consistent-read opening API at engine level is
+separate work. This migration does not claim snapshot guarantees for ordinary
+backup runs.
+
+Core can stream non-orderable keys without deterministic key order; backups
+explicitly reject those keys instead. Core rejects scope filters carrying skip
+or limit, and replaces user ordering with ascending primary-key ordering. Tests
+cover OR filters, resume, full and partial pages, empty scans and invalid inputs.
+
+## Restore writes
+
+`StandardRestoreWriter` now constructs `RestoreWriteFlag` directly and calls
+`QBackendModuleInterface.resyncKeySequences`, consuming typed `SequenceResyncResult`
+values. Only RESYNCED results are reported as sequence changes; NOT_NEEDED and
+NOT_APPLICABLE remain normal outcomes. There is no reflection or unavailable-core
+fallback. The core flag preserves supplied dates and carries the requested
+customizer, automation, audit and event settings.
+
+Core always requires an actual session permission `backups.restore` for its
+restore flag, including hosts with general core enforcement disabled. The qbit's
+anonymous-mode convenience setting does not grant that permission. Review now
+reports this requirement before executing a restore; test sessions explicitly
+hold it. Custom writers that do not use the core flag keep their existing qbit
+permission checks.
+
+## HTTP operations and wire changes
+
+No HTTP operations are added, removed or modified: method, path and operationId
+are unchanged (no operation list entries for this migration). The RC.6 UI API
+extension documentation was reviewed; this task introduces no Javalin route or
+middleware extension. Wire changes are confined to the manifest fields and
+fingerprint values described above, plus an earlier restore permission error in
+Review for anonymous sessions lacking `backups.restore`.
+
+## Local verification (2026-10-13)
+
+- `./mvnw -B clean verify -Dqqq.snapshot.version=4.1.0-ARCHRC6-SNAPSHOT`:
+  BUILD SUCCESS; 158 tests, zero failures/errors/skips; both JaCoCo gates passed.
+  Existing backup, restore and MinIO integration tests ran.
+- Separate Checkstyle 14.3.0 audit of all 16 changed Java files (main and test):
+  zero violations, using the shared Kingsrook-derived rules with this repository's
+  Apache file header. This repository has no Checkstyle execution configured in
+  its POM; audit configuration and results are local under `.cache/style/`.
+- `git diff --check`: passed. No commit, push or core source modification.

pom.xml

@@ -18,6 +18,8 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 
    <properties>
+      <qbit.guardrails.enforce>true</qbit.guardrails.enforce>
+      <qbit.guardrails.strictUiApiRoutes>true</qbit.guardrails.strictUiApiRoutes>
       <revision>0.1.0-SNAPSHOT</revision>
       <maven.compiler.source>21</maven.compiler.source>
       <maven.compiler.target>21</maven.compiler.target>
@@ -27,7 +29,7 @@
    <parent>
       <groupId>com.kingsrook</groupId>
       <artifactId>qbit-build-parent</artifactId>
-      <version>2.0.0</version>
+      <version>2.1.0</version>
    </parent>
 
    <groupId>com.kingsrook.qbits</groupId>
@@ -72,13 +74,6 @@
          <artifactId>qqq-backend-core</artifactId>
       </dependency>
 
-      <!-- optional: the BackupsHealthIndicator (apps without qqq-middleware-health never load it) -->
-      <dependency>
-         <groupId>com.kingsrook.qqq</groupId>
-         <artifactId>qqq-middleware-health</artifactId>
-         <optional>true</optional>
-      </dependency>
-
       <!-- optional: ESB events (apps without qqq-esb never load it) -->
       <dependency>
          <groupId>com.kingsrook.qqq</groupId>
@@ -129,11 +124,33 @@
    </dependencies>
 
    <build>
+      <resources>
+         <!-- help bundles carry the artifact version (QQQ help bundle provenance) -->
+         <resource>
+            <directory>src/main/resources</directory>
+            <filtering>true</filtering>
+            <includes>
+               <include>help/*.yaml</include>
+               <include>qbit-backups.properties</include>
+            </includes>
+         </resource>
+         <resource>
+            <directory>src/main/resources</directory>
+            <filtering>false</filtering>
+            <excludes>
+               <exclude>help/*.yaml</exclude>
+               <exclude>qbit-backups.properties</exclude>
+            </excludes>
+         </resource>
+      </resources>
       <plugins>
          <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-surefire-plugin</artifactId>
             <configuration>
+               <systemPropertyVariables combine.children="append">
+                  <qbit.artifactVersion>${project.version}</qbit.artifactVersion>
+               </systemPropertyVariables>
                <!-- the MinIO test container default credentials, resolved through ${env.X} refs like production -->
                <environmentVariables>
                   <STORAGE_TEST_ACCESS_KEY>minioadmin</STORAGE_TEST_ACCESS_KEY>
@@ -192,14 +209,14 @@
    <profiles>
       <!-- Built against qqq 4.1 (QBitMetaDataProducer). Until qbit-build-parent  -->
       <!-- imports the released 4.1 BOM, this profile is active by default and     -->
-      <!-- imports qqq-bom-pom 4.1.0-RC.4 ahead of the parent's 4.0.0 BOM.         -->
+      <!-- imports qqq-bom-pom 4.1.0-RC.8 ahead of the parent's 4.0.0 BOM.         -->
       <profile>
          <id>qqq-snapshot</id>
          <activation>
             <activeByDefault>true</activeByDefault>
          </activation>
          <properties>
-            <qqq.snapshot.version>4.1.0-RC.4</qqq.snapshot.version>
+            <qqq.snapshot.version>4.1.0-RC.14</qqq.snapshot.version>
          </properties>
          <dependencyManagement>
             <dependencies>

src/main/java/com/kingsrook/qbits/backups/BackupsQBitConfig.java

@@ -47,6 +47,8 @@ public class BackupsQBitConfig implements QBitConfig
    private String parentAppName;
    private String tablePathPrefix;
    private String schedulerName;
+   private String hostSchedulerDescription;
+   private Boolean hostSchedulerRunsVerification = false;
    private String instanceName;
 
    private Integer maxConcurrentRuns       = 1;
@@ -202,6 +204,51 @@ public class BackupsQBitConfig implements QBitConfig
 
 
 
+   /*******************************************************************************
+    ** Getter for hostSchedulerDescription: when the application runs plan crons with
+    ** its own tick instead of a QQQ scheduler (schedulerName), a short name for it.
+    ** The Setup card then treats plan crons as firing.
+    *******************************************************************************/
+   public String getHostSchedulerDescription()
+   {
+      return (hostSchedulerDescription);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for hostSchedulerDescription.
+    *******************************************************************************/
+   public BackupsQBitConfig withHostSchedulerDescription(String hostSchedulerDescription)
+   {
+      this.hostSchedulerDescription = hostSchedulerDescription;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for hostSchedulerRunsVerification: whether the host's own tick also runs
+    ** the verify cron (by calling the backupVerifyPlan process). False by default.
+    *******************************************************************************/
+   public Boolean getHostSchedulerRunsVerification()
+   {
+      return (hostSchedulerRunsVerification);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for hostSchedulerRunsVerification.
+    *******************************************************************************/
+   public BackupsQBitConfig withHostSchedulerRunsVerification(Boolean hostSchedulerRunsVerification)
+   {
+      this.hostSchedulerRunsVerification = hostSchedulerRunsVerification;
+      return (this);
+   }
+
+
+
    /*******************************************************************************
     ** Getter for schedulerName (the QQQ scheduler used for plan jobs and the SLA checker)
     *******************************************************************************/

src/main/java/com/kingsrook/qbits/backups/BackupsQBitProducer.java

@@ -17,6 +17,18 @@
 package com.kingsrook.qbits.backups;
 
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+import com.kingsrook.qbits.backups.health.BackupsDestinationHealthChecks;
+import com.kingsrook.qbits.backups.health.BackupsSlaHealthCheck;
+import com.kingsrook.qqq.backend.core.health.HealthCategory;
+import com.kingsrook.qqq.backend.core.health.HealthCheckKind;
+import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData;
+import com.kingsrook.qqq.backend.core.health.HealthCheckSupplierMetaData;
+import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundles;
 import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaDataProducer;
 
@@ -29,13 +41,41 @@ public class BackupsQBitProducer implements QBitMetaDataProducer<BackupsQBitConf
 {
    public static final String GROUP_ID    = "com.kingsrook.qbits";
    public static final String ARTIFACT_ID = "qbit-backups";
-   public static final String VERSION     = "0.1.0";
+   public static final String VERSION     = loadVersion();
 
    private BackupsQBitConfig qBitConfig;
    private String namespace;
 
 
 
+   /*******************************************************************************
+    ** Read the artifact version from the Maven-filtered resource.
+    *******************************************************************************/
+   private static String loadVersion()
+   {
+      try(InputStream inputStream = BackupsQBitProducer.class.getClassLoader().getResourceAsStream("qbit-backups.properties"))
+      {
+         if(inputStream == null)
+         {
+            throw (new IllegalStateException("Missing qbit-backups.properties; build with Maven to generate the artifact version"));
+         }
+         Properties properties = new Properties();
+         properties.load(inputStream);
+         String version = properties.getProperty("version");
+         if(version == null || version.isBlank() || version.contains("${"))
+         {
+            throw (new IllegalStateException("Missing or unfiltered version in qbit-backups.properties; build with Maven"));
+         }
+         return (version);
+      }
+      catch(IOException e)
+      {
+         throw (new IllegalStateException("Could not read artifact version from qbit-backups.properties", e));
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Fluent setter for the config.
     *******************************************************************************/
@@ -119,13 +159,8 @@ public class BackupsQBitProducer implements QBitMetaDataProducer<BackupsQBitConf
       qInstance.addQBit(qBit);
       BackupsRuntime.install(qBitConfig);
 
-      ///////////////////////////////////////////////////////////////////////////////
-      // bind the Alerts qbit producer SPI when it is on the class path (optional) //
-      ///////////////////////////////////////////////////////////////////////////////
-      if(com.kingsrook.qbits.backups.alerts.AlertsQBitAlertSink.bindIfPresent())
-      {
-         new com.kingsrook.qbits.backups.alerts.AlertsQBitAlertSink().registerAlertTypes(qInstance);
-      }
+      com.kingsrook.qbits.backups.alerts.AlertsQBitAlertSink.bindIfAbsent();
+      new com.kingsrook.qbits.backups.alerts.AlertsQBitAlertSink().registerAlertTypes(qInstance);
 
       com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput output = new com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput();
       for(var possibleValueSource : com.kingsrook.qbits.backups.model.BackupsTables.possibleValueSources())
@@ -165,6 +200,15 @@ public class BackupsQBitProducer implements QBitMetaDataProducer<BackupsQBitConf
       com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData app = com.kingsrook.qbits.backups.app.BackupsApp.app(qBitConfig, tables);
       output.add(app);
       output.addSelfToInstance(qInstance);
+      // Registration is metadata-only; core guards startup sync when helpContent is absent.
+      HelpBundles.register(qInstance, "help/qbit-backups.yaml");
+
+      HealthChecksMetaData.ofOrWithNew(qInstance)
+         .withCheck(new HealthCheckMetaData().withName(BackupsSlaHealthCheck.NAME).withLabel("Backup SLA")
+            .withComponent("backups").withKind(HealthCheckKind.DEPENDENCY).withCategory(HealthCategory.STORAGE)
+            .withIntervalSeconds(900).withCheck(new QCodeReference(BackupsSlaHealthCheck.class)))
+         .withSupplier(new HealthCheckSupplierMetaData().withNamePrefix(BackupsDestinationHealthChecks.PREFIX)
+            .withComponent("backups").withSupplier(new QCodeReference(BackupsDestinationHealthChecks.class)));
 
       // A parent app only lists the apps that are its children, so attach ourselves (for example Admin, then Backups).
       com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData parent = qBitConfig.getParentAppName() == null ? null : qInstance.getApp(qBitConfig.getParentAppName());

src/main/java/com/kingsrook/qbits/backups/BackupsRuntime.java

@@ -25,6 +25,7 @@ import com.kingsrook.qbits.backups.alerts.BackupAlertSink;
 import com.kingsrook.qbits.backups.alerts.NoOpBackupAlertSink;
 import com.kingsrook.qbits.backups.storage.BackupStorage;
 import com.kingsrook.qbits.backups.storage.BackupStorageException;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
 
 
 /*******************************************************************************
@@ -34,7 +35,8 @@ import com.kingsrook.qbits.backups.storage.BackupStorageException;
  *******************************************************************************/
 public final class BackupsRuntime
 {
-   private static final Map<String, String> PROCESS_PERMISSIONS = new ConcurrentHashMap<>();
+   private static final Map<String, String> PROCESS_PERMISSIONS = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
+      "built from the qbit config at startup, the same on every node");
 
    private static volatile BackupsQBitConfig config;
    private static volatile Clock             clock           = Clock.systemUTC();

src/main/java/com/kingsrook/qbits/backups/alerts/AlertsQBitAlertSink.java

@@ -17,105 +17,39 @@
 package com.kingsrook.qbits.backups.alerts;
 
 
-import java.lang.reflect.Method;
 import java.util.Map;
 import java.util.stream.Collectors;
 import com.kingsrook.qbits.backups.BackupsRuntime;
+import com.kingsrook.qqq.backend.core.alerts.AlertSeverity;
+import com.kingsrook.qqq.backend.core.alerts.AlertSignal;
+import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData;
+import com.kingsrook.qqq.backend.core.alerts.QAlertTypeMetaData;
+import com.kingsrook.qqq.backend.core.alerts.QAlerts;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
-import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
- ** The BackupAlertSink adapter for the Alerts qbit producer SPI (QRun-IO/qqq
- ** PR #1108, com.kingsrook.qqq.backend.core.alerts: QAlerts, AlertSignal,
- ** AlertResolution, AlertSeverity, QAlertTypeMetaData; design spec
- ** docs/specs/2026-10-12-qbit-alerts-design.md).
- **
- ** STAND-IN: the SPI is an open core PR and is not in QQQ 4.1.0-RC.4, so this
- ** adapter is bound REFLECTIVELY and optionally.  It compiles against nothing in
- ** the SPI; it looks the classes up by name, and every call is guarded:
- ** <ul>
- **    <li>no SPI classes on the class path: {@link #bindIfPresent()} binds nothing
- **    and the sink stays the no-op;</li>
- **    <li>the SPI is present but the instance has no alerts meta-data (the Alerts
- **    qbit is not installed): every call is a no-op;</li>
- **    <li>any reflective or SPI failure is logged once and swallowed, because an
- **    alerting problem must never fail a backup.</li>
- ** </ul>
- ** Replace with a direct binding (QAlerts.raise and friends) when RC.5 ships.
- **
- ** Mapping: raise to QAlerts.raise(AlertSignal), resolve to QAlerts.resolve(type,
- ** dedupKey, note), heartbeat to QAlerts.heartbeat(key).  The SPI has no per-plan
- ** heartbeat registration (monitors are declared on the alert type), so the
- ** expected interval and grace are not passed on; the SLA checker raises
- ** backups.plan.missed itself, independent of any heartbeat (spec 10.4).
+ ** Direct adapter to core QAlerts. Calls are no-ops without a configured alerts
+ ** sink, including when only the backup alert types have been registered.
+ ** QAlerts catches and logs dispatch failures; adapter failures are logged once.
+ ** Heartbeat intervals remain the responsibility of the backup SLA checker.
  *******************************************************************************/
 public class AlertsQBitAlertSink implements BackupAlertSink
 {
    private static final QLogger LOG = QLogger.getLogger(AlertsQBitAlertSink.class);
 
-   /** The package of the Alerts producer SPI in qqq-backend-core. */
-   public static final String DEFAULT_API_PACKAGE = "com.kingsrook.qqq.backend.core.alerts";
-
-   private final String apiPackage;
-
    private boolean warned = false;
 
-   private Class<?> qAlerts;
-   private Class<?> alertSignal;
-   private Class<?> alertSeverity;
-   private Class<?> instanceMetaData;
-
 
 
    /*******************************************************************************
-    ** Constructor for the real SPI package.
+    ** Bind the core adapter unless the host already supplied a backup sink.
     *******************************************************************************/
-   public AlertsQBitAlertSink()
+   public static boolean bindIfAbsent()
    {
-      this(DEFAULT_API_PACKAGE);
-   }
-
-
-
-   /*******************************************************************************
-    ** Constructor for another package (a test double of the SPI shape).
-    *******************************************************************************/
-   public AlertsQBitAlertSink(String apiPackage)
-   {
-      this.apiPackage = apiPackage;
-   }
-
-
-
-   /*******************************************************************************
-    ** Whether the SPI classes are on the class path.
-    *******************************************************************************/
-   public static boolean isPresent(String apiPackage)
-   {
-      try
-      {
-         Class.forName(apiPackage + ".QAlerts");
-         Class.forName(apiPackage + ".AlertSignal");
-         return (true);
-      }
-      catch(ClassNotFoundException | LinkageError e)
-      {
-         return (false);
-      }
-   }
-
-
-
-   /*******************************************************************************
-    ** Bind this adapter as the runtime's alert sink when the SPI is present and no
-    ** other sink was bound already.  Returns whether it bound.
-    *******************************************************************************/
-   public static boolean bindIfPresent()
-   {
-      if(!(BackupsRuntime.getAlertSink() instanceof NoOpBackupAlertSink) || !isPresent(DEFAULT_API_PACKAGE))
+      if(!(BackupsRuntime.getAlertSink() instanceof NoOpBackupAlertSink))
       {
          return (false);
       }
@@ -126,38 +60,32 @@ public class AlertsQBitAlertSink implements BackupAlertSink
 
 
    /*******************************************************************************
-    ** Declare the backup alert types with the Alerts qbit (QAlertTypeMetaData), when
-    ** the SPI is present.  Returns how many were added.
+    ** Declare missing backup alert types on each instance, independently of sink
+    ** installation order. Returns how many were added.
     *******************************************************************************/
    public int registerAlertTypes(QInstance qInstance)
    {
       int added = 0;
-      try
+      for(BackupAlerts.AlertType alertType : BackupAlerts.alertTypes())
       {
-         Class<?> typeClass = Class.forName(apiPackage + ".QAlertTypeMetaData");
-         load();
-         for(BackupAlerts.AlertType alertType : BackupAlerts.alertTypes())
+         try
          {
-            try
+            if(QAlertTypeMetaData.of(qInstance, alertType.name()) == null)
             {
-               Object type = typeClass.getConstructor().newInstance();
-               typeClass.getMethod("withName", String.class).invoke(type, alertType.name());
-               typeClass.getMethod("withLabel", String.class).invoke(type, alertType.label());
-               typeClass.getMethod("withGroupName", String.class).invoke(type, "Backups");
-               typeClass.getMethod("withHowToFix", String.class).invoke(type, alertType.howToFix());
-               typeClass.getMethod("withDefaultSeverity", alertSeverity).invoke(type, severity(alertType.defaultSeverity()));
-               typeClass.getMethod("addSelfToInstance", QInstance.class).invoke(type, qInstance);
+               new QAlertTypeMetaData()
+                  .withName(alertType.name())
+                  .withLabel(alertType.label())
+                  .withGroupName("Backups")
+                  .withHowToFix(alertType.howToFix())
+                  .withDefaultSeverity(AlertSeverity.valueOf(alertType.defaultSeverity().name()))
+                  .addSelfToInstance(qInstance);
                added++;
             }
-            catch(ReflectiveOperationException | RuntimeException e)
-            {
-               warnOnce("Could not declare the alert type " + alertType.name(), e);
-            }
          }
-      }
-      catch(ReflectiveOperationException | LinkageError e)
-      {
-         warnOnce("The alerts SPI is not available", e);
+         catch(RuntimeException | LinkageError e)
+         {
+            warnOnce("Could not declare the alert type " + alertType.name(), e);
+         }
       }
       return (added);
    }
@@ -170,28 +98,24 @@ public class AlertsQBitAlertSink implements BackupAlertSink
    @Override
    public void raise(String type, String dedupeKey, Severity severity, String title, Map<String, String> detail)
    {
-      if(!isAlertsQBitInstalled())
-      {
-         return;
-      }
       try
       {
-         Object signal = alertSignal.getConstructor(String.class).newInstance(type);
-         alertSignal.getMethod("setDedupKey", String.class).invoke(signal, dedupeKey);
-         alertSignal.getMethod("setSeverity", alertSeverity).invoke(signal, severity(severity));
-         alertSignal.getMethod("setTitle", String.class).invoke(signal, title);
+         if(!isAlertsQBitInstalled())
+         {
+            return;
+         }
+         AlertSignal signal = new AlertSignal(type)
+            .withDedupKey(dedupeKey)
+            .withSeverity(AlertSeverity.valueOf(severity.name()))
+            .withTitle(title);
          if(detail != null && !detail.isEmpty())
          {
-            alertSignal.getMethod("setSummary", String.class).invoke(signal, detail.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(", ")));
-            Method withAttribute = alertSignal.getMethod("withAttribute", String.class, String.class);
-            for(Map.Entry<String, String> entry : detail.entrySet())
-            {
-               withAttribute.invoke(signal, entry.getKey(), entry.getValue());
-            }
+            signal.withSummary(detail.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(", ")));
+            detail.forEach(signal::withAttribute);
          }
-         qAlerts.getMethod("raise", alertSignal).invoke(null, signal);
+         QAlerts.raise(signal);
       }
-      catch(ReflectiveOperationException | RuntimeException e)
+      catch(RuntimeException | LinkageError e)
       {
          warnOnce("Could not raise an alert", e);
       }
@@ -205,15 +129,14 @@ public class AlertsQBitAlertSink implements BackupAlertSink
    @Override
    public void resolve(String type, String dedupeKey)
    {
-      if(!isAlertsQBitInstalled())
-      {
-         return;
-      }
       try
       {
-         qAlerts.getMethod("resolve", String.class, String.class, String.class).invoke(null, type, dedupeKey, "Cleared by the Backups qbit");
+         if(isAlertsQBitInstalled())
+         {
+            QAlerts.resolve(type, dedupeKey, "Cleared by the Backups qbit");
+         }
       }
-      catch(ReflectiveOperationException | RuntimeException e)
+      catch(RuntimeException | LinkageError e)
       {
          warnOnce("Could not resolve an alert", e);
       }
@@ -227,15 +150,14 @@ public class AlertsQBitAlertSink implements BackupAlertSink
    @Override
    public void heartbeat(String monitorKey, long expectedEverySeconds, long graceSeconds)
    {
-      if(!isAlertsQBitInstalled())
-      {
-         return;
-      }
       try
       {
-         qAlerts.getMethod("heartbeat", String.class).invoke(null, monitorKey);
+         if(isAlertsQBitInstalled())
+         {
+            QAlerts.heartbeat(monitorKey);
+         }
       }
-      catch(ReflectiveOperationException | RuntimeException e)
+      catch(RuntimeException | LinkageError e)
       {
          warnOnce("Could not check in a heartbeat", e);
       }
@@ -244,67 +166,26 @@ public class AlertsQBitAlertSink implements BackupAlertSink
 
 
    /*******************************************************************************
-    ** The SPI classes load, and the instance has alerts meta-data (the Alerts qbit
-    ** is installed).  Checked on every call so the order qbits are produced in
-    ** does not matter.
+    ** Check on every call so installing the alerts sink after this qbit works.
+    ** Type declarations alone do not mean an alerts implementation is installed.
     *******************************************************************************/
    boolean isAlertsQBitInstalled()
    {
-      try
-      {
-         load();
-         QInstance qInstance = QContext.getQInstance();
-         if(qInstance == null)
-         {
-            return (false);
-         }
-         return (instanceMetaData.getMethod("of", QInstance.class).invoke(null, qInstance) != null);
-      }
-      catch(ReflectiveOperationException | RuntimeException | LinkageError e)
-      {
-         warnOnce("The alerts SPI is not usable", e);
-         return (false);
-      }
+      AlertsInstanceMetaData metaData = AlertsInstanceMetaData.of(QContext.getQInstance());
+      return (metaData != null && metaData.getSink() != null);
    }
 
 
 
    /*******************************************************************************
-    **
+    ** Log adapter failures once; dispatch failures are already logged by core.
     *******************************************************************************/
-   private synchronized void load() throws ClassNotFoundException
-   {
-      if(qAlerts == null)
-      {
-         qAlerts = Class.forName(apiPackage + ".QAlerts");
-         alertSignal = Class.forName(apiPackage + ".AlertSignal");
-         alertSeverity = Class.forName(apiPackage + ".AlertSeverity");
-         instanceMetaData = Class.forName(apiPackage + ".AlertsInstanceMetaData");
-      }
-   }
-
-
-
-   /*******************************************************************************
-    ** Map our severity to the SPI's enum constant of the same name.
-    *******************************************************************************/
-   @SuppressWarnings({ "unchecked", "rawtypes" })
-   private Object severity(Severity severity)
-   {
-      return (Enum.valueOf((Class<Enum>) alertSeverity, severity.name()));
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private void warnOnce(String message, Throwable t)
+   private synchronized void warnOnce(String message, Throwable t)
    {
       if(!warned)
       {
          warned = true;
-         LOG.warn(message + " (the Alerts qbit adapter will keep failing quietly)", t, logPair("apiPackage", apiPackage));
+         LOG.warn(message + " (the Alerts qbit adapter will keep failing quietly)", t);
       }
    }
 }

src/main/java/com/kingsrook/qbits/backups/alerts/BackupAlerts.java

@@ -36,11 +36,11 @@ public final class BackupAlerts
 {
    public static final String PLAN_MISSED          = "backups.plan.missed";
    public static final String RUN_FAILED           = "backups.run.failed";
-   public static final String RUN_MISSING          = "backups.run.missing_in_storage";
+   public static final String RUN_MISSING          = "backups.run.missing-in-storage";
    public static final String FILES_MISSING        = "backups.files.missing";
    public static final String VERIFY_FAILED        = "backups.verify.failed";
    public static final String VERIFY_OVERDUE       = "backups.verify.overdue";
-   public static final String PLAN_OWNER_INVALID   = "backups.plan.owner_invalid";
+   public static final String PLAN_OWNER_INVALID   = "backups.plan.owner-invalid";
    public static final String STORAGE_DEBRIS       = "backups.storage.debris";
    public static final String RESTORE_COMPLETED    = "backups.restore.completed";
 

src/main/java/com/kingsrook/qbits/backups/app/BackupsHomeView.java

@@ -69,9 +69,11 @@ public final class BackupsHomeView
    public static String render(BackupsHomeData data, String pathPrefix)
    {
       StringBuilder html = new StringBuilder("<div class=\"backups-home\" style=\"" + ROOT + "\">");
+      List<BackupsSetupStatus.Item> setupItems = BackupsSetupStatus.load(data.plans());
       planStatus(html, data, pathPrefix);
+      setup(html, setupItems, pathPrefix);
       html.append("<div style=\"").append(GRID).append("\">");
-      needsAttention(html, data, pathPrefix);
+      needsAttention(html, data, setupItems, pathPrefix);
       latestTestRestore(html, data, pathPrefix);
       recentActivity(html, data, pathPrefix);
       trend(html, data);
@@ -140,23 +142,70 @@ public final class BackupsHomeView
 
 
    /*******************************************************************************
-    ** Needs attention.
+    ** The Setup card: what a working backup needs, whether each part is in place,
+    ** and the next step, linked to its table.
     *******************************************************************************/
-   private static void needsAttention(StringBuilder html, BackupsHomeData data, String prefix)
+   private static void setup(StringBuilder html, List<BackupsSetupStatus.Item> items, String prefix)
+   {
+      open(html, "Setup");
+      long todo = items.stream().filter(i -> i.state() == BackupsSetupStatus.State.TODO).count();
+      html.append("<p style=\"margin:0 0 .6rem;font-size:.9rem;\">")
+         .append(todo == 0 ? "Everything backups need is set up." : esc(todo + (todo == 1 ? " item needs" : " items need") + " attention."))
+         .append("</p><ul style=\"list-style:none;margin:0;padding:0;\">");
+      for(BackupsSetupStatus.Item item : items)
+      {
+         String color = switch(item.state())
+         {
+            case OK -> "#16a34a";
+            case TODO -> "#dc2626";
+            case DEFAULT -> "#d97706";
+            case OPTIONAL -> "#9ca3af";
+         };
+         String label = switch(item.state())
+         {
+            case OK -> "Set up";
+            case TODO -> "Needs setup";
+            case DEFAULT -> "Using defaults";
+            case OPTIONAL -> "Optional";
+         };
+         html.append("<li style=\"").append(ROW).append("\"><div style=\"display:flex;flex-wrap:wrap;align-items:center;gap:.5rem .75rem;\">")
+            .append("<strong>").append("<a style=\"").append(LINK).append("\" href=\"").append(esc(prefix + "/" + item.tableName())).append("\">").append(esc(item.label())).append("</a></strong>")
+            .append("<span style=\"").append(CHIP).append("\"><span aria-hidden=\"true\" style=\"width:.55rem;height:.55rem;border-radius:50%;background:").append(color).append(";display:inline-block;\"></span>").append(esc(label)).append("</span>")
+            .append("</div><div style=\"font-size:.9rem;margin-top:.35rem;\">").append(esc(item.detail())).append("</div>");
+         if(item.nextStep() != null)
+         {
+            html.append("<div style=\"").append(MUTED).append("font-size:.9rem;margin-top:.2rem;\">Next: ").append(esc(item.nextStep())).append("</div>");
+         }
+         html.append("</li>");
+      }
+      html.append("</ul>");
+      close(html);
+   }
+
+
+
+   /*******************************************************************************
+    ** Needs attention: every Setup item that needs setup (the same list the Setup
+    ** card counts), plus plans in a problem state and recent failed runs, so the two
+    ** cards never disagree.
+    *******************************************************************************/
+   private static void needsAttention(StringBuilder html, BackupsHomeData data, List<BackupsSetupStatus.Item> setupItems, String prefix)
    {
       open(html, "Needs attention");
-      List<QRecord> plans = data.plansNeedingAttention();
-      if(data.plans().isEmpty())
+      List<BackupsSetupStatus.Item> setupTodo = setupItems.stream().filter(i -> i.state() == BackupsSetupStatus.State.TODO).toList();
+      List<QRecord>                 plans     = data.plansNeedingAttention();
+      if(setupTodo.isEmpty() && plans.isEmpty() && data.failedRuns().isEmpty())
       {
-         empty(html, "Nothing to check yet: no plans.");
-      }
-      else if(plans.isEmpty() && data.failedRuns().isEmpty())
-      {
-         empty(html, "Nothing needs attention.");
+         empty(html, data.plans().isEmpty() ? "Nothing to check yet: no plans." : "Nothing needs attention.");
       }
       else
       {
          html.append("<ul style=\"").append(ITEMS).append("\">");
+         for(BackupsSetupStatus.Item item : setupTodo)
+         {
+            html.append("<li>Setup: <a style=\"").append(LINK).append("\" href=\"").append(esc(prefix + "/" + item.tableName())).append("\">").append(esc(item.label())).append("</a>: ")
+               .append(esc(item.detail())).append("</li>");
+         }
          for(QRecord plan : plans)
          {
             html.append("<li>").append(recordLink(prefix, BackupsTables.PLAN, plan.getValue("id"), plan.getValueString("name")))

src/main/java/com/kingsrook/qbits/backups/app/BackupsSetupStatus.java

@@ -0,0 +1,207 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.app;
+
+
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import com.kingsrook.qbits.backups.BackupsQBitConfig;
+import com.kingsrook.qbits.backups.BackupsRuntime;
+import com.kingsrook.qbits.backups.model.BackupsTables;
+import com.kingsrook.qbits.backups.processes.DefaultPlanProcess;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** The Setup card on the Backups home: what a working backup needs (a plan, a
+ ** storage bucket that resolves, a schedule, verification, a separate prune
+ ** bucket), whether each is in place, and the next step.  Computed from the plans
+ ** the home already loaded and the runtime's storage resolver; it reads nothing
+ ** else.
+ *******************************************************************************/
+public final class BackupsSetupStatus
+{
+   /*******************************************************************************
+    ** OK: in place. TODO: backups need it. DEFAULT: working, with a default worth a
+    ** look. OPTIONAL: a feature that is off until set up.
+    *******************************************************************************/
+   public enum State
+   {
+      OK, TODO, DEFAULT, OPTIONAL
+   }
+
+
+
+   /*******************************************************************************
+    ** One line of the card.
+    *******************************************************************************/
+   public record Item(String label, String tableName, State state, String detail, String nextStep)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private BackupsSetupStatus()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Build the items from the plans.
+    *******************************************************************************/
+   public static List<Item> load(List<QRecord> plans)
+   {
+      BackupsQBitConfig config = BackupsRuntime.isInstalled() ? BackupsRuntime.getConfig() : null;
+      return (load(plans, config != null && hasContent(config.getSchedulerName()), config == null ? null : config.getHostSchedulerDescription(),
+         config != null && Boolean.TRUE.equals(config.getHostSchedulerRunsVerification())));
+   }
+
+
+
+   /*******************************************************************************
+    ** Build the items from the plans, given whether a scheduler is configured (a plan
+    ** cron only fires through the scheduler the qbit config names).
+    *******************************************************************************/
+   public static List<Item> load(List<QRecord> plans, boolean schedulerConfigured)
+   {
+      return (load(plans, schedulerConfigured, null, false));
+   }
+
+
+
+   /*******************************************************************************
+    ** Build the items from the plans. A host that runs plan crons with its own tick
+    ** (hostScheduler names it) has crons that fire; hostRunsVerification says whether
+    ** that tick also runs the verify cron.
+    *******************************************************************************/
+   public static List<Item> load(List<QRecord> plans, boolean schedulerConfigured, String hostScheduler, boolean hostRunsVerification)
+   {
+      boolean host = hasContent(hostScheduler);
+      List<Item> items = new ArrayList<>();
+      List<QRecord> active = plans.stream().filter(p -> Boolean.TRUE.equals(p.getValueBoolean("isActive"))).toList();
+
+      ///////////
+      // plans //
+      ///////////
+      if(plans.isEmpty())
+      {
+         items.add(new Item("Backup plans", BackupsTables.PLAN, State.TODO, "No plan: nothing is backed up.",
+            "Run Create default plan on Backup plans (it asks only for the storage bucket alias), or add a plan with New."));
+         items.add(new Item("Plan scope rules", BackupsTables.PLAN_SCOPE, State.OPTIONAL, "None needed yet.",
+            "A plan that backs up everything (scope Instance) needs no rules. Add them on a plan that is limited to apps, qbits or tables."));
+         return (items);
+      }
+      items.add(active.isEmpty()
+         ? new Item("Backup plans", BackupsTables.PLAN, State.TODO, plans.size() + (plans.size() == 1 ? " plan, but it is" : " plans, but none is") + " active: nothing runs on its own.", "Turn a plan on (Active).")
+         : new Item("Backup plans", BackupsTables.PLAN, State.OK, active.size() + " active plan" + (active.size() == 1 ? "" : "s") + " of " + plans.size() + ".", null));
+
+      /////////////////////////////////////////
+      // scope rules: needed unless INSTANCE //
+      /////////////////////////////////////////
+      List<String> limited = plans.stream().filter(p -> p.getValueString("scopeType") != null && !"INSTANCE".equals(p.getValueString("scopeType"))).map(p -> p.getValueString("name")).toList();
+      items.add(limited.isEmpty()
+         ? new Item("Plan scope rules", BackupsTables.PLAN_SCOPE, State.OPTIONAL, "Every plan backs up the whole instance, so no rules are needed.",
+            "Add rules to a plan only to include or leave out apps, qbits, tables or roles.")
+         : new Item("Plan scope rules", BackupsTables.PLAN_SCOPE, State.OK, limited.size() + (limited.size() == 1 ? " plan is" : " plans are") + " limited by scope rules: " + String.join(", ", limited) + ".",
+            "Check each plan's Scope rules to see what it covers."));
+
+      ///////////////////////////////////////////////
+      // storage: every alias a plan names resolves //
+      ///////////////////////////////////////////////
+      Set<String> aliases = new LinkedHashSet<>();
+      for(QRecord plan : plans)
+      {
+         for(String field : List.of("storageBucketAlias", "pruneBucketAlias"))
+         {
+            if(plan.getValueString(field) != null)
+            {
+               aliases.add(plan.getValueString(field));
+            }
+         }
+      }
+      List<String> broken = new ArrayList<>();
+      for(String alias : aliases)
+      {
+         try
+         {
+            BackupsRuntime.storageFor(alias);
+         }
+         catch(RuntimeException e)
+         {
+            broken.add(alias);
+         }
+      }
+      items.add(broken.isEmpty()
+         ? new Item("Storage", BackupsTables.PLAN, State.OK, aliases.size() + " storage bucket alias" + (aliases.size() == 1 ? "" : "es") + " in use, and each resolves.", "Run Test destination on a plan to prove it can write.")
+         : new Item("Storage", BackupsTables.PLAN, State.TODO, "These aliases do not resolve to a bucket: " + String.join(", ", broken) + ".",
+            "Fix the alias on the plan, or add the bucket to the application's storage resolver."));
+
+      /////////////////////////////////////////
+      // schedule: an active plan has a cron //
+      /////////////////////////////////////////
+      long scheduled = active.stream().filter(p -> hasContent(p.getValueString("cronExpression"))).count();
+      String noScheduler = "No scheduler is configured (schedulerName on the qbit config), so plan crons are saved but never fire";
+      items.add(!schedulerConfigured && !host
+         ? new Item("Schedule", BackupsTables.PLAN, State.TODO, noScheduler + ". Backups run only when someone clicks Run backup now.",
+            "Set schedulerName on the qbit config (or run your own tick if the application does), then give each plan a cron expression.")
+         : scheduled == 0
+         ? new Item("Schedule", BackupsTables.PLAN, State.TODO, "No active plan has a schedule: backups run only when someone clicks Run backup now.", "Set a Cron expression on a plan (for example " + DefaultPlanProcess.CRON + ").")
+         : new Item("Schedule", BackupsTables.PLAN, State.OK, scheduled + " active plan" + (scheduled == 1 ? " runs" : "s run") + " on a schedule"
+            + (host && !schedulerConfigured ? ", run by " + hostScheduler + " (the application scheduler)." : "."), null));
+
+      //////////////////
+      // verification //
+      //////////////////
+      long verified = active.stream().filter(p -> hasContent(p.getValueString("verifyCronExpression"))).count();
+      items.add(host && !schedulerConfigured && !hostRunsVerification
+         ? new Item("Verification", BackupsTables.VERIFICATION, State.OPTIONAL, "Not run by " + hostScheduler + ": verify cron expressions are saved but nothing fires them.",
+            "Run Verify backup on a run by hand, or have the application scheduler run the backupVerifyPlan process for each plan.")
+         : !schedulerConfigured && !host
+         ? new Item("Verification", BackupsTables.VERIFICATION, State.TODO, noScheduler + ", so no verification is scheduled.",
+            "Set schedulerName on the qbit config, then a verify cron expression on a plan; or run Verify backup on a run by hand.")
+         : verified == 0
+         ? new Item("Verification", BackupsTables.VERIFICATION, State.OPTIONAL, "No plan verifies its backups on a schedule.",
+            "Set a Verify cron expression on a plan, or run Verify backup on a run. A backup nobody has restored is a guess.")
+         : new Item("Verification", BackupsTables.VERIFICATION, State.OK, verified + " active plan" + (verified == 1 ? " verifies" : "s verify") + " its latest backup on a schedule.", null));
+
+      ////////////////////////////////////////////
+      // prune bucket: separate delete authority //
+      ////////////////////////////////////////////
+      long noPrune = plans.stream().filter(p -> !hasContent(p.getValueString("pruneBucketAlias"))).count();
+      items.add(noPrune == 0
+         ? new Item("Prune bucket", BackupsTables.PLAN, State.OK, "Every plan prunes through its own delete-capable bucket alias.", null)
+         : new Item("Prune bucket", BackupsTables.PLAN, State.DEFAULT, noPrune + (noPrune == 1 ? " plan has" : " plans have") + " no prune bucket: Prune uses the backup bucket, so one key can both write and delete backups.",
+            "Give the plan a separate delete-capable Prune bucket alias."));
+      return (items);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static boolean hasContent(String value)
+   {
+      return (value != null && !value.isBlank());
+   }
+}

src/main/java/com/kingsrook/qbits/backups/health/BackupsDestinationHealthCheck.java

@@ -0,0 +1,95 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.health;
+
+
+import com.kingsrook.qbits.backups.model.BackupsTables;
+import com.kingsrook.qbits.backups.plan.DestinationProbe;
+import com.kingsrook.qbits.backups.run.BackupCatalog;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.health.HealthCheck;
+import com.kingsrook.qqq.backend.core.health.HealthCheckInput;
+import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome;
+import com.kingsrook.qqq.backend.core.health.HealthDetails;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
+
+
+/*******************************************************************************
+ ** Reuse the destination diagnostic in read-only mode; never publish its text.
+ *******************************************************************************/
+public class BackupsDestinationHealthCheck implements HealthCheck
+{
+   private final Integer planId;
+
+
+
+   /*******************************************************************************
+    ** Retain only the id; destination and active state are read on each run.
+    *******************************************************************************/
+   public BackupsDestinationHealthCheck(Integer planId)
+   {
+      this.planId = planId;
+   }
+
+
+
+   /*******************************************************************************
+    ** A removed or disabled plan can remain registered until supplier refresh.
+    *******************************************************************************/
+   @Override
+   public HealthCheckOutcome run(HealthCheckInput input)
+   {
+      boolean ownsContext = QContext.getQInstance() == null;
+      if(ownsContext)
+      {
+         QContext.init(input.getQInstance(), new QSystemUserSession(input.getQInstance()));
+      }
+      try
+      {
+         if(BackupsSlaHealthCheck.expired(input))
+         {
+            return (HealthCheckOutcome.down("Backup destination check timed out"));
+         }
+         QRecord plan = new BackupCatalog().get(BackupsTables.PLAN, planId).orElse(null);
+         if(plan == null || !Boolean.TRUE.equals(plan.getValueBoolean("isActive")))
+         {
+            return (HealthCheckOutcome.unknown("Backup plan is no longer active"));
+         }
+         DestinationProbe.Report report = DestinationProbe.probeReadOnly(plan, input.getDeadline());
+         if(BackupsSlaHealthCheck.expired(input))
+         {
+            return (HealthCheckOutcome.down("Backup destination check timed out"));
+         }
+         return ((report.ok() ? HealthCheckOutcome.up("Backup destination is readable") : HealthCheckOutcome.down("Backup destination probe failed"))
+            .withDetails(new HealthDetails().withValue("planId", planId).withValue("readOnly", true)
+               .withValue("checks", report.checks().size()).withValue("failedChecks", report.checks().stream().filter(check -> !check.ok()).count())));
+      }
+      catch(QException | RuntimeException e)
+      {
+         return (HealthCheckOutcome.down("Backup destination could not be checked"));
+      }
+      finally
+      {
+         if(ownsContext)
+         {
+            QContext.clear();
+         }
+      }
+   }
+}

src/main/java/com/kingsrook/qbits/backups/health/BackupsDestinationHealthChecks.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.health;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import com.kingsrook.qbits.backups.model.BackupsTables;
+import com.kingsrook.qbits.backups.run.BackupCatalog;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.health.HealthCategory;
+import com.kingsrook.qqq.backend.core.health.HealthCheckKind;
+import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData;
+import com.kingsrook.qqq.backend.core.health.HealthCheckSupplier;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
+
+
+/*******************************************************************************
+ ** Discover one daily, read-only destination check per active backup plan.
+ *******************************************************************************/
+public class BackupsDestinationHealthChecks implements HealthCheckSupplier
+{
+   public static final String PREFIX = "backups.destination";
+
+
+
+   /*******************************************************************************
+    ** Resolve from current catalog data, without opening any storage connections.
+    *******************************************************************************/
+   @Override
+   public List<HealthCheckMetaData> supply(QInstance qInstance) throws QException
+   {
+      boolean ownsContext = QContext.getQInstance() == null;
+      if(ownsContext)
+      {
+         QContext.init(qInstance, new QSystemUserSession(qInstance));
+      }
+      try
+      {
+         List<HealthCheckMetaData> checks = new ArrayList<>();
+         for(QRecord plan : new BackupCatalog().where(BackupsTables.PLAN, "isActive", true))
+         {
+            Integer planId = plan.getValueInteger("id");
+            checks.add(new HealthCheckMetaData().withName(PREFIX + "." + planId.toString().toLowerCase(Locale.ROOT))
+               .withLabel("Backup destination " + planId).withComponent("backups")
+               .withKind(HealthCheckKind.DEPENDENCY).withCategory(HealthCategory.STORAGE)
+               .withIntervalSeconds(86400).withTimeoutMs(30000).withCheckInstance(new BackupsDestinationHealthCheck(planId)));
+         }
+         return (checks);
+      }
+      finally
+      {
+         if(ownsContext)
+         {
+            QContext.clear();
+         }
+      }
+   }
+}

src/main/java/com/kingsrook/qbits/backups/health/BackupsHealthIndicator.java

@@ -1,120 +0,0 @@
-/*
- * Copyright 2026 Kingsrook, LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kingsrook.qbits.backups.health;
-
-
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import com.kingsrook.qbits.backups.model.BackupsTables;
-import com.kingsrook.qbits.backups.run.BackupCatalog;
-import com.kingsrook.qqq.backend.core.context.QContext;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
-import com.kingsrook.qqq.backend.core.model.session.QSession;
-import com.kingsrook.qqq.middleware.health.HealthIndicator;
-import com.kingsrook.qqq.middleware.health.model.metadata.HealthCheckResult;
-import com.kingsrook.qqq.middleware.health.model.metadata.HealthStatus;
-
-
-/*******************************************************************************
- ** The Backups health indicator (design spec section 10.4), for
- ** qqq-middleware-health (an optional dependency: an app that does not have that
- ** module never loads this class).  It reports DOWN if any active plan marked
- ** critical is LATE, FAILING or MISSING_IN_STORAGE, and UP with details otherwise.
- ** It reads the slaStatus the SLA checker maintains; it never touches storage, so
- ** a health call is cheap.
- **
- ** It is OFF the liveness path by design.  Add it to the indicators of the
- ** instance's health check (the readiness and monitoring view), and do not point a
- ** Kubernetes liveness probe at a check that includes it: a backup problem must
- ** never restart pods.
- *******************************************************************************/
-public class BackupsHealthIndicator implements HealthIndicator
-{
-   public static final String NAME = "backups";
-
-   private static final Set<String> CRITICAL_STATUSES = Set.of("LATE", "FAILING", "MISSING_IN_STORAGE");
-
-
-
-   /*******************************************************************************
-    ** {@inheritDoc}
-    *******************************************************************************/
-   @Override
-   public String getName()
-   {
-      return (NAME);
-   }
-
-
-
-   /*******************************************************************************
-    ** {@inheritDoc}
-    *******************************************************************************/
-   @Override
-   public HealthCheckResult check(QInstance qInstance) throws QException
-   {
-      long                started = System.currentTimeMillis();
-      boolean             ownsContext = QContext.getQInstance() == null;
-      if(ownsContext)
-      {
-         QContext.init(qInstance, new QSession());
-      }
-      try
-      {
-         List<QRecord> plans = new BackupCatalog().query(BackupsTables.PLAN, new QQueryFilter(new QFilterCriteria("isActive", QCriteriaOperator.EQUALS, true)));
-
-         List<String> criticalProblems = new ArrayList<>();
-         Map<String, Integer> byStatus = new LinkedHashMap<>();
-         for(QRecord plan : plans)
-         {
-            String status = plan.getValueString("slaStatus") == null ? "UNKNOWN" : plan.getValueString("slaStatus");
-            byStatus.merge(status, 1, Integer::sum);
-            if(Boolean.TRUE.equals(plan.getValueBoolean("critical")) && CRITICAL_STATUSES.contains(status))
-            {
-               criticalProblems.add(plan.getValueString("planKey") + ": " + status);
-            }
-         }
-
-         Map<String, Object> details = new LinkedHashMap<>();
-         details.put("activePlans", plans.size());
-         details.put("plansByStatus", byStatus);
-         if(!criticalProblems.isEmpty())
-         {
-            details.put("criticalPlansWithProblems", criticalProblems);
-         }
-         return (new HealthCheckResult()
-            .withStatus(criticalProblems.isEmpty() ? HealthStatus.UP : HealthStatus.DOWN)
-            .withDurationMs(System.currentTimeMillis() - started)
-            .withDetails(details));
-      }
-      finally
-      {
-         if(ownsContext)
-         {
-            QContext.clear();
-         }
-      }
-   }
-}

src/main/java/com/kingsrook/qbits/backups/health/BackupsSlaHealthCheck.java

@@ -0,0 +1,132 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.health;
+
+
+import java.time.Instant;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import com.kingsrook.qbits.backups.model.BackupsTables;
+import com.kingsrook.qbits.backups.run.BackupCatalog;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.health.HealthCheck;
+import com.kingsrook.qqq.backend.core.health.HealthCheckInput;
+import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome;
+import com.kingsrook.qqq.backend.core.health.HealthDetails;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
+
+
+/*******************************************************************************
+ ** Read persisted SLA state using the former indicator's critical-plan rule.
+ ** SLA calculation and storage verification remain the scheduled checker's job.
+ *******************************************************************************/
+public class BackupsSlaHealthCheck implements HealthCheck
+{
+   public static final String NAME = "backups.sla";
+
+   private static final Set<String> BAD_STATUSES = Set.of("LATE", "FAILING", "MISSING_IN_STORAGE");
+   private static final List<String> STATUSES = List.of("OK", "LATE", "FAILING", "MISSING_IN_STORAGE", "UNVERIFIED", "ABANDONED", "OWNER_INVALID", "UNKNOWN");
+
+
+
+   /*******************************************************************************
+    ** Counts use a fixed vocabulary so stored strings cannot become detail keys.
+    *******************************************************************************/
+   @Override
+   public HealthCheckOutcome run(HealthCheckInput input)
+   {
+      boolean ownsContext = QContext.getQInstance() == null;
+      if(ownsContext)
+      {
+         QContext.init(input.getQInstance(), new QSystemUserSession(input.getQInstance()));
+      }
+      try
+      {
+         if(expired(input))
+         {
+            return (HealthCheckOutcome.down("Backup SLA check timed out"));
+         }
+         List<QRecord> plans = new BackupCatalog().where(BackupsTables.PLAN, "isActive", true);
+         Map<String, Integer> counts = new LinkedHashMap<>();
+         STATUSES.forEach(status -> counts.put(status, 0));
+         boolean criticalBad = false;
+         boolean ordinaryBad = false;
+         for(QRecord plan : plans)
+         {
+            String status = plan.getValueString("slaStatus");
+            status = status != null && STATUSES.contains(status) ? status : "UNKNOWN";
+            counts.merge(status, 1, Integer::sum);
+            if(BAD_STATUSES.contains(status))
+            {
+               if(Boolean.TRUE.equals(plan.getValueBoolean("critical")))
+               {
+                  criticalBad = true;
+               }
+               else
+               {
+                  ordinaryBad = true;
+               }
+            }
+         }
+         if(expired(input))
+         {
+            return (HealthCheckOutcome.down("Backup SLA check timed out"));
+         }
+         HealthDetails details = new HealthDetails().withValue("activePlans", plans.size());
+         counts.forEach((status, count) -> details.withValue("slaStatus." + status, count));
+         HealthCheckOutcome outcome = HealthCheckOutcome.up("Backup SLA has no failures");
+         if(criticalBad)
+         {
+            outcome = HealthCheckOutcome.down("Critical backup plans have SLA failures");
+         }
+         else if(ordinaryBad)
+         {
+            outcome = HealthCheckOutcome.degraded("Backup plans have SLA failures");
+         }
+         else if(!plans.isEmpty() && counts.get("UNVERIFIED") + counts.get("UNKNOWN") == plans.size())
+         {
+            outcome = HealthCheckOutcome.unknown("Backup SLA is not verified yet");
+         }
+         return (outcome.withDetails(details));
+      }
+      catch(QException | RuntimeException e)
+      {
+         return (HealthCheckOutcome.down("Backup SLA state could not be read"));
+      }
+      finally
+      {
+         if(ownsContext)
+         {
+            QContext.clear();
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Stop between catalog operations when the runner's budget is exhausted.
+    *******************************************************************************/
+   static boolean expired(HealthCheckInput input)
+   {
+      return (Thread.currentThread().isInterrupted() || !Instant.now().isBefore(input.getDeadline()));
+   }
+}

src/main/java/com/kingsrook/qbits/backups/manifest/BackupManifest.java

@@ -137,6 +137,7 @@ public record BackupManifest(
       String backend,
       String primaryKey,
       String fingerprint,
+      String fingerprintAlgorithm,
       Map<String, Object> schema,
       String filter,
       List<ExcludedField> excludedFields,

src/main/java/com/kingsrook/qbits/backups/manifest/LegacyTableFingerprint.java

@@ -0,0 +1,170 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.manifest;
+
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import com.kingsrook.qbits.backups.codec.HashingOutputStream;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+
+
+/*******************************************************************************
+ ** Frozen pre-RC.6 fingerprint algorithm, used only to read existing manifests.
+ ** Keep its descriptor and escaping byte-compatible with the original writer.
+ *******************************************************************************/
+final class LegacyTableFingerprint
+{
+
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private LegacyTableFingerprint()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** The descriptor of a table (sorted fields; plain maps and lists, so it is also
+    ** what goes in the manifest schema).
+    *******************************************************************************/
+   public static Map<String, Object> descriptor(QTableMetaData table, String backendType)
+   {
+      Map<String, Object> d = new LinkedHashMap<>();
+      d.put("name", table.getName());
+      d.put("primaryKey", table.getPrimaryKeyField());
+      d.put("backendType", backendType);
+
+      Map<String, Object> fields = new TreeMap<>();
+      for(QFieldMetaData field : new TreeMap<>(table.getFields()).values())
+      {
+         Map<String, Object> f = new LinkedHashMap<>();
+         f.put("type", field.getType().name());
+         f.put("required", field.getIsRequired());
+         f.put("maxLength", field.getMaxLength());
+         f.put("pvs", field.getPossibleValueSourceName());
+         f.put("heavy", field.getIsHeavy());
+         f.put("sensitive", field.getIsSensitive());
+         fields.put(field.getName(), f);
+      }
+      d.put("fields", fields);
+
+      List<String> uniqueKeys = new ArrayList<>();
+      if(table.getUniqueKeys() != null)
+      {
+         for(UniqueKey key : table.getUniqueKeys())
+         {
+            uniqueKeys.add(String.join("+", key.getFieldNames().stream().sorted().toList()));
+         }
+      }
+      uniqueKeys.sort(String::compareTo);
+      d.put("uniqueKeys", uniqueKeys);
+
+      List<String> associations = new ArrayList<>();
+      if(table.getAssociations() != null)
+      {
+         for(Association association : table.getAssociations())
+         {
+            associations.add(association.getName() + "->" + association.getAssociatedTableName());
+         }
+      }
+      associations.sort(String::compareTo);
+      d.put("associations", associations);
+      return (d);
+   }
+
+
+
+   /*******************************************************************************
+    ** The fingerprint of a table: sha256:hex of the canonical descriptor.
+    *******************************************************************************/
+   public static String of(QTableMetaData table, String backendType)
+   {
+      return (fingerprintOf(descriptor(table, backendType)));
+   }
+
+
+
+   /*******************************************************************************
+    ** The fingerprint of a descriptor (maps sorted by key at every level, no whitespace).
+    *******************************************************************************/
+   public static String fingerprintOf(Map<String, Object> descriptor)
+   {
+      StringBuilder sb = new StringBuilder();
+      canonical(descriptor, sb);
+      return ("sha256:" + HashingOutputStream.sha256Hex(sb.toString().getBytes(StandardCharsets.UTF_8)));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("unchecked")
+   private static void canonical(Object o, StringBuilder sb)
+   {
+      if(o == null)
+      {
+         sb.append("null");
+      }
+      else if(o instanceof Map<?, ?> map)
+      {
+         sb.append('{');
+         boolean first = true;
+         for(Map.Entry<String, Object> e : new TreeMap<>((Map<String, Object>) map).entrySet())
+         {
+            if(!first)
+            {
+               sb.append(',');
+            }
+            first = false;
+            sb.append('"').append(e.getKey()).append("\":");
+            canonical(e.getValue(), sb);
+         }
+         sb.append('}');
+      }
+      else if(o instanceof List<?> list)
+      {
+         sb.append('[');
+         for(int i = 0; i < list.size(); i++)
+         {
+            if(i > 0)
+            {
+               sb.append(',');
+            }
+            canonical(list.get(i), sb);
+         }
+         sb.append(']');
+      }
+      else if(o instanceof Number || o instanceof Boolean)
+      {
+         sb.append(o);
+      }
+      else
+      {
+         sb.append('"').append(String.valueOf(o).replace("\\", "\\\\").replace("\"", "\\\"")).append('"');
+      }
+   }
+}

src/main/java/com/kingsrook/qbits/backups/manifest/TableFingerprints.java

@@ -18,28 +18,23 @@ package com.kingsrook.qbits.backups.manifest;
 
 
 import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.TreeMap;
 import com.kingsrook.qbits.backups.codec.HashingOutputStream;
-import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.TableSchemaFingerprint;
 
 
 /*******************************************************************************
- ** A stable schema descriptor and SHA-256 fingerprint of a table (spec 5.5).
- **
- ** This is an internal stand-in for the QQQ core TableSchemaFingerprint (core PR
- ** feature/schema-fingerprint).  When that ships in the next 4.1.x RC, this class
- ** delegates to it and the descriptor below is kept only for reading old manifests.
- ** Labels, adornments and help text are excluded, so cosmetic changes do not count.
+ ** Core schema fingerprints with read compatibility for pre-RC.6 manifests.
  *******************************************************************************/
 public final class TableFingerprints
 {
+   public static final String CORE_ALGORITHM = "qqq-table-schema-v1";
+   public static final String LEGACY_ALGORITHM = "qbit-backups-table-v1";
+
+
 
    /*******************************************************************************
     ** Utility class.
@@ -51,134 +46,83 @@ public final class TableFingerprints
 
 
    /*******************************************************************************
-    ** The descriptor of a table (sorted fields; plain maps and lists, so it is also
-    ** what goes in the manifest schema).
+    ** The core descriptor stored in new manifests.
     *******************************************************************************/
    public static Map<String, Object> descriptor(QTableMetaData table, String backendType)
    {
-      Map<String, Object> d = new LinkedHashMap<>();
-      d.put("name", table.getName());
-      d.put("primaryKey", table.getPrimaryKeyField());
-      d.put("backendType", backendType);
-
-      Map<String, Object> fields = new TreeMap<>();
-      for(QFieldMetaData field : new TreeMap<>(table.getFields()).values())
-      {
-         Map<String, Object> f = new LinkedHashMap<>();
-         f.put("type", field.getType().name());
-         f.put("required", field.getIsRequired());
-         f.put("maxLength", field.getMaxLength());
-         f.put("pvs", field.getPossibleValueSourceName());
-         f.put("heavy", field.getIsHeavy());
-         f.put("sensitive", field.getIsSensitive());
-         fields.put(field.getName(), f);
-      }
-      d.put("fields", fields);
-
-      List<String> uniqueKeys = new ArrayList<>();
-      if(table.getUniqueKeys() != null)
-      {
-         for(UniqueKey key : table.getUniqueKeys())
-         {
-            uniqueKeys.add(String.join("+", key.getFieldNames().stream().sorted().toList()));
-         }
-      }
-      uniqueKeys.sort(String::compareTo);
-      d.put("uniqueKeys", uniqueKeys);
-
-      List<String> associations = new ArrayList<>();
-      if(table.getAssociations() != null)
-      {
-         for(Association association : table.getAssociations())
-         {
-            associations.add(association.getName() + "->" + association.getAssociatedTableName());
-         }
-      }
-      associations.sort(String::compareTo);
-      d.put("associations", associations);
-      return (d);
+      return (TableSchemaFingerprint.of(table, backendType).getDescriptor());
    }
 
 
 
    /*******************************************************************************
-    ** The fingerprint of a table: sha256:hex of the canonical descriptor.
+    ** Retain the manifest's sha256 prefix around the core hash.
     *******************************************************************************/
    public static String of(QTableMetaData table, String backendType)
    {
-      return (fingerprintOf(descriptor(table, backendType)));
+      return ("sha256:" + TableSchemaFingerprint.of(table, backendType).getHash());
    }
 
 
 
    /*******************************************************************************
-    ** The fingerprint of a descriptor (maps sorted by key at every level, no whitespace).
+    ** An absent algorithm identifies manifests written before RC.6. Unknown
+    ** algorithms must not silently fall back to a different fingerprint.
     *******************************************************************************/
-   public static String fingerprintOf(Map<String, Object> descriptor)
+   public static boolean matches(BackupManifest.TableEntry entry, QTableMetaData table, String backendType)
    {
-      StringBuilder sb = new StringBuilder();
-      canonical(descriptor, sb);
-      return ("sha256:" + HashingOutputStream.sha256Hex(sb.toString().getBytes(StandardCharsets.UTF_8)));
+      if(entry.fingerprint() == null)
+      {
+         return (false);
+      }
+      String algorithm = entry.fingerprintAlgorithm();
+      if(algorithm == null || LEGACY_ALGORITHM.equals(algorithm))
+      {
+         return (entry.fingerprint().equals(LegacyTableFingerprint.of(table, backendType)));
+      }
+      return (CORE_ALGORITHM.equals(algorithm) && entry.fingerprint().equals(of(table, backendType)));
    }
 
 
 
    /*******************************************************************************
-    ** The fingerprint of a whole instance's tables: a hash of the table fingerprints.
+    ** Normalize both descriptor versions to field-name maps for restore's
+    ** compatibility analysis. Core fields use a list and explicit flag names.
+    *******************************************************************************/
+   @SuppressWarnings("unchecked")
+   public static Map<String, Object> fields(Map<String, Object> descriptor)
+   {
+      Object fields = descriptor == null ? null : descriptor.get("fields");
+      if(fields instanceof Map<?, ?> map)
+      {
+         return ((Map<String, Object>) map);
+      }
+      Map<String, Object> result = new LinkedHashMap<>();
+      if(fields instanceof List<?> list)
+      {
+         for(Object item : list)
+         {
+            Map<String, Object> field = (Map<String, Object>) item;
+            Map<String, Object> normalized = new LinkedHashMap<>();
+            normalized.put("type", field.get("type"));
+            normalized.put("required", field.get("isRequired"));
+            normalized.put("maxLength", field.get("maxLength"));
+            normalized.put("pvs", field.get("possibleValueSource"));
+            normalized.put("heavy", field.get("isHeavy"));
+            normalized.put("sensitive", field.get("isSensitive"));
+            result.put((String) field.get("name"), normalized);
+         }
+      }
+      return (result);
+   }
+
+
+
+   /*******************************************************************************
+    ** The instance fingerprint remains a hash of the ordered table fingerprints.
     *******************************************************************************/
    public static String combine(List<String> tableFingerprints)
    {
       return ("sha256:" + HashingOutputStream.sha256Hex(String.join("\n", tableFingerprints).getBytes(StandardCharsets.UTF_8)));
    }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @SuppressWarnings("unchecked")
-   private static void canonical(Object o, StringBuilder sb)
-   {
-      if(o == null)
-      {
-         sb.append("null");
-      }
-      else if(o instanceof Map<?, ?> map)
-      {
-         sb.append('{');
-         boolean first = true;
-         for(Map.Entry<String, Object> e : new TreeMap<>((Map<String, Object>) map).entrySet())
-         {
-            if(!first)
-            {
-               sb.append(',');
-            }
-            first = false;
-            sb.append('"').append(e.getKey()).append("\":");
-            canonical(e.getValue(), sb);
-         }
-         sb.append('}');
-      }
-      else if(o instanceof List<?> list)
-      {
-         sb.append('[');
-         for(int i = 0; i < list.size(); i++)
-         {
-            if(i > 0)
-            {
-               sb.append(',');
-            }
-            canonical(list.get(i), sb);
-         }
-         sb.append(']');
-      }
-      else if(o instanceof Number || o instanceof Boolean)
-      {
-         sb.append(o);
-      }
-      else
-      {
-         sb.append('"').append(String.valueOf(o).replace("\\", "\\\\").replace("\"", "\\\"")).append('"');
-      }
-   }
 }

src/main/java/com/kingsrook/qbits/backups/metrics/BackupsMetrics.java

@@ -20,10 +20,12 @@ package com.kingsrook.qbits.backups.metrics;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
 import com.kingsrook.qbits.backups.BackupsQBitConfig;
 import com.kingsrook.qbits.backups.BackupsRuntime;
 import com.kingsrook.qbits.backups.model.BackupsTables;
 import com.kingsrook.qbits.backups.run.BackupCatalog;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
@@ -56,10 +58,21 @@ public final class BackupsMetrics
 {
    public static final String ENDPOINT_PATH = "/metrics/backups";
 
+   /** Cluster cache name of the rendered text: cleared on every node when a run finishes or a plan changes. */
+   public static final String CACHE_NAME = "qbit-backups.metrics";
+
    private static final Duration CACHE = Duration.ofSeconds(60);
 
-   private static String  cachedText;
-   private static Instant cachedAt;
+   private static final AtomicReference<Rendered> CACHED = ClusterCaches.shared(CACHE_NAME, new AtomicReference<>(), BackupsMetrics::resetCache, null);
+
+
+
+   /*******************************************************************************
+    ** The rendered text and when it was rendered.
+    *******************************************************************************/
+   private record Rendered(String text, Instant at)
+   {
+   }
 
 
 
@@ -84,12 +97,21 @@ public final class BackupsMetrics
 
 
    /*******************************************************************************
-    ** Forget the cached text (tests).
+    ** Forget the cached text on this node.
     *******************************************************************************/
-   public static synchronized void resetCache()
+   public static void resetCache()
    {
-      cachedText = null;
-      cachedAt = null;
+      CACHED.set(null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Forget the cached text on every node (the backup data changed).
+    *******************************************************************************/
+   public static void invalidate()
+   {
+      ClusterCaches.invalidate(CACHE_NAME);
    }
 
 
@@ -99,13 +121,14 @@ public final class BackupsMetrics
     *******************************************************************************/
    public static synchronized String renderCached() throws QException
    {
-      Instant now = BackupsRuntime.getClock().instant();
-      if(cachedText == null || cachedAt == null || cachedAt.plus(CACHE).isBefore(now))
+      Instant  now    = BackupsRuntime.getClock().instant();
+      Rendered cached = CACHED.get();
+      if(cached == null || cached.at().plus(CACHE).isBefore(now))
       {
-         cachedText = render();
-         cachedAt = now;
+         cached = new Rendered(render(), now);
+         CACHED.set(cached);
       }
-      return (cachedText);
+      return (cached.text());
    }
 
 

src/main/java/com/kingsrook/qbits/backups/model/BackupsTables.java

@@ -38,6 +38,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
@@ -412,15 +413,22 @@ public final class BackupsTables
       QCodeReference validation = new QCodeReference(BackupPlanTableCustomizer.class);
       table.withCustomizer(TableCustomizers.PRE_INSERT_RECORD, validation);
       table.withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, validation);
+
+      /////////////////////////////////////////////////////////////////////////////
+      // after a save or delete, the plan customizer clears the cached metrics on //
+      // every node                                                               //
+      /////////////////////////////////////////////////////////////////////////////
+      QCodeReference post = validation;
       if(StringUtils.hasContent(config.getSchedulerName()))
       {
-         QCodeReferenceWithProperties sync = MultiCustomizer.of(
+         post = MultiCustomizer.of(
             BaseSyncToScheduledJobTableCustomizer.makeCodeReference(table, new BackupPlanSyncToScheduledJobProcess()),
-            BaseSyncToScheduledJobTableCustomizer.makeCodeReference(table, new BackupPlanVerifySyncToScheduledJobProcess()));
-         table.withCustomizer(TableCustomizers.POST_INSERT_RECORD, sync);
-         table.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, sync);
-         table.withCustomizer(TableCustomizers.POST_DELETE_RECORD, sync);
+            BaseSyncToScheduledJobTableCustomizer.makeCodeReference(table, new BackupPlanVerifySyncToScheduledJobProcess()),
+            validation);
       }
+      table.withCustomizer(TableCustomizers.POST_INSERT_RECORD, post);
+      table.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, post);
+      table.withCustomizer(TableCustomizers.POST_DELETE_RECORD, post);
       return (table);
    }
 
@@ -446,8 +454,8 @@ public final class BackupsTables
          .withField(new QFieldMetaData("pruneBucketAlias", QFieldType.STRING).withMaxLength(100))
          .withField(new QFieldMetaData("prefix", QFieldType.STRING).withMaxLength(255))
          .withField(new QFieldMetaData("critical", QFieldType.BOOLEAN).withDefaultValue(false))
-         .withField(enumField("format", "format", "JSONL"))
-         .withField(enumField("compression", "compression", "GZIP"))
+         .withField(enumField("format", "format", "JSONL").withHelpContent(new QHelpContent("JSONL, CSV and JSONL_AND_CSV are available. PARQUET is not available yet and is refused when the plan is saved.")))
+         .withField(enumField("compression", "compression", "GZIP").withHelpContent(new QHelpContent("GZIP and NONE are available. ZSTD is not available yet and is refused when the plan is saved.")))
          .withField(enumField("encryptionMode", "encryption_mode", "NONE"))
          .withField(new QFieldMetaData("encryptionKeyRef", QFieldType.STRING).withMaxLength(100))
          .withField(enumField("sensitiveFieldPolicy", "sensitive_policy", "EXCLUDE"))
@@ -455,12 +463,15 @@ public final class BackupsTables
          .withField(enumField("fileObjectPolicy", "file_policy", "COPY"))
          .withField(enumField("fileMissingPolicy", "file_missing_policy", "WARN"))
          .withField(enumField("consistency", "consistency", "SNAPSHOT_PREFERRED"))
-         .withField(new QFieldMetaData("useReadReplica", QFieldType.BOOLEAN).withDefaultValue(false))
-         .withField(enumField("incrementalMode", "incremental_mode", "FULL"))
-         .withField(enumField("nativeDump", "native_dump", "NONE"))
+         .withField(new QFieldMetaData("useReadReplica", QFieldType.BOOLEAN).withDefaultValue(false)
+            .withHelpContent(new QHelpContent("Not available yet: backups always read from the application's own backend. Leave this off; turning it on is refused when the plan is saved.")))
+         .withField(enumField("incrementalMode", "incremental_mode", "FULL")
+            .withHelpContent(new QHelpContent("Only FULL is available yet. CHANGED_SINCE_LAST is not available yet and is refused when the plan is saved.")))
+         .withField(enumField("nativeDump", "native_dump", "NONE")
+            .withHelpContent(new QHelpContent("Only NONE is available yet. Native database dumps (PG_DUMP_CUSTOM) are not available yet and are refused when the plan is saved.")))
          .withField(new QFieldMetaData("cronExpression", QFieldType.STRING).withMaxLength(100))
          .withField(new QFieldMetaData("cronTimeZoneId", QFieldType.STRING).withMaxLength(60))
-         .withField(roInt("scheduledJobId"))
+         .withField(roInt("scheduledJobId").withHelpContent(new QHelpContent("Not used: a plan's schedules are scheduled jobs found by their foreign key (backups_plan for the backup, backups_plan_verify for the verification), so look there.")))
          .withField(new QFieldMetaData("slaHours", QFieldType.INTEGER).withDefaultValue(26))
          .withField(new QFieldMetaData("verifySlaDays", QFieldType.INTEGER).withDefaultValue(8))
          .withField(new QFieldMetaData("verifyFileSamplePercent", QFieldType.INTEGER).withDefaultValue(100))

src/main/java/com/kingsrook/qbits/backups/plan/BackupPlanTableCustomizer.java

@@ -22,6 +22,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import com.kingsrook.qbits.backups.metrics.BackupsMetrics;
 import com.kingsrook.qbits.backups.model.BackupsTables;
 import com.kingsrook.qbits.backups.run.BackupCatalog;
 import com.kingsrook.qbits.backups.run.BackupRunFactory;
@@ -31,6 +32,9 @@ import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterfa
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
@@ -43,12 +47,13 @@ import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
  ** configuration (the processes write the last run and SLA fields) is not
  ** validated again.
  **
- ** The saving user becomes the plan owner when none is given.
+ ** The saving user becomes the plan owner when none is given.  After an insert,
+ ** update or delete, the cached Prometheus text is cleared on every node.
  *******************************************************************************/
 public class BackupPlanTableCustomizer implements TableCustomizerInterface
 {
    private static final List<String> CONFIGURATION_FIELDS = List.of("planKey", "isActive", "scopeType", "includeRoles", "storageBucketAlias", "pruneBucketAlias",
-      "format", "compression", "encryptionMode", "encryptionKeyRef", "sensitiveFieldPolicy", "incrementalMode", "nativeDump", "cronExpression", "cronTimeZoneId",
+      "format", "compression", "encryptionMode", "encryptionKeyRef", "sensitiveFieldPolicy", "incrementalMode", "nativeDump", "useReadReplica", "cronExpression", "cronTimeZoneId",
       "verifyCronExpression", "retainDaily", "retainWeekly", "retainMonthly", "retainYearly", "retainMinimum", "objectLockMode", "objectLockDays",
       "slaHours", "pageSize", "ownerUserId");
 
@@ -122,6 +127,42 @@ public class BackupPlanTableCustomizer implements TableCustomizerInterface
 
 
 
+   /*******************************************************************************
+    ** A saved plan changes the metrics on every node.
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
+   {
+      BackupsMetrics.invalidate();
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    ** A saved plan changes the metrics on every node.
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      BackupsMetrics.invalidate();
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    ** A deleted plan changes the metrics on every node.
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postDelete(DeleteInput deleteInput, List<QRecord> records) throws QException
+   {
+      BackupsMetrics.invalidate();
+      return (records);
+   }
+
+
+
    /*******************************************************************************
     ** The scope rules to resolve: the ones sent with the plan (associated rows), or
     ** the saved ones.

src/main/java/com/kingsrook/qbits/backups/plan/BackupPlanValidator.java

@@ -176,6 +176,10 @@ public final class BackupPlanValidator
       {
          errors.add("Native database dumps are not available yet. Use NONE.");
       }
+      if(settings.bool("useReadReplica", false))
+      {
+         errors.add("Reading from a replica is not available yet: backups read from the application's own backend. Turn Use read replica off.");
+      }
    }
 
 

src/main/java/com/kingsrook/qbits/backups/plan/DestinationProbe.java

@@ -20,6 +20,7 @@ package com.kingsrook.qbits.backups.plan;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.nio.charset.StandardCharsets;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.UUID;
@@ -81,6 +82,28 @@ public final class DestinationProbe
     ** Probe the destination of a plan.
     *******************************************************************************/
    public static Report probe(QRecord plan)
+   {
+      return (probe(plan, false, null));
+   }
+
+
+
+   /*******************************************************************************
+    ** Health monitoring only lists and heads; it never resolves prune credentials
+    ** or probes capabilities. The runner bounds in-flight storage calls; check the
+    ** deadline between calls because BackupStorage has no per-call timeout API.
+    *******************************************************************************/
+   public static Report probeReadOnly(QRecord plan, Instant deadline)
+   {
+      return (probe(plan, true, deadline));
+   }
+
+
+
+   /*******************************************************************************
+    ** Share storage resolution and the existing locked-bucket read path.
+    *******************************************************************************/
+   private static Report probe(QRecord plan, boolean readOnly, Instant deadline)
    {
       List<Check>  checks   = new ArrayList<>();
       List<String> warnings = new ArrayList<>();
@@ -89,15 +112,23 @@ public final class DestinationProbe
       BackupStorage storage;
       try
       {
+         requireTime(deadline);
          storage = BackupsRuntime.storageFor(alias);
+         requireTime(deadline);
       }
       catch(RuntimeException e)
       {
          checks.add(new Check("Open the backup bucket " + alias, false, e.getMessage()));
-         return (new Report(false, checks, List.of(), warnings));
+         return (new Report(readOnly, checks, List.of(), warnings));
       }
       checks.add(new Check("Open the backup bucket " + alias, true, "found"));
 
+      if(readOnly)
+      {
+         headOnly(plan, storage, BackupPruner.planPrefix(plan), checks, deadline);
+         return (new Report(true, checks, List.of(), warnings));
+      }
+
       BackupStorage deleteSide = storage;
       String        pruneAlias = plan.getValueString("pruneBucketAlias");
       if(pruneAlias == null || pruneAlias.isBlank())
@@ -133,7 +164,7 @@ public final class DestinationProbe
       String prefix = BackupPruner.planPrefix(plan);
       if(locked)
       {
-         headOnly(plan, storage, prefix, checks);
+         headOnly(plan, storage, prefix, checks, null);
       }
       else
       {
@@ -147,10 +178,11 @@ public final class DestinationProbe
    /*******************************************************************************
     ** The probe for a locked bucket: list the prefix, and head the newest manifest.
     *******************************************************************************/
-   private static void headOnly(QRecord plan, BackupStorage storage, String prefix, List<Check> checks)
+   private static void headOnly(QRecord plan, BackupStorage storage, String prefix, List<Check> checks, Instant deadline)
    {
-      checks.add(attempt("Write and delete a probe object", () -> "skipped: object lock is on, a probe object could not be deleted again"));
-      checks.add(attempt("List the plan prefix", () ->
+      checks.add(new Check("Write and delete a probe object", true,
+         deadline == null ? "skipped: object lock is on, a probe object could not be deleted again" : "skipped: read-only probe"));
+      checks.add(attempt("List the plan prefix", deadline, () ->
       {
          List<StoredObjectInfo> objects = storage.list(prefix);
          return (objects.size() + " objects under " + prefix);
@@ -162,10 +194,11 @@ public final class DestinationProbe
          checks.add(new Check("Head the newest manifest", true, "there is no successful run yet"));
          return;
       }
-      checks.add(attempt("Head the newest manifest", () ->
+      checks.add(attempt("Head the newest manifest", deadline, () ->
       {
          QRecord run = new com.kingsrook.qbits.backups.run.BackupCatalog().get(com.kingsrook.qbits.backups.model.BackupsTables.RUN, lastSuccess).orElseThrow();
          String  key = run.getValueString("manifestKey");
+         requireTime(deadline);
          if(key == null || storage.head(key).isEmpty())
          {
             throw (new IllegalStateException("the manifest of the newest successful run is not in the bucket"));
@@ -238,6 +271,9 @@ public final class DestinationProbe
    @FunctionalInterface
    private interface Attempt
    {
+      /*******************************************************************************
+       ** Execute one diagnostic operation.
+       *******************************************************************************/
       String run() throws Exception;
    }
 
@@ -247,14 +283,40 @@ public final class DestinationProbe
     **
     *******************************************************************************/
    private static Check attempt(String name, Attempt attempt)
+   {
+      return (attempt(name, null, attempt));
+   }
+
+
+
+   /*******************************************************************************
+    ** Never begin another storage operation after the health budget expires.
+    *******************************************************************************/
+   private static Check attempt(String name, Instant deadline, Attempt attempt)
    {
       try
       {
-         return (new Check(name, true, attempt.run()));
+         requireTime(deadline);
+         String detail = attempt.run();
+         requireTime(deadline);
+         return (new Check(name, true, detail));
       }
       catch(Exception e)
       {
          return (new Check(name, false, e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage()));
       }
    }
+
+
+
+   /*******************************************************************************
+    ** Existing on-demand diagnostics have no deadline; health probes always do.
+    *******************************************************************************/
+   private static void requireTime(Instant deadline)
+   {
+      if(deadline != null && (Thread.currentThread().isInterrupted() || !Instant.now().isBefore(deadline)))
+      {
+         throw (new IllegalStateException("Destination probe timed out"));
+      }
+   }
 }

src/main/java/com/kingsrook/qbits/backups/processes/BackupsProcesses.java

@@ -22,6 +22,8 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
 import com.kingsrook.qbits.backups.BackupsRuntime;
 import com.kingsrook.qbits.backups.model.BackupsTables;
 import com.kingsrook.qbits.backups.permissions.BackupsPermissions;
@@ -30,6 +32,11 @@ import com.kingsrook.qbits.backups.run.BackupRunFactory;
 import com.kingsrook.qbits.backups.run.BackupRunResult;
 import com.kingsrook.qbits.backups.scope.ScopeResolver;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.jobs.DurableProcessMetaData;
+import com.kingsrook.qqq.backend.core.jobs.model.JobIdempotency;
+import com.kingsrook.qqq.backend.core.jobs.plans.JobChunkScope;
+import com.kingsrook.qqq.backend.core.jobs.plans.JobInterruptedException;
+import com.kingsrook.qqq.backend.core.jobs.progress.JobProgress;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
@@ -71,7 +78,8 @@ public final class BackupsProcesses
    {
       List<QProcessMetaData> processes = new ArrayList<>();
       processes.add(permission(etl(RUN_PLAN, "Run backup now", BackupsTables.PLAN, RunPlanTransform.class, RunPlanLoad.class,
-         "This is a preview of what the backup will include. Nothing is written until you continue."), BackupsPermissions.RUN));
+         "This is a preview of what the backup will include. Nothing is written until you continue.").withSupplementalMetaData(durability()), BackupsPermissions.RUN));
+      processes.add(DefaultPlanProcess.process());
       processes.addAll(VerifyProcesses.processes());
       processes.addAll(RestoreProcesses.processes());
       processes.addAll(CatalogProcesses.processes());
@@ -81,6 +89,40 @@ public final class BackupsProcesses
 
 
 
+   /*******************************************************************************
+    ** External writes cannot commit with the job chunk. Require review before retry
+    ** instead of claiming that replay is naturally idempotent.
+    *******************************************************************************/
+   public static DurableProcessMetaData durability()
+   {
+      return (new DurableProcessMetaData().withEligible(true).withAlwaysDurable(true)
+         .withChunkSize(1).withIdempotency(JobIdempotency.NONE));
+   }
+
+
+
+   /*******************************************************************************
+    ** A durable worker must see failures/cancellation instead of a successful ETL
+    ** summary. Inline callers retain the existing result-screen behavior.
+    *******************************************************************************/
+   public static void requireSuccessfulJob(BackupRunResult result) throws QException
+   {
+      if(JobChunkScope.current().isEmpty())
+      {
+         return;
+      }
+      if("CANCELLED".equals(result.getStatus()))
+      {
+         throw (new JobInterruptedException(JobInterruptedException.Reason.CANCEL));
+      }
+      if(!result.isUsable())
+      {
+         throw (new QException(result.getErrorMessage() == null ? "Backup did not complete" : result.getErrorMessage()));
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Record the permission a process needs.
     *******************************************************************************/
@@ -138,6 +180,10 @@ public final class BackupsProcesses
       private final ProcessSummaryLine tables   = line(Status.INFO, "table will be backed up", "tables will be backed up", "table was backed up", "tables were backed up");
       private final ProcessSummaryLine excluded = line(Status.INFO, "table is left out by its role or scope", "tables are left out by their role or scope",
          "table was left out by its role or scope", "tables were left out by their role or scope");
+      private final ProcessSummaryLine unreadable = line(Status.INFO, "table is left out because its backend cannot be queried (storage only)",
+         "tables are left out because their backend cannot be queried (storage only)", "table was left out because its backend cannot be queried (storage only)",
+         "tables were left out because their backend cannot be queried (storage only)");
+      private final Set<String>        unreadableNames = new TreeSet<>();
       private final ProcessSummaryLine inactive = line(Status.WARNING, "plan is not active (it can still be run by hand)", "plans are not active (they can still be run by hand)",
          "plan was not active", "plans were not active");
       private final ProcessSummaryLine invalid  = line(Status.ERROR, "plan cannot run", "plans cannot run", "plan could not run", "plans could not run");
@@ -165,6 +211,14 @@ public final class BackupsProcesses
                plans.incrementCountAndAddPrimaryKey(plan.getValue("id"));
                tables.incrementCount(scope.tables().size());
                excluded.incrementCount(scope.excluded().size());
+               for(ScopeResolver.ExcludedTable out : scope.excluded())
+               {
+                  if(ScopeResolver.NOT_QUERYABLE.equals(out.reason()) && unreadableNames.add(out.name()))
+                  {
+                     unreadable.incrementCount();
+                     unreadable.setMessageSuffix(": " + String.join(", ", unreadableNames));
+                  }
+               }
                if(!Boolean.TRUE.equals(plan.getValueBoolean("isActive")))
                {
                   inactive.incrementCountAndAddPrimaryKey(plan.getValue("id"));
@@ -195,6 +249,7 @@ public final class BackupsProcesses
          plans.addSelfToListIfAnyCount(lines);
          tables.addSelfToListIfAnyCount(lines);
          excluded.addSelfToListIfAnyCount(lines);
+         unreadable.addSelfToListIfAnyCount(lines);
          inactive.addSelfToListIfAnyCount(lines);
          invalid.addSelfToListIfAnyCount(lines);
          return (lines);
@@ -228,8 +283,9 @@ public final class BackupsProcesses
       {
          for(QRecord plan : input.getRecords())
          {
-            BackupRunResult result = new BackupRunCoordinator().runPlan(plan, "MANUAL");
+            BackupRunResult result = new BackupRunCoordinator().runPlan(plan, "MANUAL", JobProgress.of(input), null);
             results.add(result);
+            requireSuccessfulJob(result);
             switch(result.getStatus())
             {
                case "SUCCEEDED" -> succeeded.incrementCountAndAddPrimaryKey(result.getRunId());

src/main/java/com/kingsrook/qbits/backups/processes/DefaultPlanProcess.java

@@ -0,0 +1,314 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.processes;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qbits.backups.model.BackupsTables;
+import com.kingsrook.qbits.backups.permissions.BackupsPermissions;
+import com.kingsrook.qbits.backups.plan.BackupPlanValidator;
+import com.kingsrook.qbits.backups.run.BackupCatalog;
+import com.kingsrook.qbits.backups.scope.PlanSettings;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+
+
+/*******************************************************************************
+ ** "Create default plan": a table action on backups_plan that adds one working
+ ** plan, "Nightly full", so a fresh install has something to run.  The only thing
+ ** it asks for is the storage bucket alias, because that is the one value the qbit
+ ** cannot know (it names a bucket the host's storage resolver provides).  Review
+ ** shows the plan that will be created and checks it against the same rules a
+ ** hand-made plan must pass; Result links the new plan.
+ **
+ ** Insert-only: when a plan with the key nightly-full exists (edited or not), nothing
+ ** is created or changed.  The plan goes through the normal insert, so the owner is
+ ** the user running the action and the schedule syncs like any other plan.
+ *******************************************************************************/
+public final class DefaultPlanProcess
+{
+   public static final String NAME        = "backupsPlanCreateDefault";
+   public static final String FIELD_ALIAS = "storageBucketAlias";
+   public static final String PLAN_KEY    = "nightly-full";
+   public static final String PLAN_NAME   = "Nightly full";
+   public static final String CRON        = "0 0 2 * * ?";
+   public static final String VERIFY_CRON = "0 0 4 ? * SUN";
+
+
+
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private DefaultPlanProcess()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** The process: a table-level action with one question (the bucket alias).
+    *******************************************************************************/
+   public static QProcessMetaData process()
+   {
+      Map<String, Serializable> values = new HashMap<>();
+      values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, BackupsTables.PLAN);
+      values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, BackupsTables.PLAN);
+      values.put(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION, true);
+      values.put(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true);
+      values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, "This plan will be created. Nothing is written until you continue. You can edit every setting afterwards.");
+      QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(Extract.class, Transform.class, Load.class, values);
+      process.setName(NAME);
+      process.setLabel("Create default plan");
+      process.setIcon(new QIcon("playlist_add"));
+      process.setTableName(BackupsTables.PLAN);
+      process.withStep(0, new QFrontendStepMetaData()
+         .withName("input")
+         .withLabel("Where should the backups go?")
+         .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
+         .withFormField(new QFieldMetaData(FIELD_ALIAS, QFieldType.STRING).withLabel("Storage bucket alias").withIsRequired(true)
+            .withHelpContent(new QHelpContent("The alias of a bucket your application provides to Backups (never a key or an endpoint). It becomes the plan's Storage bucket alias."))));
+      process.setMinInputRecords(0);
+      process.setPermissionRules(BackupsTables.permissionRules());
+      return (BackupsProcesses.permission(process, BackupsPermissions.ADMIN));
+   }
+
+
+
+   /*******************************************************************************
+    ** The values of the default plan for a bucket alias.  Every setting is explicit
+    ** so the plan does not depend on the backend applying field defaults.
+    *******************************************************************************/
+   public static Map<String, Serializable> planValues(String alias)
+   {
+      Map<String, Serializable> plan = new LinkedHashMap<>();
+      plan.put("name", PLAN_NAME);
+      plan.put("planKey", PLAN_KEY);
+      plan.put("description", "Every night: all data and audit tables of the application, and the files they reference, to the storage bucket. Edit any setting on this plan.");
+      plan.put("isActive", true);
+      plan.put("scopeType", "INSTANCE");
+      plan.put("includeRoles", "DATA,AUDIT");
+      plan.put("storageBucketAlias", alias);
+      plan.put("critical", true);
+      plan.put("format", "JSONL");
+      plan.put("compression", "GZIP");
+      plan.put("encryptionMode", "NONE");
+      plan.put("sensitiveFieldPolicy", "EXCLUDE");
+      plan.put("heavyFieldPolicy", "INLINE");
+      plan.put("fileObjectPolicy", "COPY");
+      plan.put("fileMissingPolicy", "WARN");
+      plan.put("consistency", "SNAPSHOT_PREFERRED");
+      plan.put("useReadReplica", false);
+      plan.put("incrementalMode", "FULL");
+      plan.put("nativeDump", "NONE");
+      plan.put("cronExpression", CRON);
+      plan.put("cronTimeZoneId", "UTC");
+      plan.put("slaHours", 26);
+      plan.put("verifySlaDays", 8);
+      plan.put("verifyFileSamplePercent", 100);
+      plan.put("verifyCronExpression", VERIFY_CRON);
+      plan.put("verifyLevel", "CONTENT");
+      plan.put("retainDaily", 7);
+      plan.put("retainWeekly", 4);
+      plan.put("retainMonthly", 12);
+      plan.put("retainYearly", 0);
+      plan.put("retainMinimum", 3);
+      plan.put("objectLockMode", "NONE");
+      plan.put("maxDurationMinutes", 240);
+      plan.put("pageSize", 5000);
+      plan.put("targetPartBytes", 268435456L);
+      return (plan);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the default plan already exists.
+    *******************************************************************************/
+   private static boolean exists() throws QException
+   {
+      return (!new BackupCatalog().where(BackupsTables.PLAN, "planKey", PLAN_KEY).isEmpty());
+   }
+
+
+
+   /*******************************************************************************
+    ** Extract: the plan to create, or nothing when it exists already.
+    *******************************************************************************/
+   public static class Extract extends AbstractExtractStep
+   {
+      /*******************************************************************************
+       ** {@inheritDoc}
+       *******************************************************************************/
+      @Override
+      public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
+      {
+         String alias = input.getValueString(FIELD_ALIAS);
+         if(alias == null || alias.isBlank())
+         {
+            throw (new QUserFacingException("Enter the storage bucket alias the backups should go to."));
+         }
+         if(!exists())
+         {
+            QRecord record = new QRecord().withTableName(BackupsTables.PLAN);
+            record.setValues(new LinkedHashMap<>(planValues(alias.strip())));
+            getRecordPipe().addRecord(record);
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Review: the plan, checked against the rules a plan must pass.
+    *******************************************************************************/
+   public static class Transform extends AbstractTransformStep implements ProcessSummaryProviderInterface
+   {
+      private final ProcessSummaryLine create   = BackupsProcesses.line(Status.OK, "plan will be created", "plans will be created", "plan was created", "plans were created");
+      private final ProcessSummaryLine invalid  = BackupsProcesses.line(Status.ERROR, "plan cannot be created", "plans cannot be created", "plan could not be created", "plans could not be created");
+      private final List<String>       problems = new ArrayList<>();
+      private boolean                  sawRecord;
+
+
+
+      /*******************************************************************************
+       ** {@inheritDoc}
+       *******************************************************************************/
+      @Override
+      public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
+      {
+         for(QRecord record : input.getRecords())
+         {
+            sawRecord = true;
+            BackupPlanValidator.Result result = BackupPlanValidator.validate(PlanSettings.of(record), null, false, List.of());
+            boolean nameTaken = !new BackupCatalog().where(BackupsTables.PLAN, "name", record.getValueString("name")).isEmpty();
+            if(nameTaken)
+            {
+               problems.add("A plan named " + record.getValueString("name") + " already exists. Rename it or edit that plan.");
+            }
+            if(result.ok() && !nameTaken)
+            {
+               create.incrementCountAndAddPrimaryKey(record.getValueString("planKey"));
+               output.getRecords().add(record);
+            }
+            else
+            {
+               invalid.incrementCount();
+               problems.addAll(result.errors());
+            }
+         }
+      }
+
+
+
+      /*******************************************************************************
+       ** {@inheritDoc}
+       *******************************************************************************/
+      @Override
+      public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
+      {
+         ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
+         create.addSelfToListIfAnyCount(lines);
+         invalid.setBulletsOfText(new ArrayList<>(problems));
+         invalid.addSelfToListIfAnyCount(lines);
+         if(!sawRecord)
+         {
+            lines.add(new ProcessSummaryLine(Status.INFO, null, "A plan with the key " + PLAN_KEY + " already exists, so nothing will be created. Edit that plan instead."));
+         }
+         return (lines);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Load: insert through the normal insert (validation, owner, schedule).
+    *******************************************************************************/
+   public static class Load extends AbstractLoadStep implements ProcessSummaryProviderInterface
+   {
+      private final ProcessSummaryLine created  = BackupsProcesses.line(Status.OK, "plan will be created", "plans will be created", "plan was created", "plans were created");
+      private final ProcessSummaryLine failed   = BackupsProcesses.line(Status.ERROR, "plan could not be created", "plans could not be created", "plan could not be created", "plans could not be created");
+      private final List<String>       problems = new ArrayList<>();
+
+
+
+      /*******************************************************************************
+       ** {@inheritDoc}
+       *******************************************************************************/
+      @Override
+      public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
+      {
+         InsertInput insert = new InsertInput(BackupsTables.PLAN);
+         insert.setRecords(input.getRecords());
+         InsertOutput inserted = new InsertAction().execute(insert);
+         for(QRecord record : inserted.getRecords())
+         {
+            if(record.getErrors() == null || record.getErrors().isEmpty())
+            {
+               created.incrementCountAndAddPrimaryKey(record.getValue("id"));
+               output.getRecords().add(record);
+            }
+            else
+            {
+               failed.incrementCount();
+               record.getErrors().forEach(error -> problems.add(error.getMessage()));
+            }
+         }
+      }
+
+
+
+      /*******************************************************************************
+       ** {@inheritDoc}
+       *******************************************************************************/
+      @Override
+      public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
+      {
+         ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
+         created.addSelfToListIfAnyCount(lines);
+         failed.setBulletsOfText(new ArrayList<>(problems));
+         failed.addSelfToListIfAnyCount(lines);
+         return (lines);
+      }
+   }
+}

src/main/java/com/kingsrook/qbits/backups/processes/ExportTableProcess.java

@@ -118,7 +118,7 @@ public final class ExportTableProcess
             continue;
          }
          TableRole role = TableRoles.roleOf(qInstance, table);
-         if(role == TableRole.DATA || role == TableRole.AUDIT || role == TableRole.DERIVED)
+         if((role == TableRole.DATA || role == TableRole.AUDIT || role == TableRole.DERIVED) && TableRoles.isQueryable(qInstance, table))
          {
             processes.add(processFor(table.getName()));
          }

src/main/java/com/kingsrook/qbits/backups/processes/LifecycleProcesses.java

@@ -53,6 +53,7 @@ public final class LifecycleProcesses
    {
       List<QProcessMetaData> processes = new ArrayList<>();
       processes.add(BackupScheduledRunProcess.processMetaData());
+      processes.add(com.kingsrook.qbits.backups.schedule.BackupScheduledVerifyProcess.processMetaData());
       processes.addAll(CancelRunProcess.processes());
       processes.add(ResumeRunProcess.process());
       processes.add(TestDestinationProcess.process());

src/main/java/com/kingsrook/qbits/backups/processes/RestoreProcesses.java

@@ -33,6 +33,8 @@ import com.kingsrook.qbits.backups.restore.RestoreRequest;
 import com.kingsrook.qbits.backups.restore.RestoreSummaries;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.jobs.progress.JobPhase;
+import com.kingsrook.qqq.backend.core.jobs.progress.JobProgress;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
@@ -107,7 +109,7 @@ public final class RestoreProcesses
       process.setLabel(label);
       process.setTableName(table);
       process.setPermissionRules(BackupsTables.permissionRules());
-      return (process);
+      return (process.withSupplementalMetaData(BackupsProcesses.durability()));
    }
 
 
@@ -241,13 +243,38 @@ public final class RestoreProcesses
       @Override
       public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
       {
+         JobProgress progress = JobProgress.of(input);
+         progress.phases(JobPhase.of("restore", "Restore backups", 100)).phase("restore");
+         Map<Integer, Set<String>> sources = sources(input.getRecords());
+         progress.total(sources.size(), "backups").done(0);
+         int done = 0;
          RestoreEngine engine = new RestoreEngine();
-         for(Map.Entry<Integer, Set<String>> source : sources(input.getRecords()).entrySet())
+         try
          {
-            RestoreRequest request = RestoreRequest.fromInput(input, source.getKey(), new ArrayList<>(source.getValue()));
-            outcomes.add(engine.execute(request));
+            for(Map.Entry<Integer, Set<String>> source : sources.entrySet())
+            {
+               progress.message("Restoring backup " + source.getKey());
+               RestoreRequest request = RestoreRequest.fromInput(input, source.getKey(), new ArrayList<>(source.getValue()));
+               outcomes.add(engine.execute(request));
+               progress.info("Restored backup " + source.getKey());
+               progress.done(++done);
+            }
+            progress.getSnapshot().getPhases().getFirst().setStatus(JobPhase.Status.DONE);
+            output.getRecords().addAll(input.getRecords());
+         }
+         catch(QException | RuntimeException e)
+         {
+            progress.getSnapshot().getPhases().getFirst().setStatus(JobPhase.Status.FAILED);
+            if(com.kingsrook.qqq.backend.core.jobs.plans.JobInterruptedException.find(e) == null)
+            {
+               progress.error("Restore failed (" + e.getClass().getSimpleName() + ")");
+            }
+            throw (e);
+         }
+         finally
+         {
+            progress.flush();
          }
-         output.getRecords().addAll(input.getRecords());
       }
 
 

src/main/java/com/kingsrook/qbits/backups/processes/ResumeRunProcess.java

@@ -70,7 +70,7 @@ public final class ResumeRunProcess
    public static QProcessMetaData process()
    {
       return (BackupsProcesses.permission(BackupsProcesses.etl(NAME, "Resume", BackupsTables.RUN, Transform.class, Load.class,
-         "This is a preview of where each selected run will carry on. Nothing is written until you continue."), BackupsPermissions.RUN));
+         "This is a preview of where each selected run will carry on. Nothing is written until you continue.").withSupplementalMetaData(BackupsProcesses.durability()), BackupsPermissions.RUN));
    }
 
 
@@ -167,7 +167,8 @@ public final class ResumeRunProcess
       {
          for(QRecord run : input.getRecords())
          {
-            BackupRunResult result = new BackupRunCoordinator().resume(run);
+            BackupRunResult result = new BackupRunCoordinator().resume(run, com.kingsrook.qqq.backend.core.jobs.progress.JobProgress.of(input));
+            BackupsProcesses.requireSuccessfulJob(result);
             switch(result.getStatus())
             {
                case "SUCCEEDED" -> succeeded.incrementCountAndAddPrimaryKey(run.getValue("id"));

src/main/java/com/kingsrook/qbits/backups/processes/VerifyProcesses.java

@@ -30,6 +30,9 @@ import com.kingsrook.qbits.backups.verify.VerificationOutcome;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.jobs.plans.JobChunkScope;
+import com.kingsrook.qqq.backend.core.jobs.progress.JobPhase;
+import com.kingsrook.qqq.backend.core.jobs.progress.JobProgress;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
@@ -85,7 +88,7 @@ public final class VerifyProcesses
    {
       List<QProcessMetaData> processes = new ArrayList<>();
       processes.add(BackupsProcesses.permission(BackupsProcesses.etl(VERIFY, "Verify backup", BackupsTables.RUN, VerifyTransform.class, VerifyLoad.class,
-         "This checks that the backup's objects are all in storage, then reads every byte back, decrypts and decodes it, and compares counts and hashes with the manifest. Nothing is changed in the backup."),
+         "This checks that the backup's objects are all in storage, then reads every byte back, decrypts and decodes it, and compares counts and hashes with the manifest. Nothing is changed in the backup.").withSupplementalMetaData(BackupsProcesses.durability()),
          BackupsPermissions.RUN));
       processes.add(checkSlaProcess(BackupsRuntime.getConfig()));
       return (processes);
@@ -220,12 +223,31 @@ public final class VerifyProcesses
       @Override
       public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
       {
+         JobProgress progress = JobProgress.of(input);
+         progress.phases(JobPhase.of("verify", "Verify", 100)).phase("verify").total(input.getRecords().size(), "backups").done(0);
+         int done = 0;
          String level = input.getValueString(FIELD_VERIFY_LEVEL) == null ? BackupVerifier.CONTENT : input.getValueString(FIELD_VERIFY_LEVEL);
          String user  = QContext.getQSession() == null || QContext.getQSession().getUser() == null ? "system" : QContext.getQSession().getUser().getIdReference();
          for(QRecord run : input.getRecords())
          {
+            JobChunkScope.checkBetweenPages();
+            progress.message("Verifying backup " + run.getValueInteger("id"));
             List<VerificationOutcome> outcomes = new BackupVerifier().verifyThrough(run.getValueInteger("id"), level, "MANUAL", user);
             VerificationOutcome       last     = outcomes.get(outcomes.size() - 1);
+            for(VerificationOutcome outcome : outcomes)
+            {
+               progress.info("Verification " + outcome.level() + ": " + outcome.status());
+            }
+            if(!last.passed())
+            {
+               progress.getSnapshot().getPhases().getFirst().setStatus(JobPhase.Status.FAILED);
+               progress.error("Backup verification " + last.status());
+               if(JobChunkScope.current().isPresent())
+               {
+                  throw (new QException("Backup verification " + last.status()));
+               }
+            }
+            progress.done(++done);
             switch(last.status())
             {
                case "PASSED" -> passed.incrementCountAndAddPrimaryKey(run.getValue("id"));
@@ -248,6 +270,8 @@ public final class VerifyProcesses
             }
             output.getRecords().add(run);
          }
+         progress.getSnapshot().getPhases().getFirst().setStatus(failed.getCount() > 0 || errored.getCount() > 0 ? JobPhase.Status.FAILED : JobPhase.Status.DONE);
+         progress.flush();
       }
 
 

src/main/java/com/kingsrook/qbits/backups/restore/RestoreEngine.java

@@ -49,6 +49,8 @@ import com.kingsrook.qqq.backend.core.actions.audits.AuditAction;
 import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.jobs.plans.JobChunkScope;
+import com.kingsrook.qqq.backend.core.jobs.plans.JobInterruptedException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
@@ -105,7 +107,8 @@ public class RestoreEngine
       Map<String, String> targets  = new LinkedHashMap<>();
       QInstance           instance = QContext.getQInstance();
 
-      if(!BackupsPermissionChecker.has(BackupsPermissions.RESTORE))
+      if(!BackupsPermissionChecker.has(BackupsPermissions.RESTORE)
+         || (request.getWriter().usesCoreRestoreFlag() && (QContext.getQSession() == null || !QContext.getQSession().hasPermission(BackupsPermissions.RESTORE))))
       {
          errors.add("Restoring needs the " + BackupsPermissions.RESTORE + " permission.");
       }
@@ -372,6 +375,15 @@ public class RestoreEngine
       }
       catch(QException | RuntimeException e)
       {
+         JobInterruptedException interrupted = JobInterruptedException.find(e);
+         if(interrupted != null)
+         {
+            if(interrupted.getReason() == JobInterruptedException.Reason.CANCEL)
+            {
+               catalog.update(BackupsTables.RESTORE_RUN, restoreRunId, Map.of("status", "CANCELLED", "endDate", BackupsRuntime.getClock().instant()));
+            }
+            throw (interrupted);
+         }
          LOG.warn("Restore failed", e, logPair("restoreRunId", restoreRunId));
          String message = e.getClass().getSimpleName() + ": " + e.getMessage();
          try
@@ -446,6 +458,7 @@ public class RestoreEngine
       }
 
       BackupRunRequest backup = new BackupRunRequest()
+         .withProgress(new com.kingsrook.qqq.backend.core.jobs.progress.JobProgress((com.kingsrook.qqq.backend.core.actions.async.AsyncJobCallback) null))
          .withSettings(settings)
          .withScope(new ScopeResolver(QContext.getQInstance()).resolve(settings, rules))
          .withTrigger("PRE_RESTORE")
@@ -506,10 +519,11 @@ public class RestoreEngine
     ** index.  A conflicting file (different content at the target) is only replaced
     ** by a REPLACE restore.
     *******************************************************************************/
-   private void restoreFiles(RestoreRequest request, Plan plan, RestorePass pass, RestoreOutcome outcome, Integer restoreRunId)
+   private void restoreFiles(RestoreRequest request, Plan plan, RestorePass pass, RestoreOutcome outcome, Integer restoreRunId) throws QException
    {
       for(RestorePass.FileWork work : pass.getFileWork())
       {
+         JobChunkScope.checkBetweenPages();
          if(work.conflict() && !RestoreRequest.MODE_REPLACE.equals(request.getMode()))
          {
             continue;

src/main/java/com/kingsrook/qbits/backups/restore/RestorePass.java

@@ -46,6 +46,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.jobs.plans.JobChunkScope;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
@@ -200,6 +201,7 @@ public class RestorePass
 
       for(String name : order)
       {
+         JobChunkScope.checkBetweenPages();
          restoreTable(name);
       }
       retryPass();
@@ -341,14 +343,14 @@ public class RestorePass
       String    backendType = instance.getBackend(c.target.getBackendName()) == null ? null : instance.getBackend(c.target.getBackendName()).getBackendType();
       RestoreOutcome.TableOutcome o = c.outcome;
 
-      if(c.entry.fingerprint() != null && c.entry.fingerprint().equals(TableFingerprints.of(c.target, backendType)))
+      if(TableFingerprints.matches(c.entry, c.target, backendType))
       {
          o.setSchemaStatus("MATCH");
          return;
       }
 
-      Map<String, Object> backupFields = c.entry.schema() == null ? Map.of() : (Map<String, Object>) c.entry.schema().getOrDefault("fields", Map.of());
-      Map<String, Object> targetFields = (Map<String, Object>) TableFingerprints.descriptor(c.target, backendType).get("fields");
+      Map<String, Object> backupFields = TableFingerprints.fields(c.entry.schema());
+      Map<String, Object> targetFields = TableFingerprints.fields(TableFingerprints.descriptor(c.target, backendType));
       boolean incompatible = false;
       boolean different    = false;
 
@@ -566,6 +568,7 @@ public class RestorePass
     *******************************************************************************/
    private void processPage(TableContext c, List<QRecord> page) throws QException
    {
+      JobChunkScope.checkBetweenPages();
       RestoreOutcome.TableOutcome o = c.outcome;
       o.addRows(page.size());
 

src/main/java/com/kingsrook/qbits/backups/restore/RestoreWriter.java

@@ -28,9 +28,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 
 /*******************************************************************************
  ** How a restore writes (spec 5.3 and 5.4).  The default implementation uses the
- ** plain Insert, Update and Delete actions, and selects the core RestoreWriteFlag
- ** (QRun-IO/qqq#1111) and the core key-sequence resync (#1112) reflectively when the
- ** running QQQ has them.  A backend specific writer can be supplied on the request.
+ ** plain Insert, Update and Delete actions with core RestoreWriteFlag and typed
+ ** key-sequence resync. A backend specific writer can be supplied on the request.
  **
  ** Whatever the implementation, record security, field security, unique keys and
  ** required-field validation still apply to every write.

src/main/java/com/kingsrook/qbits/backups/restore/StandardRestoreWriter.java

@@ -18,7 +18,6 @@ package com.kingsrook.qbits.backups.restore;
 
 
 import java.io.Serializable;
-import java.lang.reflect.Method;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.LinkedHashMap;
@@ -30,8 +29,8 @@ import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.ActionFlag;
+import com.kingsrook.qqq.backend.core.model.actions.tables.RestoreWriteFlag;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
@@ -41,48 +40,14 @@ import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
-import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+import com.kingsrook.qqq.backend.core.modules.backend.SequenceResyncResult;
 
 
 /*******************************************************************************
- ** The default RestoreWriter: the standard Insert, Update and Delete actions.
- **
- ** Stand-ins (replace with the core API when QQQ 4.1.0-RC.5 ships):
- ** <ul>
- ** <li>RestoreWriteFlag (QRun-IO/qqq#1111) is created reflectively when the class is
- ** present.  Without it, the DML audit is omitted (a summary audit is written per
- ** table instead), but created and modified dates may be set by the normal defaults.</li>
- ** <li>resyncKeySequences (QRun-IO/qqq#1112) is called reflectively on the backend
- ** module when it exists; otherwise it is a no-op that the Review reports.</li>
- ** </ul>
+ ** The default RestoreWriter using core restore flags and key sequence resync.
  *******************************************************************************/
 public class StandardRestoreWriter implements RestoreWriter
 {
-   private static final QLogger LOG = QLogger.getLogger(StandardRestoreWriter.class);
-
-   private static final String FLAG_CLASS = "com.kingsrook.qqq.backend.core.model.actions.tables.RestoreWriteFlag";
-
-   private static final Class<?> FLAG = lookup(FLAG_CLASS);
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static Class<?> lookup(String name)
-   {
-      try
-      {
-         return (Class.forName(name));
-      }
-      catch(ClassNotFoundException | LinkageError e)
-      {
-         return (null);
-      }
-   }
-
-
-
    /*******************************************************************************
     ** {@inheritDoc}
     *******************************************************************************/
@@ -148,27 +113,15 @@ public class StandardRestoreWriter implements RestoreWriter
 
 
    /*******************************************************************************
-    ** Create the core flag (when the running QQQ has it) and hand it to the input.
+    ** Apply the core restore behavior requested by the caller.
     *******************************************************************************/
-   private void addFlag(RestoreRequest options, java.util.function.Consumer<ActionFlag> sink) throws QException
+   private void addFlag(RestoreRequest options, java.util.function.Consumer<ActionFlag> sink)
    {
-      if(FLAG == null)
-      {
-         return;
-      }
-      try
-      {
-         Object flag = FLAG.getDeclaredConstructor().newInstance();
-         FLAG.getMethod("withSuppressAutomations", boolean.class).invoke(flag, options.getSuppressAutomations());
-         FLAG.getMethod("withRunCustomizers", boolean.class).invoke(flag, options.getRunCustomizers());
-         FLAG.getMethod("withPerRecordAudit", boolean.class).invoke(flag, options.getPerRecordAudit());
-         FLAG.getMethod("withPublishEvents", boolean.class).invoke(flag, options.getPublishEvents());
-         sink.accept((ActionFlag) flag);
-      }
-      catch(ReflectiveOperationException | ClassCastException e)
-      {
-         throw (new QException("Could not use the core restore flag: " + e.getMessage(), e));
-      }
+      sink.accept(new RestoreWriteFlag()
+         .withSuppressAutomations(options.getSuppressAutomations())
+         .withRunCustomizers(options.getRunCustomizers())
+         .withPerRecordAudit(options.getPerRecordAudit())
+         .withPublishEvents(options.getPublishEvents()));
    }
 
 
@@ -179,7 +132,7 @@ public class StandardRestoreWriter implements RestoreWriter
    @Override
    public boolean usesCoreRestoreFlag()
    {
-      return (FLAG != null);
+      return (true);
    }
 
 
@@ -190,24 +143,7 @@ public class StandardRestoreWriter implements RestoreWriter
    @Override
    public boolean canResyncSequences()
    {
-      return (resyncMethod() != null);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static Method resyncMethod()
-   {
-      try
-      {
-         return (QBackendModuleInterface.class.getMethod("resyncKeySequences", QBackendMetaData.class, Collection.class));
-      }
-      catch(NoSuchMethodException | LinkageError e)
-      {
-         return (null);
-      }
+      return (true);
    }
 
 
@@ -218,12 +154,6 @@ public class StandardRestoreWriter implements RestoreWriter
    @Override
    public List<String> resyncSequences(Collection<QTableMetaData> tables) throws QException
    {
-      Method method = resyncMethod();
-      if(method == null)
-      {
-         return (List.of());
-      }
-
       Map<String, List<QTableMetaData>> byBackend = new LinkedHashMap<>();
       for(QTableMetaData table : tables)
       {
@@ -234,23 +164,14 @@ public class StandardRestoreWriter implements RestoreWriter
       for(Map.Entry<String, List<QTableMetaData>> entry : byBackend.entrySet())
       {
          QBackendMetaData backend = QContext.getQInstance().getBackend(entry.getKey());
-         try
+         QBackendModuleInterface module = new QBackendModuleDispatcher().getQBackendModule(backend);
+         for(SequenceResyncResult result : module.resyncKeySequences(backend, entry.getValue()))
          {
-            QBackendModuleInterface module = new QBackendModuleDispatcher().getQBackendModule(backend);
-            for(Object result : (List<?>) method.invoke(module, backend, entry.getValue()))
+            if(result.getStatus() == SequenceResyncResult.Status.RESYNCED)
             {
-               Object status = result.getClass().getMethod("getStatus").invoke(result);
-               if("RESYNCED".equals(String.valueOf(status)))
-               {
-                  resynced.add(String.valueOf(result.getClass().getMethod("getTableName").invoke(result)));
-               }
+               resynced.add(result.getTableName());
             }
          }
-         catch(ReflectiveOperationException e)
-         {
-            LOG.warn("Could not resync key sequences", e, logPair("backend", entry.getKey()));
-            throw (new QException("Could not resync key sequences on backend " + entry.getKey() + ": " + e.getMessage(), e));
-         }
       }
       return (resynced);
    }

src/main/java/com/kingsrook/qbits/backups/run/BackupEngine.java

@@ -17,16 +17,21 @@
 package com.kingsrook.qbits.backups.run;
 
 
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.io.Serializable;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.time.Instant;
 import java.time.ZoneOffset;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.UUID;
 import java.util.function.Function;
@@ -35,18 +40,24 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.kingsrook.qbits.backups.BackupsRuntime;
 import com.kingsrook.qbits.backups.alerts.BackupAlertSink;
 import com.kingsrook.qbits.backups.codec.HashingOutputStream;
-import com.kingsrook.qbits.backups.events.BackupEvents;
 import com.kingsrook.qbits.backups.codec.TableArtifactWriter;
+import com.kingsrook.qbits.backups.events.BackupEvents;
 import com.kingsrook.qbits.backups.files.FileCopier;
 import com.kingsrook.qbits.backups.files.FileReference;
 import com.kingsrook.qbits.backups.files.FileReferenceProvider;
 import com.kingsrook.qbits.backups.manifest.BackupManifest;
 import com.kingsrook.qbits.backups.manifest.TableFingerprints;
+import com.kingsrook.qbits.backups.metrics.BackupsMetrics;
 import com.kingsrook.qbits.backups.model.BackupsTables;
 import com.kingsrook.qbits.backups.scope.ScopeResolver;
 import com.kingsrook.qbits.backups.storage.BackupStorage;
 import com.kingsrook.qbits.backups.storage.WriteOptions;
 import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.jobs.plans.JobChunkScope;
+import com.kingsrook.qqq.backend.core.jobs.plans.JobInterruptedException;
+import com.kingsrook.qqq.backend.core.jobs.progress.JobPhase;
+import com.kingsrook.qqq.backend.core.jobs.progress.JobProgress;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -66,7 +77,7 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
  ** content-addressed store; checksums.sha256 is written; and the manifest is
  ** written LAST, then checked with a head.  A run with no manifest did not
  ** complete.  The run row is updated every page (heartbeat), and a failure marks
- ** the run FAILED with a scrubbed message (never row values).
+ ** the run FAILED with a sanitized message and detail (no secrets, no row values).
  *******************************************************************************/
 public class BackupEngine
 {
@@ -85,13 +96,19 @@ public class BackupEngine
 
    /*******************************************************************************
     ** Run a backup.  Never throws for a backup failure: the result carries the
-    ** status and message (and the run row is FAILED).
+    ** status and message (and the run row is FAILED). Cooperative job interrupts
+    ** propagate as QException so the durable worker handles cancellation/lease loss.
     *******************************************************************************/
-   public BackupRunResult execute(BackupRunRequest request)
+   public BackupRunResult execute(BackupRunRequest request) throws QException
    {
+      JobProgress progress = request.getProgress();
+      if(!"prepare".equals(progress.getSnapshot().getCurrentPhase()))
+      {
+         prepare(progress);
+      }
       BackupRunResult result  = new BackupRunResult();
       Instant         started = request.getClock().instant();
-      String          runKey  = request.getResume() != null ? request.getResume().runKey() : RUN_KEY_FORMAT.format(started) + "-" + UUID.randomUUID().toString().substring(0, 4);
+      String          runKey  = request.getResume() != null ? request.getResume().runKey() : request.getRunKey() != null ? request.getRunKey() : RUN_KEY_FORMAT.format(started) + "-" + UUID.randomUUID().toString().substring(0, 4);
       String          planPrefix = request.getSettings().string("prefix", "qqq-backups");
       String          instancePrefix = trimSlashes(planPrefix) + "/" + request.getInstanceName() + "/";
       String          runPrefix  = request.getResume() != null ? request.getResume().runPrefix() : instancePrefix + request.getSettings().planKey() + "/" + DATE_PATH.format(started) + "/" + runKey + "/";
@@ -125,6 +142,7 @@ public class BackupEngine
             runValues.remove("startDate");
             runValues.put("endDate", null);
             runValues.put("errorMessage", null);
+            runValues.put("errorDetail", null);
          }
          if(request.getExistingRunId() != null)
          {
@@ -138,27 +156,55 @@ public class BackupEngine
       }
       catch(Exception e)
       {
+         LOG.warn("Could not create the backup run record", e, logPair("runKey", runKey));
+         String message = FailureText.summary("Could not create the run record", e);
          result.setStatus("FAILED");
-         result.setErrorMessage("Could not create the run record: " + e.getMessage());
+         result.setErrorMessage(message);
+         result.setErrorDetail(FailureText.detail(e));
+         failProgress(progress, message);
          return (result);
       }
 
       try
       {
+         checkJob();
+         if(request.getResume() != null)
+         {
+            long complete = request.getResume().tables().values().stream().filter(ResumeState.TableState::complete).count();
+            progress.info("Resuming run " + result.getRunId() + ": " + complete + " tables already done");
+         }
          doRun(request, result, runPrefix, instancePrefix, started);
+         completePhase(progress, JobPhase.Status.DONE);
       }
       catch(CancelledException e)
       {
-         finish(request, result, "CANCELLED", "Cancelled", started);
+         finish(request, result, "CANCELLED", "Cancelled", null, started);
+         progress.info("Backup cancelled");
       }
       catch(TimedOutException e)
       {
-         finish(request, result, "TIMED_OUT", e.getMessage(), started);
+         finish(request, result, "TIMED_OUT", e.getMessage(), null, started);
+         failProgress(progress, e.getMessage());
       }
       catch(Exception e)
       {
+         JobInterruptedException interrupted = JobInterruptedException.find(e);
+         if(interrupted != null)
+         {
+            if(interrupted.getReason() != JobInterruptedException.Reason.LEASE_LOST)
+            {
+               finish(request, result, interrupted.getReason() == JobInterruptedException.Reason.CANCEL ? "CANCELLED" : "ABANDONED", "Job interrupted", null, started);
+            }
+            throw (interrupted);
+         }
          LOG.warn("Backup run failed", e, logPair("runKey", runKey));
-         finish(request, result, "FAILED", scrub(e), started);
+         String message = scrub(e);
+         failProgress(progress, message);
+         finish(request, result, "FAILED", message, FailureText.detail(e), started);
+      }
+      finally
+      {
+         progress.flush();
       }
       return (result);
    }
@@ -170,6 +216,7 @@ public class BackupEngine
     *******************************************************************************/
    private void doRun(BackupRunRequest request, BackupRunResult result, String runPrefix, String instancePrefix, Instant started) throws Exception
    {
+      JobProgress      progress = request.getProgress();
       QInstance        qInstance = QContext.getQInstance();
       BackupStorage    storage   = request.getStorage();
       var              settings  = request.getSettings();
@@ -188,6 +235,7 @@ public class BackupEngine
          throw (new IllegalStateException("The sensitive field policy INCLUDE_ENCRYPTED_ONLY requires client-side encryption."));
       }
       String consistency = consistencyFor(settings.string("consistency", "SNAPSHOT_PREFERRED"), result);
+      result.getWarnings().forEach(progress::warn);
       catalog.update(BackupsTables.RUN, result.getRunId(), Map.of("consistencyAchieved", result.getConsistencyAchieved()));
 
       //////////////////////////////////////////////////////////////////////////////////////
@@ -230,10 +278,15 @@ public class BackupEngine
       long                            fileBytes      = 0;
       long                            fileBytesRef   = 0;
 
-      try
+      progress.phase("tables").total(request.getScope().tables().size(), "tables").done(0).bytesDone(0);
+      try(PendingFiles pendingFiles = new PendingFiles())
       {
          for(ScopeResolver.ResolvedTable resolved : request.getScope().tables())
          {
+            checkJob();
+            long tableStarted = System.nanoTime();
+            long tableBytesBefore = bytesStored;
+            progress.message("Backing up " + resolved.name() + " (" + (done + 1) + " of " + request.getScope().tables().size() + ")");
             QTableMetaData table = qInstance.getTable(resolved.name());
             String         backendType = qInstance.getBackend(table.getBackendName()) == null ? null : qInstance.getBackend(table.getBackendName()).getBackendType();
 
@@ -242,6 +295,7 @@ public class BackupEngine
 
             ResumeState.TableState prior = request.getResume() == null ? null : request.getResume().table(table.getName());
 
+            long[] tableWritten = { 0 };
             List<TableArtifactWriter> writers = new ArrayList<>();
             String tablePrefix = runPrefix + "tables/" + table.getName() + "/";
             Map<TableArtifactWriter, Integer> lastArtifactIds = new LinkedHashMap<>();
@@ -277,6 +331,9 @@ public class BackupEngine
                   {
                      try
                      {
+                        tableWritten[0] += part.bytesStored();
+                        progress.bytesDone(tableBytesBefore + tableWritten[0]);
+                        progress.info("Wrote " + part.key().substring(runPrefix.length()));
                         lastArtifactIds.put(writer, catalog.insertArtifact(result.getRunId(), "TABLE_DATA", table.getName(), part.partNumber(), part.key().substring(runPrefix.length()),
                            part.format(), part.compression(), part.encrypted(), part.rows(), part.bytesRaw(), part.bytesStored(), part.sha256Stored(), part.sha256Raw(), null,
                            part.firstKey(), part.lastKey(), TableFingerprints.of(table, backendType)));
@@ -296,6 +353,8 @@ public class BackupEngine
                   if(canContinue)
                   {
                      TableArtifactWriter writer = writers.get(0);
+                     tableWritten[0] = prior.parts().stream().mapToLong(TableArtifactWriter.PartResult::bytesStored).sum();
+                     progress.bytesDone(tableBytesBefore + tableWritten[0]);
                      writer.resumeFrom(prior.parts());
                      seedRowsHash(storage, runPrefix, prior.parts(), writer);
                      lastArtifactIds.put(writer, prior.artifactIds().get(prior.artifactIds().size() - 1));
@@ -368,8 +427,31 @@ public class BackupEngine
             }
             rows += counter[0];
 
-            for(FileReference ref : references)
+            pendingFiles.write(references);
+
+            Map<String, String> keyRange = new LinkedHashMap<>();
+            keyRange.put("min", allResults.isEmpty() ? null : allResults.get(0).firstKey());
+            keyRange.put("max", allResults.isEmpty() ? null : allResults.get(allResults.size() - 1).lastKey());
+            manifestTables.add(new BackupManifest.TableEntry(table.getName(), resolved.role().name(), table.getBackendName(), table.getPrimaryKeyField(),
+               fingerprint, TableFingerprints.CORE_ALGORITHM, TableFingerprints.descriptor(table, backendType), resolved.filterJson(), excluded, counter[0], rowsHash, keyRange, parts, consistency));
+            done++;
+            catalogProgress(request, result, done, rows, bytesRaw, bytesStored);
+            progress.done(done).bytesDone(bytesStored);
+            progress.info(String.format(Locale.US, "%s: %,d rows, %.1f MB in %.1f s", table.getName(), counter[0],
+               (bytesStored - tableBytesBefore) / 1_000_000.0, (System.nanoTime() - tableStarted) / 1_000_000_000.0));
+            if(request.getCancelRequested().getAsBoolean())
             {
+               throw (new CancelledException());
+            }
+         }
+         progress.phase("media").total(pendingFiles.count, "files").done(0).bytesDone(bytesStored);
+         try(BufferedReader pending = pendingFiles.openReader())
+         {
+            String line;
+            while((line = pending.readLine()) != null)
+            {
+               FileReference ref = MAPPER.readValue(line, FileReference.class);
+               checkJob();
                if(indexWriter == null)
                {
                   indexWriter = new FileIndexWriter(storage, runPrefix + "files/index-00001.jsonl.gz", result.getRunKey(), fileLockMode, fileRetainUntil);
@@ -388,24 +470,16 @@ public class BackupEngine
                   case FileCopier.MISSING -> missing++;
                   default -> fileBytesRef += Math.max(0, entry.size());
                }
+               progress.done(referenced).bytesDone(bytesStored + fileBytes)
+                  .message("Media: " + copied + " copied, " + deduplicated + " already stored (" + referenced + " of " + pendingFiles.count + ")");
                if(request.getCancelRequested().getAsBoolean())
                {
                   throw (new CancelledException());
                }
             }
-
-            Map<String, String> keyRange = new LinkedHashMap<>();
-            keyRange.put("min", allResults.isEmpty() ? null : allResults.get(0).firstKey());
-            keyRange.put("max", allResults.isEmpty() ? null : allResults.get(allResults.size() - 1).lastKey());
-            manifestTables.add(new BackupManifest.TableEntry(table.getName(), resolved.role().name(), table.getBackendName(), table.getPrimaryKeyField(),
-               fingerprint, TableFingerprints.descriptor(table, backendType), resolved.filterJson(), excluded, counter[0], rowsHash, keyRange, parts, consistency));
-            done++;
-            catalogProgress(result, done, rows, bytesRaw, bytesStored);
-            if(request.getCancelRequested().getAsBoolean())
-            {
-               throw (new CancelledException());
-            }
          }
+
+         progress.info("Media: " + referenced + " files, " + copied + " copied, " + deduplicated + " already stored, " + missing + " missing");
       }
       finally
       {
@@ -415,10 +489,13 @@ public class BackupEngine
          }
       }
 
+      checkJob();
+      progress.phase("upload").message("Writing catalog and manifest");
       List<String> indexKeys = new ArrayList<>();
       if(indexWriter != null)
       {
          indexKeys.add("files/index-00001.jsonl.gz");
+         progress.info("Wrote files/index-00001.jsonl.gz");
          checksums.add(indexWriter.sha256Stored() + "  files/index-00001.jsonl.gz");
          catalog.insertArtifact(result.getRunId(), "FILE_OBJECTS_INDEX", null, 1, "files/index-00001.jsonl.gz", "JSONL", "GZIP", request.getTransform().encrypts(),
             indexWriter.rows(), indexWriter.bytesRaw(), indexWriter.bytesStored(), indexWriter.sha256Stored(), indexWriter.sha256Raw(), null, null, null, null);
@@ -431,7 +508,9 @@ public class BackupEngine
          {
             throw (new IllegalStateException(missing + " referenced files are missing at the source"));
          }
-         result.getWarnings().add(missing + " referenced files were missing at the source");
+         String warning = missing + " referenced files were missing at the source";
+         result.getWarnings().add(warning);
+         progress.warn(warning);
       }
 
       /////////////////////////////////////////////////////////
@@ -439,6 +518,7 @@ public class BackupEngine
       /////////////////////////////////////////////////////////
       byte[] checksumBytes = (String.join("\n", checksums) + "\n").getBytes(StandardCharsets.UTF_8);
       putBytes(storage, runPrefix + "checksums.sha256", checksumBytes, result.getRunKey(), lockMode, retainUntil);
+      progress.info("Wrote checksums.sha256");
       catalog.insertArtifact(result.getRunId(), "CHECKSUMS", null, 1, "checksums.sha256", "TEXT", "NONE", false, checksums.size(), checksumBytes.length,
          checksumBytes.length, HashingOutputStream.sha256Hex(checksumBytes), HashingOutputStream.sha256Hex(checksumBytes), null, null, null, null);
 
@@ -463,6 +543,7 @@ public class BackupEngine
       {
          throw (new IllegalStateException("The manifest was written but is not visible in the bucket"));
       }
+      progress.info("Wrote manifest.json");
       catalog.insertArtifact(result.getRunId(), "MANIFEST", null, 1, "manifest.json", "JSON", "NONE", false, manifestTables.size(), manifestBytes.length,
          manifestBytes.length, HashingOutputStream.sha256Hex(manifestBytes), HashingOutputStream.sha256Hex(manifestBytes), null, null, null, null);
 
@@ -496,7 +577,59 @@ public class BackupEngine
       extra.put("bytesRaw", bytesRaw);
       extra.put("bytesStored", bytesStored);
       extra.put("warningCount", result.getWarnings().size());
-      finishWith(request, result, result.getWarnings().isEmpty() ? "SUCCEEDED" : "SUCCEEDED_WITH_WARNINGS", null, started, extra);
+      finishWith(request, result, result.getWarnings().isEmpty() ? "SUCCEEDED" : "SUCCEEDED_WITH_WARNINGS", null, null, started, extra);
+   }
+
+
+
+   /*******************************************************************************
+    ** Declare the work before scope resolution. No byte estimate requires a scan.
+    *******************************************************************************/
+   public static void prepare(JobProgress progress)
+   {
+      progress.phases(JobPhase.of("prepare", "Resolve scope", 2), JobPhase.of("tables", "Back up tables", 75),
+         JobPhase.of("media", "Copy media files", 18), JobPhase.of("upload", "Write catalog and manifest", 5));
+      progress.phase("prepare").message("Resolving scope");
+   }
+
+
+
+   /*******************************************************************************
+    ** Complete or fail the current phase; the core has no phase-finish method.
+    *******************************************************************************/
+   private static void completePhase(JobProgress progress, JobPhase.Status status)
+   {
+      progress.getSnapshot().getPhases().stream().filter(p -> p.getStatus() == JobPhase.Status.RUNNING).forEach(p -> p.setStatus(status));
+      progress.flush();
+   }
+
+
+
+   /*******************************************************************************
+    ** Log only a controlled failure description, never provider or row text.
+    *******************************************************************************/
+   private static void failProgress(JobProgress progress, String message)
+   {
+      completePhase(progress, JobPhase.Status.FAILED);
+      progress.error(message);
+   }
+
+
+
+   /*******************************************************************************
+    ** Bridge checked job interrupts through the row consumer, preserving the cause
+    ** so execute can propagate it back to the durable worker.
+    *******************************************************************************/
+   private static void checkJob()
+   {
+      try
+      {
+         JobChunkScope.checkBetweenPages();
+      }
+      catch(QException e)
+      {
+         throw (new IllegalStateException(e));
+      }
    }
 
 
@@ -563,6 +696,7 @@ public class BackupEngine
     *******************************************************************************/
    private void heartbeat(BackupRunRequest request, BackupRunResult result, Instant deadline, String tableName, long rowsSoFar)
    {
+      checkJob();
       Instant now = request.getClock().instant();
       if(request.getCancelRequested().getAsBoolean())
       {
@@ -579,6 +713,7 @@ public class BackupEngine
       catch(Exception e)
       {
          LOG.warn("Could not record a heartbeat", e);
+         request.getProgress().warn("Could not record a heartbeat");
       }
    }
 
@@ -587,7 +722,7 @@ public class BackupEngine
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void catalogProgress(BackupRunResult result, int done, long rows, long bytesRaw, long bytesStored)
+   private void catalogProgress(BackupRunRequest request, BackupRunResult result, int done, long rows, long bytesRaw, long bytesStored)
    {
       try
       {
@@ -601,6 +736,7 @@ public class BackupEngine
       catch(Exception e)
       {
          LOG.warn("Could not record progress", e);
+         request.getProgress().warn("Could not record progress");
       }
    }
 
@@ -609,17 +745,19 @@ public class BackupEngine
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void finish(BackupRunRequest request, BackupRunResult result, String status, String message, Instant started)
+   private void finish(BackupRunRequest request, BackupRunResult result, String status, String message, String detail, Instant started)
    {
       try
       {
-         finishWith(request, result, status, message, started, new LinkedHashMap<>());
+         finishWith(request, result, status, message, detail, started, new LinkedHashMap<>());
       }
       catch(Exception e)
       {
          LOG.warn("Could not finish the run record", e);
+         request.getProgress().warn("Could not finish the run record");
          result.setStatus(status);
          result.setErrorMessage(message);
+         result.setErrorDetail(detail);
       }
    }
 
@@ -628,7 +766,7 @@ public class BackupEngine
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void finishWith(BackupRunRequest request, BackupRunResult result, String status, String message, Instant started, Map<String, Serializable> extra) throws Exception
+   private void finishWith(BackupRunRequest request, BackupRunResult result, String status, String message, String detail, Instant started, Map<String, Serializable> extra) throws Exception
    {
       Instant end = request.getClock().instant();
       Map<String, Serializable> v = new LinkedHashMap<>(extra);
@@ -637,11 +775,14 @@ public class BackupEngine
       v.put("durationMs", end.toEpochMilli() - started.toEpochMilli());
       v.put("heartbeatDate", end);
       v.put("errorMessage", message);
+      v.put("errorDetail", detail);
       v.put("warningCount", result.getWarnings().size());
       catalog.update(BackupsTables.RUN, result.getRunId(), v);
       result.setStatus(status);
       result.setErrorMessage(message);
+      result.setErrorDetail(detail);
       signal(request, result, status, message);
+      BackupsMetrics.invalidate();
    }
 
 
@@ -694,13 +835,17 @@ public class BackupEngine
 
 
    /*******************************************************************************
-    ** A message safe to store: the exception type and message, never row values (the
-    ** engine never puts values in messages), truncated.
+    ** The error_message of a failed run: "Backup failed: SimpleClass: message".
+    ** Provider, driver and reader messages can contain credentials or row values,
+    ** so the message goes through FailureText.sanitize (URL user-info, presigned
+    ** query parameters, key=value secrets, Authorization values, AWS key ids, PEM
+    ** blocks and Postgres row-value fragments are redacted; object keys and bucket
+    ** names are kept).  The class, the cause chain and the root cause frames go to
+    ** error_detail (FailureText.detail); the full trace stays in the server log.
     *******************************************************************************/
    private static String scrub(Exception e)
    {
-      String message = e.getClass().getSimpleName() + ": " + e.getMessage();
-      return (message.length() > 2000 ? message.substring(0, 2000) : message);
+      return (FailureText.summary(e));
    }
 
 
@@ -769,6 +914,11 @@ public class BackupEngine
             {
                throw (new StopReadingException());
             }
+            checkJob();
+            if(request.getCancelRequested().getAsBoolean())
+            {
+               throw (new CancelledException());
+            }
             for(FileReferenceProvider provider : request.getFileProviders())
             {
                references.addAll(provider.referencesIn(table, record));
@@ -854,6 +1004,75 @@ public class BackupEngine
 
 
 
+   /***************************************************************************
+    ** Spool references locally between table export and media copy. Keeping the
+    ** whole run's references in heap would make memory grow with every table.
+    ** The temporary file is removed on success, failure and cancellation.
+    ***************************************************************************/
+   private static class PendingFiles implements AutoCloseable
+   {
+      private final Path path;
+      private final BufferedWriter writer;
+      private long count;
+
+
+
+      /*******************************************************************************
+       ** Create a private temporary spool, outside the stored backup artifacts.
+       *******************************************************************************/
+      PendingFiles() throws IOException
+      {
+         path = Files.createTempFile("qqq-backup-references-", ".jsonl");
+         writer = Files.newBufferedWriter(path, StandardCharsets.UTF_8);
+      }
+
+
+
+      /*******************************************************************************
+       ** Append this table's references without retaining earlier tables in memory.
+       *******************************************************************************/
+      void write(List<FileReference> references) throws IOException
+      {
+         for(FileReference reference : references)
+         {
+            writer.write(MAPPER.writeValueAsString(reference));
+            writer.newLine();
+            count++;
+         }
+      }
+
+
+
+      /*******************************************************************************
+       ** Finish the spool and stream it back during the media phase.
+       *******************************************************************************/
+      BufferedReader openReader() throws IOException
+      {
+         writer.close();
+         return (Files.newBufferedReader(path, StandardCharsets.UTF_8));
+      }
+
+
+
+      /*******************************************************************************
+       ** Always release the local references, including on exceptions.
+       *******************************************************************************/
+      @Override
+      public void close() throws IOException
+      {
+         try
+         {
+            writer.close();
+         }
+         finally
+         {
+            Files.deleteIfExists(path);
+         }
+      }
+   }
+
+
+
    /***************************************************************************
     ** The per-run file index, written as it goes: one JSON line per reference.
     ***************************************************************************/

src/main/java/com/kingsrook/qbits/backups/run/BackupRunCoordinator.java

@@ -30,6 +30,8 @@ import com.kingsrook.qbits.backups.BackupsRuntime;
 import com.kingsrook.qbits.backups.alerts.BackupAlertSink;
 import com.kingsrook.qbits.backups.model.BackupsTables;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.jobs.plans.JobChunkScope;
+import com.kingsrook.qqq.backend.core.jobs.progress.JobProgress;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@@ -85,7 +87,28 @@ public class BackupRunCoordinator
     *******************************************************************************/
    public BackupRunResult runPlan(QRecord plan, String trigger, java.util.function.Consumer<BackupRunRequest> customizer) throws QException
    {
-      BackupRunRequest request = BackupRunFactory.forPlan(plan, trigger);
+      return (runPlan(plan, trigger, JobProgress.current(), customizer));
+   }
+
+
+
+   /*******************************************************************************
+    ** Share the step reporter and reuse a durable job's run on an explicit retry.
+    ** NONE idempotency prevents automatic replay of non-atomic external writes.
+    *******************************************************************************/
+   public BackupRunResult runPlan(QRecord plan, String trigger, JobProgress progress, java.util.function.Consumer<BackupRunRequest> customizer) throws QException
+   {
+      BackupEngine.prepare(progress);
+      BackupRunRequest request;
+      try
+      {
+         request = BackupRunFactory.forPlan(plan, trigger).withProgress(progress);
+      }
+      catch(QException | RuntimeException e)
+      {
+         progress.error("Could not resolve backup scope (" + e.getClass().getSimpleName() + ")");
+         throw (new QException("Could not resolve backup scope", e));
+      }
       if(customizer != null)
       {
          customizer.accept(request);
@@ -97,21 +120,72 @@ public class BackupRunCoordinator
          BackupRunResult result = new BackupRunResult();
          result.setStatus("FAILED");
          result.setErrorMessage(refusal);
+         progress.error(refusal);
          return (result);
       }
 
-      Claim claim = claim(plan.getValueInteger("id"), null, trigger, request.getRequestedBy(), true);
+      JobChunkScope.ChunkIdentity identity = JobChunkScope.current().map(JobChunkScope::getIdentity).orElse(null);
+      Integer existingRunId = null;
+      if(identity != null)
+      {
+         String key = "job-" + identity.jobId() + "-plan-" + plan.getValueInteger("id");
+         request.withRunKey(key);
+         List<QRecord> previous = catalog.where(BackupsTables.RUN, "runKey", key);
+         if(!previous.isEmpty())
+         {
+            QRecord run = previous.get(0);
+            existingRunId = run.getValueInteger("id");
+            if(run.getValueString("manifestKey") != null)
+            {
+               progress.info("Run " + existingRunId + " already has a manifest; nothing was rewritten. Use Verify backup to recheck it.");
+               BackupRunResult result = new BackupRunResult();
+               result.setRunId(existingRunId);
+               result.setRunKey(key);
+               result.setRunPrefix(run.getValueString("runPrefix"));
+               result.setManifestKey(run.getValueString("manifestKey"));
+               result.setStatus(run.getValueString("status"));
+               return (result);
+            }
+            if(run.getValueString("runPrefix") != null)
+            {
+               // Only the same durable job can recover its interrupted RUNNING row.
+               if("RUNNING".equals(run.getValueString("status")))
+               {
+                  catalog.update(BackupsTables.RUN, existingRunId, Map.of("status", "ABANDONED"));
+                  run.setValue("status", "ABANDONED");
+               }
+               String why = BackupResumePlanner.whyNotResumable(run, plan);
+               if(why != null)
+               {
+                  progress.error(why);
+                  throw (new QException(why));
+               }
+               request.withResume(BackupResumePlanner.plan(run));
+            }
+         }
+      }
+      Claim claim = claim(plan.getValueInteger("id"), existingRunId, trigger, request.getRequestedBy(), identity == null, request.getRunKey());
       try
       {
          if(claim.outcome != Outcome.STARTED)
          {
+            progress.warn(claim.message);
+            if(identity != null)
+            {
+               if(claim.outcome == Outcome.QUEUED)
+               {
+                  catalog.update(BackupsTables.RUN, claim.runId, Map.of("status", "FAILED", "endDate", now(), "errorMessage", claim.message));
+               }
+               progress.error("Backup could not start: " + claim.message);
+               throw (new QException("Backup could not start: " + claim.message));
+            }
             return (claim.toResult());
          }
          return (execute(request, claim.runId));
       }
       finally
       {
-         if(claim.outcome == Outcome.STARTED)
+         if(claim.outcome == Outcome.STARTED && JobChunkScope.current().isEmpty())
          {
             drainQueue();
          }
@@ -133,7 +207,10 @@ public class BackupRunCoordinator
       }
       finally
       {
-         drainQueue();
+         if(JobChunkScope.current().isEmpty())
+         {
+            drainQueue();
+         }
       }
    }
 
@@ -144,6 +221,16 @@ public class BackupRunCoordinator
     ** message) when it cannot be resumed, or when a run of the plan is live.
     *******************************************************************************/
    public BackupRunResult resume(QRecord run) throws QException
+   {
+      return (resume(run, JobProgress.current()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Resume using the caller's reporter (including an inline async callback).
+    *******************************************************************************/
+   public BackupRunResult resume(QRecord run, JobProgress progress) throws QException
    {
       QRecord plan = run.getValueInteger("backupPlanId") == null ? null : catalog.get(BackupsTables.PLAN, run.getValueInteger("backupPlanId")).orElse(null);
       String  why  = BackupResumePlanner.whyNotResumable(run, plan);
@@ -156,7 +243,7 @@ public class BackupRunCoordinator
          return (refused);
       }
 
-      BackupRunRequest request = BackupRunFactory.forPlan(plan, "MANUAL");
+      BackupRunRequest request = BackupRunFactory.forPlan(plan, "MANUAL").withProgress(progress);
       Claim claim = claim(plan.getValueInteger("id"), run.getValueInteger("id"), run.getValueString("trigger"), request.getRequestedBy(), false);
       if(claim.outcome != Outcome.STARTED)
       {
@@ -173,7 +260,10 @@ public class BackupRunCoordinator
       }
       finally
       {
-         drainQueue();
+         if(JobChunkScope.current().isEmpty())
+         {
+            drainQueue();
+         }
       }
    }
 
@@ -238,7 +328,7 @@ public class BackupRunCoordinator
    /*******************************************************************************
     ** Run the engine on a claimed run row, then return its result.
     *******************************************************************************/
-   private BackupRunResult execute(BackupRunRequest request, Integer runId)
+   private BackupRunResult execute(BackupRunRequest request, Integer runId) throws QException
    {
       request.withExistingRunId(runId).withCancelRequested(() -> catalog.isCancelRequested(runId));
       return (new BackupEngine().execute(request));
@@ -250,6 +340,16 @@ public class BackupRunCoordinator
     ** The lease: insert (or reuse) the run row and decide.
     *******************************************************************************/
    private Claim claim(Integer planId, Integer existingRunId, String trigger, String requestedBy, boolean queueIfFull) throws QException
+   {
+      return (claim(planId, existingRunId, trigger, requestedBy, queueIfFull, null));
+   }
+
+
+
+   /*******************************************************************************
+    ** Persist the job identity in the initial claim, before any external writes.
+    *******************************************************************************/
+   private Claim claim(Integer planId, Integer existingRunId, String trigger, String requestedBy, boolean queueIfFull, String runKey) throws QException
    {
       synchronized(LEASE_LOCK)
       {
@@ -261,7 +361,7 @@ public class BackupRunCoordinator
          {
             Map<String, Serializable> values = new LinkedHashMap<>();
             values.put("backupPlanId", planId);
-            values.put("runKey", "queued-" + UUID.randomUUID().toString().substring(0, 8));
+            values.put("runKey", runKey != null ? runKey : "queued-" + UUID.randomUUID().toString().substring(0, 8));
             values.put("trigger", trigger);
             values.put("requestedByUserId", requestedBy);
             values.put("status", QUEUED);

src/main/java/com/kingsrook/qbits/backups/run/BackupRunRequest.java

@@ -26,6 +26,7 @@ import com.kingsrook.qbits.backups.files.FileReferenceProvider;
 import com.kingsrook.qbits.backups.scope.PlanSettings;
 import com.kingsrook.qbits.backups.scope.ScopeResolver;
 import com.kingsrook.qbits.backups.storage.BackupStorage;
+import com.kingsrook.qqq.backend.core.jobs.progress.JobProgress;
 
 
 /*******************************************************************************
@@ -54,6 +55,50 @@ public class BackupRunRequest
    private java.util.function.Function<String, BackupStorage> sourceResolver;
    private Integer                    existingRunId;
    private ResumeState                resume;
+   private JobProgress                progress = JobProgress.current();
+   private String                     runKey;
+
+
+
+   /*******************************************************************************
+    ** The reporter shared by the process, coordinator and engine.
+    *******************************************************************************/
+   public JobProgress getProgress()
+   {
+      return (progress);
+   }
+
+
+
+   /*******************************************************************************
+    ** Supply the process reporter, including its inline async callback.
+    *******************************************************************************/
+   public BackupRunRequest withProgress(JobProgress progress)
+   {
+      this.progress = progress;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** A stable identity for an explicitly retried durable job.
+    *******************************************************************************/
+   public String getRunKey()
+   {
+      return (runKey);
+   }
+
+
+
+   /*******************************************************************************
+    ** Set the durable job's backup identity before creating any artifacts.
+    *******************************************************************************/
+   public BackupRunRequest withRunKey(String runKey)
+   {
+      this.runKey = runKey;
+      return (this);
+   }
 
 
 

src/main/java/com/kingsrook/qbits/backups/run/BackupRunResult.java

@@ -43,6 +43,7 @@ public class BackupRunResult
    private int          filesMissing;
    private long         fileBytesCopied;
    private String       errorMessage;
+   private String       errorDetail;
    private List<String> warnings = new ArrayList<>();
 
 
@@ -397,6 +398,26 @@ public class BackupRunResult
 
 
 
+   /*******************************************************************************
+    ** Getter for errorDetail (sanitized class, message, cause chain and root frames)
+    *******************************************************************************/
+   public String getErrorDetail()
+   {
+      return (errorDetail);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for errorDetail
+    *******************************************************************************/
+   public void setErrorDetail(String errorDetail)
+   {
+      this.errorDetail = errorDetail;
+   }
+
+
+
    /*******************************************************************************
     ** Getter for warnings
     *******************************************************************************/

src/main/java/com/kingsrook/qbits/backups/run/FailureText.java

@@ -0,0 +1,184 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.run;
+
+
+import java.util.Collections;
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+
+/*******************************************************************************
+ ** Safe text for a failure: what is persisted on a run row (error_message and
+ ** error_detail) and shown in job output.
+ **
+ ** Exception messages from storage providers, drivers and readers can carry
+ ** credentials (URL user-info, presigned query strings, key=value secrets,
+ ** Authorization headers, AWS access key ids, PEM blocks) and row values (the
+ ** Postgres "Key (col)=(value)" and "Detail:" fragments).  sanitize() redacts
+ ** those and keeps the rest (class names, object keys, bucket names), which is
+ ** what an operator needs to act on a failure without reading the server log.
+ *******************************************************************************/
+public final class FailureText
+{
+   /** Redaction marker. */
+   public static final String REDACTED = "[REDACTED]";
+
+   /** The longest error_detail written. */
+   public static final int MAX_DETAIL_CHARS = 8192;
+
+   /** The longest exception message kept in error_message. */
+   public static final int MAX_SUMMARY_MESSAGE_CHARS = 500;
+
+   /** Stack frames of the root cause kept in error_detail. */
+   public static final int ROOT_CAUSE_FRAMES = 8;
+
+   private static final int MAX_CAUSES = 20;
+
+   private static final String SECRET_NAME = "[\\w.-]*(?:password|passwd|pwd|secret|token|credential|signature|api[_-]?key|access[_-]?key|private[_-]?key)[\\w.-]*";
+
+   private static final Pattern PEM_BLOCK     = Pattern.compile("-----BEGIN [A-Z0-9 ]+-----.*?(?:-----END [A-Z0-9 ]+-----|$)", Pattern.DOTALL);
+   private static final Pattern URL_USERINFO  = Pattern.compile("(?i)\\b([a-z][a-z0-9+.-]*://)[^/\\s@?#]+@");
+   private static final Pattern QUERY_PARAM   = Pattern.compile("(?i)([?&;](?:X-Amz-[\\w-]+|Signature|Credential|AWSAccessKeyId|Policy|sig|key|" + SECRET_NAME + ")=)[^&\\s\"'<>]*");
+   private static final Pattern PG_KEY_VALUE  = Pattern.compile("(?i)\\b(Key \\([^)]*\\)=\\().*?(\\))(?=\\s|$|[.,;:])");
+   private static final Pattern PG_DETAIL     = Pattern.compile("(?i)\\b(Detail:|Failing row contains)[^\\r\\n]*");
+   private static final Pattern AUTHORIZATION = Pattern.compile("(?i)\\b((?:Proxy-)?Authorization\"?\\s*[:=]\\s*)(\"[^\"]*\"|[^\\r\\n,;]+)");
+   private static final Pattern BEARER        = Pattern.compile("(?i)\\b(Bearer|Basic)\\s+[A-Za-z0-9._~+/=-]{6,}");
+   private static final Pattern SECRET_JSON   = Pattern.compile("(?i)(\"" + SECRET_NAME + "\"\\s*:\\s*)\"[^\"]*\"");
+   private static final Pattern SECRET_ASSIGN = Pattern.compile("(?i)(?<![\\w.-])((?:" + SECRET_NAME + "|key)\\s*=\\s*)(?!\\[REDACTED)(\"[^\"]*\"|'[^']*'|[^\\s&,;)\\]}]+)");
+   private static final Pattern AWS_KEY_ID    = Pattern.compile("\\b(?:AKIA|ASIA|AGPA|AIDA|AROA|ANPA|ANVA|AIPA)[A-Z0-9]{16}\\b");
+
+   private static final String R = Matcher.quoteReplacement(REDACTED);
+
+
+
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private FailureText()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Redact secrets and row values from a message.  Null stays null.
+    *******************************************************************************/
+   public static String sanitize(String text)
+   {
+      if(text == null)
+      {
+         return (null);
+      }
+      String s = PEM_BLOCK.matcher(text).replaceAll(Matcher.quoteReplacement("[REDACTED PEM]"));
+      s = URL_USERINFO.matcher(s).replaceAll("$1" + R + "@");
+      s = QUERY_PARAM.matcher(s).replaceAll("$1" + R);
+      s = PG_KEY_VALUE.matcher(s).replaceAll("$1" + R + "$2");
+      s = PG_DETAIL.matcher(s).replaceAll("$1 " + R);
+      s = AUTHORIZATION.matcher(s).replaceAll("$1" + R);
+      s = BEARER.matcher(s).replaceAll("$1 " + R);
+      s = SECRET_JSON.matcher(s).replaceAll("$1\"" + R + "\"");
+      s = SECRET_ASSIGN.matcher(s).replaceAll("$1" + R);
+      s = AWS_KEY_ID.matcher(s).replaceAll(R);
+      return (s);
+   }
+
+
+
+   /*******************************************************************************
+    ** The short error_message: "Backup failed: SimpleClass: sanitized message".
+    *******************************************************************************/
+   public static String summary(Throwable t)
+   {
+      return (summary("Backup failed", t));
+   }
+
+
+
+   /*******************************************************************************
+    ** A short message: "prefix: SimpleClass: sanitized message" (bounded).
+    *******************************************************************************/
+   public static String summary(String prefix, Throwable t)
+   {
+      String message = sanitize(t.getMessage());
+      if(message == null || message.isBlank())
+      {
+         return (prefix + ": " + t.getClass().getSimpleName());
+      }
+      message = message.strip();
+      if(message.length() > MAX_SUMMARY_MESSAGE_CHARS)
+      {
+         message = message.substring(0, MAX_SUMMARY_MESSAGE_CHARS) + "...";
+      }
+      return (prefix + ": " + t.getClass().getSimpleName() + ": " + message);
+   }
+
+
+
+   /*******************************************************************************
+    ** The error_detail: the exception class and sanitized message, each cause
+    ** ("Caused by: ..."), then the top stack frames of the root cause, bounded to
+    ** MAX_DETAIL_CHARS.  Frames are class, method, file and line only (no values).
+    *******************************************************************************/
+   public static String detail(Throwable t)
+   {
+      StringBuilder  out  = new StringBuilder(line(t));
+      Set<Throwable> seen = Collections.newSetFromMap(new IdentityHashMap<>());
+      seen.add(t);
+      Throwable root  = t;
+      Throwable cause = t.getCause();
+      while(cause != null && seen.size() < MAX_CAUSES && seen.add(cause))
+      {
+         out.append("\nCaused by: ").append(line(cause));
+         root = cause;
+         cause = cause.getCause();
+      }
+
+      List<StackTraceElement> frames = List.of(root.getStackTrace());
+      int                     shown  = Math.min(ROOT_CAUSE_FRAMES, frames.size());
+      if(shown > 0)
+      {
+         out.append("\nRoot cause stack (top ").append(shown).append(" of ").append(frames.size()).append(" frames):");
+         for(StackTraceElement frame : frames.subList(0, shown))
+         {
+            out.append("\n\tat ").append(frame);
+         }
+      }
+
+      if(out.length() > MAX_DETAIL_CHARS)
+      {
+         String marker = "\n... (truncated)";
+         out.setLength(MAX_DETAIL_CHARS - marker.length());
+         out.append(marker);
+      }
+      return (out.toString());
+   }
+
+
+
+   /*******************************************************************************
+    ** "fully.qualified.Class: sanitized message".
+    *******************************************************************************/
+   private static String line(Throwable t)
+   {
+      String message = sanitize(t.getMessage());
+      return (message == null ? t.getClass().getName() : t.getClass().getName() + ": " + message);
+   }
+}

src/main/java/com/kingsrook/qbits/backups/run/TableReader.java

@@ -18,30 +18,20 @@ package com.kingsrook.qbits.backups.run;
 
 
 import java.io.Serializable;
-import java.util.List;
 import java.util.function.Consumer;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
-import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.actions.tables.KeysetPager;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 
 
 /*******************************************************************************
- ** Reads the rows of one table in primary key order, in keyset pages (WHERE pk >
- ** lastKey ORDER BY pk LIMIT n), so the cost stays flat with depth and a read can
- ** resume from a key.  A read raw: hidden fields, passwords and heavy fields are
- ** returned as stored, and possible values are not translated.
- **
- ** This is an internal stand-in for the QQQ core KeysetPager and QueryInput
- ** withRawRead (core PR feature/keyset-pager-raw-read); the run engine only
- ** depends on this interface, so it is swapped when that ships in the next RC.
+ ** Reads rows in primary key order through core KeysetPager. Each bounded page
+ ** uses the supplied transaction and returns hidden, password and heavy fields
+ ** as stored, without possible-value translation or display generation.
  *******************************************************************************/
 public interface TableReader
 {
@@ -81,48 +71,25 @@ public interface TableReader
       @Override
       public void read(QTableMetaData table, String filterJson, Serializable startAfterKey, int pageSize, QBackendTransaction transaction, Consumer<QRecord> sink) throws QException
       {
-         String       pk      = table.getPrimaryKeyField();
-         Serializable lastKey = startAfterKey;
-         QQueryFilter user    = parseFilter(filterJson);
-
-         while(true)
+         KeysetPager pager = KeysetPager.forTable(table.getName())
+            .withFilter(parseFilter(filterJson))
+            .startingAfter(startAfterKey)
+            .withPageSize(pageSize)
+            .withTransaction(transaction)
+            .withQueryInputCustomizer(input ->
+            {
+               input.setShouldOmitHiddenFields(false);
+               input.setShouldMaskPasswords(false);
+               input.setShouldFetchHeavyFields(true);
+               input.setShouldTranslatePossibleValues(false);
+               input.setShouldGenerateDisplayValues(false);
+               input.withPermissionBypass("backups run");
+            });
+         if(!pager.isPrimaryKeyOrderable())
          {
-            QQueryFilter filter = new QQueryFilter();
-            if(lastKey != null)
-            {
-               filter.addCriteria(new QFilterCriteria(pk, QCriteriaOperator.GREATER_THAN, List.of(lastKey)));
-            }
-            if(user != null)
-            {
-               filter.addSubFilter(user);
-            }
-            filter.withOrderBy(new QFilterOrderBy(pk, true));
-            filter.setLimit(pageSize);
-
-            QueryInput queryInput = new QueryInput(table.getName());
-            queryInput.setFilter(filter);
-            queryInput.setShouldOmitHiddenFields(false);
-            queryInput.setShouldMaskPasswords(false);
-            queryInput.setShouldFetchHeavyFields(true);
-            queryInput.setShouldTranslatePossibleValues(false);
-            queryInput.setShouldGenerateDisplayValues(false);
-            if(transaction != null)
-            {
-               queryInput.setTransaction(transaction);
-            }
-            queryInput.withPermissionBypass("backups run");
-
-            List<QRecord> page = new QueryAction().execute(queryInput).getRecords();
-            for(QRecord record : page)
-            {
-               sink.accept(record);
-               lastKey = record.getValue(pk);
-            }
-            if(page.size() < pageSize)
-            {
-               return;
-            }
+            throw (new QException("Backup requires an orderable primary key on table " + table.getName()));
          }
+         pager.forEachPage(page -> page.forEach(sink));
       }
    }
 }

src/main/java/com/kingsrook/qbits/backups/schedule/BackupScheduledRunProcess.java

@@ -50,7 +50,7 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
  ** run is a full-instance, system-session backup (securityScope FULL, requested by
  ** system:scheduler), which is why the plan owner is validated first: a plan whose
  ** owner has gone, or lost backups.admin, is not run silently.  It raises
- ** backups.plan.owner_invalid, marks the plan OWNER_INVALID, and stops.
+ ** backups.plan.owner-invalid, marks the plan OWNER_INVALID, and stops.
  **
  ** Everything goes through the run coordinator, so a trigger that finds the plan
  ** already running is skipped visibly, and a full instance queues the run.
@@ -80,7 +80,7 @@ public class BackupScheduledRunProcess implements BackendStep
             .withInputData(new QFunctionInputMetaData().withRecordListMetaData(new QRecordListMetaData().withTableName(BackupsTables.PLAN)))
             .withCode(new QCodeReference(BackupScheduledRunProcess.class)));
       process.setPermissionRules(BackupsTables.permissionRules());
-      return (BackupsProcesses.permission(process, BackupsPermissions.RUN));
+      return (BackupsProcesses.permission(process.withSupplementalMetaData(BackupsProcesses.durability()), BackupsPermissions.RUN));
    }
 
 
@@ -141,17 +141,18 @@ public class BackupScheduledRunProcess implements BackendStep
       if(problem != null)
       {
          catalog.update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("slaStatus", "OWNER_INVALID"));
-         BackupsRuntime.getAlertSink().raise("backups.plan.owner_invalid", "backups.plan.owner_invalid:" + plan.getValueString("planKey"), BackupAlertSink.Severity.WARNING,
+         BackupsRuntime.getAlertSink().raise("backups.plan.owner-invalid", "backups.plan.owner-invalid:" + plan.getValueString("planKey"), BackupAlertSink.Severity.WARNING,
             "A scheduled backup did not run: " + problem, Map.of("plan", plan.getValueString("planKey")));
          throw (new QUserFacingException("The scheduled backup did not run: " + problem + "."));
       }
-      BackupsRuntime.getAlertSink().resolve("backups.plan.owner_invalid", "backups.plan.owner_invalid:" + plan.getValueString("planKey"));
+      BackupsRuntime.getAlertSink().resolve("backups.plan.owner-invalid", "backups.plan.owner-invalid:" + plan.getValueString("planKey"));
 
-      BackupRunResult result = new BackupRunCoordinator().runPlan(plan, "SCHEDULED");
+      BackupRunResult result = new BackupRunCoordinator().runPlan(plan, "SCHEDULED", com.kingsrook.qqq.backend.core.jobs.progress.JobProgress.of(input), null);
       output.addValue("status", result.getStatus());
       output.addValue("runId", result.getRunId());
       if(!"SUCCEEDED".equals(result.getStatus()) && !"SUCCEEDED_WITH_WARNINGS".equals(result.getStatus()))
       {
+         BackupsProcesses.requireSuccessfulJob(result);
          LOG.info("A scheduled backup did not succeed", logPair("plan", plan.getValueString("planKey")), logPair("status", result.getStatus()), logPair("message", result.getErrorMessage()));
       }
    }

src/main/java/com/kingsrook/qbits/backups/schedule/BackupScheduledVerifyProcess.java

@@ -0,0 +1,136 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.schedule;
+
+
+import java.util.List;
+import com.kingsrook.qbits.backups.model.BackupsTables;
+import com.kingsrook.qbits.backups.permissions.BackupsPermissions;
+import com.kingsrook.qbits.backups.processes.BackupsProcesses;
+import com.kingsrook.qbits.backups.run.BackupCatalog;
+import com.kingsrook.qbits.backups.verify.BackupVerifier;
+import com.kingsrook.qbits.backups.verify.VerificationOutcome;
+import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** The scheduled verify entry point: the process the verify scheduled job of a
+ ** plan (backups_plan.verifyCronExpression) runs.  Until this class existed the
+ ** verify job was created but pointed at a process nobody had registered, so a
+ ** verify cron never verified anything.
+ **
+ ** It verifies the plan's latest successful run, through the level the plan names
+ ** (verifyLevel, CONTENT when unset), stopping at the first level that does not
+ ** pass.  Findings, alerts and the plan's last-verified fields are written by the
+ ** verifier, exactly as for Verify backup on a run.  A plan that has no
+ ** successful run yet has nothing to verify, and says so.
+ *******************************************************************************/
+public class BackupScheduledVerifyProcess implements BackendStep
+{
+   private static final QLogger LOG = QLogger.getLogger(BackupScheduledVerifyProcess.class);
+
+   public static final String NAME = BackupPlanVerifySyncToScheduledJobProcess.VERIFY_PROCESS_NAME;
+
+   public static final String SCHEDULER_USER = "system:scheduler";
+
+
+
+   /*******************************************************************************
+    ** The process metadata: not in the nav, a record process on backups_plan.
+    *******************************************************************************/
+   public static QProcessMetaData processMetaData()
+   {
+      QProcessMetaData process = new QProcessMetaData()
+         .withName(NAME)
+         .withLabel("Run scheduled verification")
+         .withTableName(BackupsTables.PLAN)
+         .withIcon(new QIcon().withName("verified"))
+         .withStep(new QBackendStepMetaData()
+            .withName("execute")
+            .withInputData(new QFunctionInputMetaData().withRecordListMetaData(new QRecordListMetaData().withTableName(BackupsTables.PLAN)))
+            .withCode(new QCodeReference(BackupScheduledVerifyProcess.class)));
+      process.setPermissionRules(BackupsTables.permissionRules());
+      return (BackupsProcesses.permission(process, BackupsPermissions.RUN));
+   }
+
+
+
+   /*******************************************************************************
+    ** The level a plan verifies to: its verifyLevel, CONTENT when unset or unknown.
+    *******************************************************************************/
+   public static String levelOf(QRecord plan)
+   {
+      String level = plan.getValueString("verifyLevel");
+      return (level != null && BackupVerifier.LEVELS.contains(level) ? level : BackupVerifier.CONTENT);
+   }
+
+
+
+   /*******************************************************************************
+    ** Verify the latest successful run of the plan.
+    *******************************************************************************/
+   @Override
+   public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
+   {
+      List<QRecord> records = input.getRecords();
+      if(records == null || records.isEmpty())
+      {
+         throw (new QUserFacingException("No backup plan was selected or found."));
+      }
+      QRecord plan = new BackupCatalog().get(BackupsTables.PLAN, records.get(0).getValueInteger("id"))
+         .orElseThrow(() -> new QUserFacingException("The backup plan no longer exists."));
+
+      if(!Boolean.TRUE.equals(plan.getValueBoolean("isActive")))
+      {
+         LOG.info("A scheduled verification fired for a plan that is not active; nothing was verified", logPair("plan", plan.getValueString("planKey")));
+         output.addValue("status", "INACTIVE");
+         return;
+      }
+
+      Integer runId = plan.getValueInteger("lastSuccessRunId");
+      if(runId == null)
+      {
+         LOG.info("A scheduled verification fired for a plan with no successful backup yet; nothing was verified", logPair("plan", plan.getValueString("planKey")));
+         output.addValue("status", "NO_BACKUP");
+         return;
+      }
+
+      String                    level    = levelOf(plan);
+      List<VerificationOutcome> outcomes = new BackupVerifier().verifyThrough(runId, level, "SCHEDULED", SCHEDULER_USER);
+      VerificationOutcome       last     = outcomes.get(outcomes.size() - 1);
+      output.addValue("status", last.status());
+      output.addValue("runId", runId);
+      output.addValue("level", level);
+      if(!last.passed())
+      {
+         LOG.info("A scheduled verification did not pass", logPair("plan", plan.getValueString("planKey")), logPair("runId", runId), logPair("status", last.status()));
+      }
+   }
+}

src/main/java/com/kingsrook/qbits/backups/scope/ScopeResolver.java

@@ -48,6 +48,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
  *******************************************************************************/
 public class ScopeResolver
 {
+   /** Exclusion reason of a table whose backend has no query interface (storage-only): it cannot be read, even when selected. */
+   public static final String NOT_QUERYABLE = "NOT_QUERYABLE";
+
    private final QInstance qInstance;
 
 
@@ -205,6 +208,10 @@ public class ScopeResolver
          {
             excluded.add(new ExcludedTable(name, "BACKUPS"));
          }
+         else if(!TableRoles.isQueryable(qInstance, table))
+         {
+            excluded.add(new ExcludedTable(name, NOT_QUERYABLE));
+         }
          else if((role == TableRole.EPHEMERAL || role == TableRole.DERIVED || role == TableRole.EXTERNAL) && !explicit.contains(name) && !roles.contains(role))
          {
             excluded.add(new ExcludedTable(name, role.name()));

src/main/java/com/kingsrook/qbits/backups/scope/TableRole.java

@@ -35,6 +35,6 @@ public enum TableRole
    SECRET,
    /** The tables of this qbit: the catalog is rebuilt from manifests. */
    BACKUPS,
-   /** Tables backed by another system (API). Excluded by default; selectable. */
+   /** Tables backed by another system (API), or by a backend that cannot be queried (storage-only: never read, even when selected). Excluded by default. */
    EXTERNAL
 }

src/main/java/com/kingsrook/qbits/backups/scope/TableRoles.java

@@ -17,12 +17,20 @@
 package com.kingsrook.qbits.backups.scope;
 
 
+import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import com.kingsrook.qbits.backups.BackupsQBitConfig;
+import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
+import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -31,6 +39,14 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
  *******************************************************************************/
 public final class TableRoles
 {
+   private static final QLogger LOG = QLogger.getLogger(TableRoles.class);
+
+   ////////////////////////////////////////////////////////////////////////////////
+   // whether a backend type's module has a query interface: the answer depends  //
+   // only on the module class, so it is cached per backend type (roleOf is hot) //
+   ////////////////////////////////////////////////////////////////////////////////
+   private static final Map<String, Boolean> QUERYABLE_BY_BACKEND_TYPE = new ConcurrentHashMap<>();
+
    private static final Set<String> AUDIT_TABLES = Set.of("audit", "auditTable", "auditUser", "auditDetail");
 
    private static final String[] SECRET_NAME_HINTS = { "refreshToken", "RefreshToken", "oauthToken", "OAuthToken", "apiKey" };
@@ -76,6 +92,10 @@ public final class TableRoles
       {
          return (TableRole.SECRET);
       }
+      if(qInstance != null && !isQueryable(qInstance, table))
+      {
+         return (TableRole.EXTERNAL);
+      }
       if(AUDIT_TABLES.contains(name))
       {
          return (TableRole.AUDIT);
@@ -108,6 +128,56 @@ public final class TableRoles
 
 
 
+   /*******************************************************************************
+    ** Whether the rows of a table can be read: its backend module must have a query
+    ** interface.  A storage-only module (for example qbit-edi's preview download
+    ** backend) throws from getQueryInterface(), and reading its table would fail the
+    ** whole run, so such a table is never backed up.  A backend whose module cannot
+    ** be resolved cannot be read either (not cached: the module may register later).
+    *******************************************************************************/
+   public static boolean isQueryable(QInstance qInstance, QTableMetaData table)
+   {
+      if(qInstance == null || table == null || table.getBackendName() == null)
+      {
+         return (true);
+      }
+      QBackendMetaData backend = qInstance.getBackend(table.getBackendName());
+      if(backend == null || backend.getBackendType() == null)
+      {
+         return (true);
+      }
+      Boolean cached = QUERYABLE_BY_BACKEND_TYPE.get(backend.getBackendType());
+      if(cached != null)
+      {
+         return (cached);
+      }
+
+      QBackendModuleInterface module;
+      try
+      {
+         module = new QBackendModuleDispatcher().getQBackendModule(backend);
+      }
+      catch(QModuleDispatchException e)
+      {
+         LOG.debug("No backend module for a table, so it cannot be backed up", logPair("table", table.getName()), logPair("backendType", backend.getBackendType()));
+         return (false);
+      }
+
+      boolean queryable;
+      try
+      {
+         queryable = module.getQueryInterface() != null;
+      }
+      catch(IllegalStateException | UnsupportedOperationException e)
+      {
+         queryable = false;
+      }
+      QUERYABLE_BY_BACKEND_TYPE.put(backend.getBackendType(), queryable);
+      return (queryable);
+   }
+
+
+
    /*******************************************************************************
     ** A table that is hidden and has a password or sensitive field (other than the
     ** keys and dates) is treated as a table of secrets.

src/main/java/com/kingsrook/qbits/backups/sla/SlaChecker.java

@@ -35,6 +35,7 @@ import com.kingsrook.qbits.backups.model.BackupsTables;
 import com.kingsrook.qbits.backups.run.BackupCatalog;
 import com.kingsrook.qbits.backups.storage.BackupStorage;
 import com.kingsrook.qbits.backups.storage.StoredObjectInfo;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
@@ -95,8 +96,11 @@ public class SlaChecker
    private static final Set<String> FAILED_STATUSES   = Set.of("FAILED", "TIMED_OUT", "ABANDONED");
    private static final Set<String> FINISHED_STATUSES = Set.of("SUCCEEDED", "SUCCEEDED_WITH_WARNINGS", "FAILED", "TIMED_OUT", "ABANDONED");
 
-   private static final Map<String, Instant> MANIFEST_OK_AT = new ConcurrentHashMap<>();
-   private static final Map<String, Instant> LAST_AUDIT_AT  = new ConcurrentHashMap<>();
+   private static final Map<String, Instant> MANIFEST_OK_AT = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
+      "throttles this node's storage HEAD of a manifest to once per MANIFEST_CACHE; it caches storage state, not editable data, "
+         + "and a pruned run is skipped by its prunedDate, so each node is as fresh as a single node");
+   private static final Map<String, Instant> LAST_AUDIT_AT  = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
+      "throttles this node's bucket audit to once per BUCKET_AUDIT_INTERVAL; another node auditing again is only repeated work");
 
    private final BackupCatalog catalog = new BackupCatalog();
 

src/main/java/com/kingsrook/qbits/backups/verify/BackupVerifier.java

@@ -33,8 +33,8 @@ import java.util.Optional;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.kingsrook.qbits.backups.BackupsQBitConfig;
 import com.kingsrook.qbits.backups.BackupsRuntime;
-import com.kingsrook.qbits.backups.alerts.BackupAlerts;
 import com.kingsrook.qbits.backups.alerts.BackupAlertSink;
+import com.kingsrook.qbits.backups.alerts.BackupAlerts;
 import com.kingsrook.qbits.backups.codec.ArtifactTransform;
 import com.kingsrook.qbits.backups.codec.HashingOutputStream;
 import com.kingsrook.qbits.backups.codec.JsonlRowCodec;
@@ -47,6 +47,8 @@ import com.kingsrook.qbits.backups.storage.BackupStorageException;
 import com.kingsrook.qbits.backups.storage.StoredObjectInfo;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.jobs.plans.JobChunkScope;
+import com.kingsrook.qqq.backend.core.jobs.plans.JobInterruptedException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -153,6 +155,10 @@ public class BackupVerifier
          }
          status = findings.status();
       }
+      catch(JobInterruptedException e)
+      {
+         throw (e);
+      }
       catch(VerifyException | BackupStorageException | QException e)
       {
          status = "ERROR";
@@ -188,7 +194,7 @@ public class BackupVerifier
    /*******************************************************************************
     ** STORAGE level: head every object, including every file-store object.
     *******************************************************************************/
-   void storageLevel(RunBundle bundle, VerifyFindings findings)
+   void storageLevel(RunBundle bundle, VerifyFindings findings) throws QException
    {
       BackupManifest manifest = bundle.manifest();
       BackupStorage  storage  = bundle.storage();
@@ -199,6 +205,7 @@ public class BackupVerifier
 
       for(BackupManifest.TableEntry table : manifest.tables())
       {
+         JobChunkScope.checkBetweenPages();
          for(BackupManifest.Part part : table.parts() == null ? List.<BackupManifest.Part>of() : table.parts())
          {
             findings.objectsChecked++;
@@ -231,6 +238,7 @@ public class BackupVerifier
       int missingFiles = 0;
       for(RunBundle.FileIndexEntry entry : bundle.distinctStoredFiles(findings))
       {
+         JobChunkScope.checkBetweenPages();
          findings.objectsChecked++;
          findings.filesChecked++;
          Optional<StoredObjectInfo> head = storage.head(entry.casKey());
@@ -262,6 +270,7 @@ public class BackupVerifier
 
       for(BackupManifest.TableEntry entry : manifest.tables())
       {
+         JobChunkScope.checkBetweenPages();
          QTableMetaData table = qInstance.getTable(entry.name());
          JsonlRowCodec  codec = null;
          boolean        roundTrip = false;
@@ -274,7 +283,7 @@ public class BackupVerifier
             List<String> excluded = entry.excludedFields() == null ? List.of() : entry.excludedFields().stream().map(BackupManifest.ExcludedField::name).toList();
             codec = new JsonlRowCodec(table, excluded);
             String backendType = qInstance.getBackend(table.getBackendName()) == null ? null : qInstance.getBackend(table.getBackendName()).getBackendType();
-            roundTrip = TableFingerprints.of(table, backendType).equals(entry.fingerprint());
+            roundTrip = TableFingerprints.matches(entry, table, backendType);
             if(!roundTrip)
             {
                findings.warn("Table " + entry.name() + " has changed since this backup; rows were parsed but not checked for an exact round trip");
@@ -291,7 +300,7 @@ public class BackupVerifier
    /*******************************************************************************
     ** All parts of one table.
     *******************************************************************************/
-   private void verifyTable(RunBundle bundle, BackupManifest.TableEntry entry, JsonlRowCodec codec, boolean roundTrip, VerifyFindings findings)
+   private void verifyTable(RunBundle bundle, BackupManifest.TableEntry entry, JsonlRowCodec codec, boolean roundTrip, VerifyFindings findings) throws QException
    {
       MessageDigest rowsDigest = HashingOutputStream.newSha256();
       long          rows       = 0;
@@ -339,6 +348,11 @@ public class BackupVerifier
          }
          catch(QException e)
          {
+            JobInterruptedException interrupted = JobInterruptedException.find(e);
+            if(interrupted != null)
+            {
+               throw (interrupted);
+            }
             findings.mismatch("The part " + part.key() + " could not be read: " + e.getMessage());
          }
       }
@@ -451,6 +465,7 @@ public class BackupVerifier
 
       for(RunBundle.FileIndexEntry entry : bundle.distinctStoredFiles(findings))
       {
+         JobChunkScope.checkBetweenPages();
          QRecord fileObject = catalog.where(BackupsTables.FILE_OBJECT, "sha256", entry.sha256()).stream().filter(r -> r.getValue("deletedDate") == null).findFirst().orElse(null);
          if(fileObject != null && "PASSED".equals(fileObject.getValueString("lastVerifyStatus"))
             && fileObject.getValueInstant("lastVerifiedDate") != null && fileObject.getValueInstant("lastVerifiedDate").isAfter(cutoff))

src/main/java/com/kingsrook/qbits/backups/verify/PartScanner.java

@@ -29,6 +29,7 @@ import java.util.zip.GZIPInputStream;
 import com.kingsrook.qbits.backups.codec.HashingOutputStream;
 import com.kingsrook.qbits.backups.manifest.BackupManifest;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.jobs.plans.JobChunkScope;
 
 
 /*******************************************************************************
@@ -80,6 +81,7 @@ final class PartScanner
     *******************************************************************************/
    static Scan scan(RunBundle bundle, BackupManifest.Part part, LineConsumer consumer) throws IOException, QException
    {
+      JobChunkScope.checkBetweenPages();
       MessageDigest storedDigest = HashingOutputStream.newSha256();
       MessageDigest rawDigest    = HashingOutputStream.newSha256();
       long          lines        = 0;
@@ -102,6 +104,10 @@ final class PartScanner
                String         line;
                while((line = reader.readLine()) != null)
                {
+                  if(lines % 1000 == 0)
+                  {
+                     JobChunkScope.checkBetweenPages();
+                  }
                   lines++;
                   consumer.accept(line);
                }

src/main/resources/help/qbit-backups.yaml

@@ -0,0 +1,782 @@
+bundle: qbit-backups
+version: ${project.version}
+layer: QBIT
+entries:
+- key: table:backups_plan;field:name
+  summary: 'The display name of the backup plan.'
+- key: table:backups_plan;field:description
+  summary: 'Your notes about the purpose and coverage of this plan.'
+- key: table:backups_plan;field:planKey
+  summary: 'A stable short name used in the storage paths of this plan’s backups.'
+  content: |
+    Choose a stable lowercase name using letters, digits, dashes or underscores, within {{maxLength:backups_plan.planKey}} characters. This name becomes part of stored object paths. It cannot change after the plan has run. If you need a different storage identity, create another plan and keep the existing plan and its run history available for recovery.
+- key: table:backups_plan;field:isActive
+  summary: 'Whether this plan is active for scheduled execution.'
+- key: table:backups_plan;field:scopeType
+  summary: 'How this plan selects the data to back up.'
+  content: |
+    Scope and table roles control which data a plan selects. Use **Backup Plan Scope** rules to include or exclude particular targets and narrow records when needed. Review the resolved selection with **Run backup now** before continuing, especially after application metadata changes. Excluded data will not be available from this backup even if the run itself completes successfully.
+- key: table:backups_plan;field:includeRoles
+  summary: 'The comma-separated table roles included by the plan.'
+  content: |
+    Scope and table roles control which data a plan selects. Use **Backup Plan Scope** rules to include or exclude particular targets and narrow records when needed. Review the resolved selection with **Run backup now** before continuing, especially after application metadata changes. Excluded data will not be available from this backup even if the run itself completes successfully.
+- key: table:backups_plan;field:pruneBucketAlias
+  summary: 'The storage connection alias used when removing expired backups.'
+  content: |
+    This alias selects the configured storage access used for pruning. It can separate deletion access from the destination used to write backups. Enter a storage alias rather than connection credentials, and check it before requesting retention cleanup. **Prune per retention** previews proposed deletions; locked objects and retained backups remain subject to their protection rules.
+- key: table:backups_plan;field:prefix
+  summary: 'The storage path prefix under which this plan writes backups.'
+- key: table:backups_plan;field:critical
+  summary: 'Whether this plan is marked as critical for backup monitoring.'
+  content: |
+    Monitoring compares successful backup and verification history with the plan’s expected ages. Use the health result to decide which **Backup Run** or **Verification** to inspect. A recent run can still need verification, and a failed run can leave an older successful backup in place. Adjust expectations to your recovery needs rather than treating a changed threshold as a repair.
+- key: table:backups_plan;field:format
+  summary: 'The file format used for exported table records.'
+  content: |
+    The default is {{default:backups_plan.format}} and the metadata choices are {{values:backups_format}}. JSONL supports structured record content, while CSV provides a tabular representation; both can be requested together. Parquet is listed but is not implemented in this version and is rejected by validation. Review the backup’s artifacts afterward to see the files actually produced.
+- key: table:backups_plan;field:compression
+  summary: 'The compression applied to exported backup data.'
+  content: |
+    The default is {{default:backups_plan.compression}} and the metadata choices are {{values:backups_compression}}. Use GZIP to compress exported data or NONE to leave it uncompressed. ZSTD is listed but is not implemented in this version. Compare raw and stored byte counts on **Backup Run** or **Artifacts** when assessing the resulting storage size.
+- key: table:backups_plan;field:encryptionMode
+  summary: 'The encryption mode requested for backup content.'
+  content: |
+    Review these settings together before including sensitive data. Encryption support depends on the configured backup implementation; unsupported client encryption is rejected. A key reference names a key and must never contain its value. Encrypted-only inclusion requires client encryption, while including sensitive fields in the clear requires backup administrator permission. Check the preview and validation messages before running the plan.
+- key: table:backups_plan;field:encryptionKeyRef
+  summary: 'The name of the encryption key to resolve, never the key value.'
+  content: |
+    Review these settings together before including sensitive data. Encryption support depends on the configured backup implementation; unsupported client encryption is rejected. A key reference names a key and must never contain its value. Encrypted-only inclusion requires client encryption, while including sensitive fields in the clear requires backup administrator permission. Check the preview and validation messages before running the plan.
+- key: table:backups_plan;field:sensitiveFieldPolicy
+  summary: 'How fields marked sensitive are handled in this backup.'
+  content: |
+    Review these settings together before including sensitive data. Encryption support depends on the configured backup implementation; unsupported client encryption is rejected. A key reference names a key and must never contain its value. Encrypted-only inclusion requires client encryption, while including sensitive fields in the clear requires backup administrator permission. Check the preview and validation messages before running the plan.
+- key: table:backups_plan;field:heavyFieldPolicy
+  summary: 'How large field values are included or omitted in this backup.'
+  content: |
+    Choose how large field values should be represented in the backup. The default is {{default:backups_plan.heavyFieldPolicy}} and the configured choices are {{values:backups_heavy_policy}}. Including values inline, placing them in separate objects and excluding them have different recovery implications. Review the resulting artifacts and verify a backup before relying on it to recover records with large content.
+- key: table:backups_plan;field:fileObjectPolicy
+  summary: 'Whether referenced files are copied or only referenced in the backup.'
+  content: |
+    File handling is separate from exporting database records. Copying referenced files keeps their content with the backup; reference-only handling does not make an independent copy. The missing-file policy determines whether an unavailable source file produces a warning or fails the run. Check copied and missing file counts on **Backup Run**, and verify media recovery as well as record recovery.
+- key: table:backups_plan;field:fileMissingPolicy
+  summary: 'Whether a missing referenced file warns or fails the backup.'
+  content: |
+    File handling is separate from exporting database records. Copying referenced files keeps their content with the backup; reference-only handling does not make an independent copy. The missing-file policy determines whether an unavailable source file produces a warning or fails the run. Check copied and missing file counts on **Backup Run**, and verify media recovery as well as record recovery.
+- key: table:backups_plan;field:consistency
+  summary: 'The requested consistency of reads across the backed-up data.'
+  content: |
+    The default is {{default:backups_plan.consistency}} and the available requests are {{values:backups_consistency}}. A shared snapshot helps keep related records consistent while the application changes. Support depends on the source backend. After a run, inspect **{{label:backups_run.consistencyAchieved}}** rather than assuming the requested level was achieved. Review warnings before relying on the backup for recovery.
+- key: table:backups_plan;field:useReadReplica
+  summary: 'Read replica requests are unavailable and are rejected when you save the plan.'
+- key: table:backups_plan;field:incrementalMode
+  summary: 'Whether the plan requests a full backup or changed records only.'
+  content: |
+    The default is {{default:backups_plan.incrementalMode}}. This version supports full backups; requesting only records changed since the previous backup is rejected by validation. Keep the plan on the supported full mode and use scope rules when you need to limit selected data. Review **Backup Run** and its artifacts to confirm the resulting coverage.
+- key: table:backups_plan;field:nativeDump
+  summary: 'The native database dump mode requested in addition to logical data.'
+  content: |
+    The default is {{default:backups_plan.nativeDump}}. Native database dumps are not implemented in this version, so selecting a native dump mode is rejected. Use the supported logical backup formats to save table records and referenced files. Those backups have their own manifests, verification and restore workflow; do not assume they contain a database-engine dump.
+- key: table:backups_plan;field:cronTimeZoneId
+  summary: 'The time zone used to interpret this plan’s schedule.'
+- key: table:backups_plan;field:scheduledJobId
+  summary: 'An unused legacy scheduler reference; you do not need to set it.'
+- key: table:backups_plan;field:slaHours
+  summary: 'The maximum expected age of a successful backup, in hours.'
+  content: |
+    Monitoring compares successful backup and verification history with the plan’s expected ages. Use the health result to decide which **Backup Run** or **Verification** to inspect. A recent run can still need verification, and a failed run can leave an older successful backup in place. Adjust expectations to your recovery needs rather than treating a changed threshold as a repair.
+- key: table:backups_plan;field:verifySlaDays
+  summary: 'The maximum expected age of a verified backup, in days.'
+  content: |
+    Monitoring compares successful backup and verification history with the plan’s expected ages. Use the health result to decide which **Backup Run** or **Verification** to inspect. A recent run can still need verification, and a failed run can leave an older successful backup in place. Adjust expectations to your recovery needs rather than treating a changed threshold as a repair.
+- key: table:backups_plan;field:verifyFileSamplePercent
+  summary: 'The percentage of referenced files selected for verification.'
+  content: |
+    Verification has its own depth, file selection and schedule. Checking stored objects is different from comparing decoded content or testing a restore. Review **Verification** to see which level actually ran and what it checked. Scheduling depends on the host integration; a saved expression alone is not evidence of a completed check. Use **Verify backup** for a direct content check.
+- key: table:backups_plan;field:verifyCronExpression
+  summary: 'The schedule expression for automatic verification.'
+  content: |
+    Verification has its own depth, file selection and schedule. Checking stored objects is different from comparing decoded content or testing a restore. Review **Verification** to see which level actually ran and what it checked. Scheduling depends on the host integration; a saved expression alone is not evidence of a completed check. Use **Verify backup** for a direct content check.
+- key: table:backups_plan;field:verifyLevel
+  summary: 'The depth of checking requested for automatic verification.'
+  content: |
+    Verification has its own depth, file selection and schedule. Checking stored objects is different from comparing decoded content or testing a restore. Review **Verification** to see which level actually ran and what it checked. Scheduling depends on the host integration; a saved expression alone is not evidence of a completed check. Use **Verify backup** for a direct content check.
+- key: table:backups_plan;field:retainDaily
+  summary: 'The number of daily backup periods retained by pruning.'
+  content: |
+    Retention selects backups to keep across daily, weekly, monthly and yearly periods, with a minimum number protected from pruning. Changing these settings does not itself prove which objects will be deleted. Use **Prune per retention** and review its proposed deletions before continuing. Storage locks can prevent deletion even when a backup falls outside the requested retention periods.
+- key: table:backups_plan;field:retainWeekly
+  summary: 'The number of weekly backup periods retained by pruning.'
+  content: |
+    Retention selects backups to keep across daily, weekly, monthly and yearly periods, with a minimum number protected from pruning. Changing these settings does not itself prove which objects will be deleted. Use **Prune per retention** and review its proposed deletions before continuing. Storage locks can prevent deletion even when a backup falls outside the requested retention periods.
+- key: table:backups_plan;field:retainMonthly
+  summary: 'The number of monthly backup periods retained by pruning.'
+  content: |
+    Retention selects backups to keep across daily, weekly, monthly and yearly periods, with a minimum number protected from pruning. Changing these settings does not itself prove which objects will be deleted. Use **Prune per retention** and review its proposed deletions before continuing. Storage locks can prevent deletion even when a backup falls outside the requested retention periods.
+- key: table:backups_plan;field:retainYearly
+  summary: 'The number of yearly backup periods retained by pruning.'
+  content: |
+    Retention selects backups to keep across daily, weekly, monthly and yearly periods, with a minimum number protected from pruning. Changing these settings does not itself prove which objects will be deleted. Use **Prune per retention** and review its proposed deletions before continuing. Storage locks can prevent deletion even when a backup falls outside the requested retention periods.
+- key: table:backups_plan;field:retainMinimum
+  summary: 'The minimum number of successful backups protected from pruning.'
+  content: |
+    Retention selects backups to keep across daily, weekly, monthly and yearly periods, with a minimum number protected from pruning. Changing these settings does not itself prove which objects will be deleted. Use **Prune per retention** and review its proposed deletions before continuing. Storage locks can prevent deletion even when a backup falls outside the requested retention periods.
+- key: table:backups_plan;field:objectLockMode
+  summary: 'The storage retention-lock mode requested for backup objects.'
+  content: |
+    Object locking asks the storage service to protect backup objects from deletion for the specified period. The selected destination must support locking, and a non-disabled mode requires a duration. These protections can prevent pruning from removing objects. Use **Test destination** to inspect capability information and review the retention requirements before choosing a locked destination.
+- key: table:backups_plan;field:objectLockDays
+  summary: 'How many days backup objects should remain locked in storage.'
+  content: |
+    Object locking asks the storage service to protect backup objects from deletion for the specified period. The selected destination must support locking, and a non-disabled mode requires a duration. These protections can prevent pruning from removing objects. Use **Test destination** to inspect capability information and review the retention requirements before choosing a locked destination.
+- key: table:backups_plan;field:maxDurationMinutes
+  summary: 'The configured backup duration limit in minutes.'
+- key: table:backups_plan;field:pageSize
+  summary: 'The number of records requested per backup read page.'
+- key: table:backups_plan;field:targetPartBytes
+  summary: 'The target size in bytes for each exported data part.'
+- key: table:backups_plan;field:ownerUserId
+  summary: 'The user whose access is checked for this plan’s scheduled backups.'
+  content: |
+    Scheduled backups check that the plan owner can read the selected tables. An active plan can run with broad data access, so saving an active plan requires backup administrator permission. Keep the owner and scope aligned as access changes. If scheduling reports an invalid owner, correct the owner or permissions and inspect **Backup Run** for subsequent results.
+- key: table:backups_plan;field:lastRunId
+  summary: 'The latest backup run associated with this plan.'
+- key: table:backups_plan;field:lastSuccessRunId
+  summary: 'The latest successful backup run for this plan.'
+- key: table:backups_plan;field:lastSuccessDate
+  summary: 'When this plan last completed a successful backup.'
+- key: table:backups_plan;field:lastVerifiedRunId
+  summary: 'The most recently verified backup run for this plan.'
+- key: table:backups_plan;field:lastVerifiedDate
+  summary: 'When a backup for this plan was last verified.'
+- key: table:backups_plan;field:slaStatus
+  summary: 'The current assessment of this plan’s backup health and timeliness.'
+  content: |
+    Monitoring compares successful backup and verification history with the plan’s expected ages. Use the health result to decide which **Backup Run** or **Verification** to inspect. A recent run can still need verification, and a failed run can leave an older successful backup in place. Adjust expectations to your recovery needs rather than treating a changed threshold as a repair.
+- key: table:backups_plan_scope;field:backupPlanId
+  summary: 'The backup plan this scope rule belongs to.'
+- key: table:backups_plan_scope;field:action
+  summary: 'Whether matching data is included in or excluded from the plan.'
+- key: table:backups_plan_scope;field:selectorType
+  summary: 'Whether the selector identifies a table, application, component or table role.'
+  content: |
+    Use these settings to describe the data matched by this scope rule. The selector identifies a target; a structured filter can narrow its records, child inclusion can expand related records, and exclusions omit fields. Review the resolved result through **Run backup now** before continuing. A successful backup cannot recover records or fields that its scope left out.
+- key: table:backups_plan_scope;field:selectorValue
+  summary: 'The name matched by the chosen selector type.'
+  content: |
+    Use these settings to describe the data matched by this scope rule. The selector identifies a target; a structured filter can narrow its records, child inclusion can expand related records, and exclusions omit fields. Review the resolved result through **Run backup now** before continuing. A successful backup cannot recover records or fields that its scope left out.
+- key: table:backups_plan_scope;field:filterJson
+  summary: 'A structured record filter that narrows the selected data.'
+  content: |
+    Use these settings to describe the data matched by this scope rule. The selector identifies a target; a structured filter can narrow its records, child inclusion can expand related records, and exclusions omit fields. Review the resolved result through **Run backup now** before continuing. A successful backup cannot recover records or fields that its scope left out.
+- key: table:backups_plan_scope;field:includeAssociatedChildren
+  summary: 'Whether the selection also follows associated child records.'
+  content: |
+    Use these settings to describe the data matched by this scope rule. The selector identifies a target; a structured filter can narrow its records, child inclusion can expand related records, and exclusions omit fields. Review the resolved result through **Run backup now** before continuing. A successful backup cannot recover records or fields that its scope left out.
+- key: table:backups_plan_scope;field:fieldExclusions
+  summary: 'The fields to omit from data selected by this scope rule.'
+  content: |
+    Use these settings to describe the data matched by this scope rule. The selector identifies a target; a structured filter can narrow its records, child inclusion can expand related records, and exclusions omit fields. Review the resolved result through **Run backup now** before continuing. A successful backup cannot recover records or fields that its scope left out.
+- key: table:backups_run;field:backupPlanId
+  summary: 'The plan that produced this backup.'
+- key: table:backups_run;field:runKey
+  summary: 'The unique storage reference for this backup execution.'
+- key: table:backups_run;field:trigger
+  summary: 'What caused this backup to run.'
+- key: table:backups_run;field:requestedByUserId
+  summary: 'The user who requested this operation.'
+- key: table:backups_run;field:status
+  summary: 'The recorded progress or final state of this operation.'
+  content: |
+    The run states are {{values:backups_run_status}}. Check the state alongside completed table counts, warnings and errors. A run that succeeded with warnings needs review, especially if referenced files were missing. Completion and verification are separate: inspect **Verification** before relying on the saved content. A cancellation request can take time to reach a final state.
+- key: table:backups_run;field:startDate
+  summary: 'When this operation started.'
+- key: table:backups_run;field:endDate
+  summary: 'When this operation ended.'
+- key: table:backups_run;field:durationMs
+  summary: 'The elapsed backup time in milliseconds.'
+- key: table:backups_run;field:heartbeatDate
+  summary: 'When the running backup last reported activity.'
+- key: table:backups_run;field:tablesPlanned
+  summary: 'The number of tables selected for this backup.'
+- key: table:backups_run;field:tablesDone
+  summary: 'The number of tables completed by this backup.'
+- key: table:backups_run;field:rowsTotal
+  summary: 'The total number of records exported.'
+- key: table:backups_run;field:bytesRaw
+  summary: 'The data size in bytes before storage encoding.'
+- key: table:backups_run;field:bytesStored
+  summary: 'The data size in bytes as stored.'
+- key: table:backups_run;field:filesReferenced
+  summary: 'The number of referenced files encountered during the backup.'
+- key: table:backups_run;field:filesCopied
+  summary: 'The number of files copied into backup storage.'
+- key: table:backups_run;field:filesDeduplicated
+  summary: 'The number of files whose existing backup copies were reused.'
+- key: table:backups_run;field:filesMissing
+  summary: 'The number of referenced files that could not be found.'
+- key: table:backups_run;field:fileBytesCopied
+  summary: 'The total bytes of file content copied into backup storage.'
+- key: table:backups_run;field:fileBytesReferenced
+  summary: 'The total bytes of file content referenced by this backup.'
+- key: table:backups_run;field:consistencyAchieved
+  summary: 'The read consistency actually achieved by this run.'
+  content: |
+    These values describe the saved backup rather than the current plan. The manifest records content and integrity information, while consistency and security scope describe how records were read. Use them when comparing backup coverage or investigating **Verification** failures. A user-selected export can contain fewer records than a broader scheduled plan because the user’s record access still applies.
+- key: table:backups_run;field:snapshotId
+  summary: 'The database snapshot identifier recorded for this backup.'
+- key: table:backups_run;field:storageBucketAlias
+  summary: 'The configured storage alias holding this backup content.'
+- key: table:backups_run;field:runPrefix
+  summary: 'The storage path prefix for this run’s objects.'
+- key: table:backups_run;field:manifestKey
+  summary: 'The storage path of the manifest describing this backup.'
+  content: |
+    These values describe the saved backup rather than the current plan. The manifest records content and integrity information, while consistency and security scope describe how records were read. Use them when comparing backup coverage or investigating **Verification** failures. A user-selected export can contain fewer records than a broader scheduled plan because the user’s record access still applies.
+- key: table:backups_run;field:manifestSha256
+  summary: 'The checksum used to identify and check the backup manifest.'
+  content: |
+    These values describe the saved backup rather than the current plan. The manifest records content and integrity information, while consistency and security scope describe how records were read. Use them when comparing backup coverage or investigating **Verification** failures. A user-selected export can contain fewer records than a broader scheduled plan because the user’s record access still applies.
+- key: table:backups_run;field:qqqVersion
+  summary: 'The framework version recorded when this backup was made.'
+- key: table:backups_run;field:appVersion
+  summary: 'The application version recorded when this backup was made.'
+- key: table:backups_run;field:instanceName
+  summary: 'The application instance from which the backup was made.'
+- key: table:backups_run;field:schemaFingerprint
+  summary: 'A fingerprint of the data structure recorded in the backup.'
+  content: |
+    These values describe the saved backup rather than the current plan. The manifest records content and integrity information, while consistency and security scope describe how records were read. Use them when comparing backup coverage or investigating **Verification** failures. A user-selected export can contain fewer records than a broader scheduled plan because the user’s record access still applies.
+- key: table:backups_run;field:securityScope
+  summary: 'Whether the backup reflects user-limited access or a broader plan execution.'
+  content: |
+    These values describe the saved backup rather than the current plan. The manifest records content and integrity information, while consistency and security scope describe how records were read. Use them when comparing backup coverage or investigating **Verification** failures. A user-selected export can contain fewer records than a broader scheduled plan because the user’s record access still applies.
+- key: table:backups_run;field:encryptionKeyRef
+  summary: 'The key reference recorded for the backup, without the key value.'
+- key: table:backups_run;field:encryptionKeyFingerprint
+  summary: 'A fingerprint identifying the encryption key without exposing it.'
+- key: table:backups_run;field:retentionClass
+  summary: 'The retention period category assigned to this backup.'
+- key: table:backups_run;field:lockedUntil
+  summary: 'The time until which storage retention protects this content from deletion.'
+- key: table:backups_run;field:prunedDate
+  summary: 'When this backup was pruned from storage.'
+- key: table:backups_run;field:prunedByUserId
+  summary: 'The user recorded as pruning this backup.'
+- key: table:backups_run;field:verificationStatus
+  summary: 'The latest recorded level and outcome of verification for this backup.'
+  content: |
+    The recorded states are {{values:backups_verification_status}}. Storage checks, content checks and test restores provide different evidence; a storage-only result does not establish that every record can be restored. Open the related **Verification** for its level, counts and diagnostics. This status describes checking after backup creation and is separate from the backup run’s own completion status.
+- key: table:backups_run;field:warningCount
+  summary: 'The number of warnings recorded during the backup.'
+- key: table:backups_run;field:errorMessage
+  summary: 'The error message recorded for this operation.'
+- key: table:backups_run;field:errorDetail
+  summary: 'Additional error information recorded for the failed backup.'
+- key: table:backups_artifact;field:backupRunId
+  summary: 'The backup run this record belongs to.'
+- key: table:backups_artifact;field:artifactType
+  summary: 'The kind of backup content stored in this artifact.'
+- key: table:backups_artifact;field:tableName
+  summary: 'The source table described by this record.'
+- key: table:backups_artifact;field:partNumber
+  summary: 'The sequence number of this part within the exported table data.'
+- key: table:backups_artifact;field:objectKey
+  summary: 'The path identifying the object within its storage bucket.'
+- key: table:backups_artifact;field:format
+  summary: 'The format recorded for this artifact’s content.'
+- key: table:backups_artifact;field:compression
+  summary: 'The compression recorded for this artifact’s content.'
+- key: table:backups_artifact;field:encrypted
+  summary: 'Whether this stored object is recorded as encrypted.'
+- key: table:backups_artifact;field:rowCount
+  summary: 'The number of records written to this artifact.'
+- key: table:backups_artifact;field:bytesRaw
+  summary: 'The data size in bytes before storage encoding.'
+- key: table:backups_artifact;field:bytesStored
+  summary: 'The data size in bytes as stored.'
+- key: table:backups_artifact;field:sha256Stored
+  summary: 'The checksum of the bytes as stored in the bucket.'
+- key: table:backups_artifact;field:sha256Raw
+  summary: 'The checksum of the content before storage encoding.'
+- key: table:backups_artifact;field:rowsHash
+  summary: 'The checksum representing the exported records.'
+- key: table:backups_artifact;field:firstKey
+  summary: 'The first record key recorded for this data part.'
+- key: table:backups_artifact;field:lastKey
+  summary: 'The last record key recorded for this data part.'
+- key: table:backups_artifact;field:tableFingerprint
+  summary: 'A fingerprint of the table structure used by this artifact.'
+- key: table:backups_artifact;field:status
+  summary: 'The recorded progress or final state of this operation.'
+- key: table:backups_artifact;field:storageVersionId
+  summary: 'The storage service’s version identifier for the saved object.'
+- key: table:backups_verification;field:backupRunId
+  summary: 'The backup run this record belongs to.'
+- key: table:backups_verification;field:level
+  summary: 'The depth of verification performed on this backup.'
+  content: |
+    The levels are {{values:backups_verify_level}}. Storage verification checks saved objects, content verification reads and compares backup data, and a test restore exercises recovery in a temporary destination. Review the result and detail to see what actually passed. A deeper check can provide more recovery evidence, but the level alone does not mean the check succeeded.
+- key: table:backups_verification;field:trigger
+  summary: 'What caused this verification to run.'
+- key: table:backups_verification;field:requestedByUserId
+  summary: 'The user who requested this operation.'
+- key: table:backups_verification;field:startDate
+  summary: 'When this operation started.'
+- key: table:backups_verification;field:endDate
+  summary: 'When this operation ended.'
+- key: table:backups_verification;field:status
+  summary: 'The recorded progress or final state of this operation.'
+  content: |
+    The possible results are {{values:backups_verify_result}}. Review the level, mismatch count and details together with the result. A check can pass with warnings or fail before completing all planned work. Use the linked **Backup Run** and its artifacts to investigate; changing a plan’s settings does not repair the stored content of an earlier backup.
+- key: table:backups_verification;field:tablesChecked
+  summary: 'The number of tables examined during verification.'
+- key: table:backups_verification;field:rowsChecked
+  summary: 'The number of records examined during verification.'
+- key: table:backups_verification;field:mismatches
+  summary: 'The number of discrepancies found during verification.'
+- key: table:backups_verification;field:scratchTarget
+  summary: 'The temporary destination recorded for a test restore.'
+- key: table:backups_verification;field:detailJson
+  summary: 'The structured results and diagnostics from verification.'
+- key: table:backups_restore_run;field:sourceBackupRunId
+  summary: 'The backup run used as the source for recovery.'
+- key: table:backups_restore_run;field:requestedByUserId
+  summary: 'The user who requested this operation.'
+- key: table:backups_restore_run;field:startDate
+  summary: 'When this operation started.'
+- key: table:backups_restore_run;field:endDate
+  summary: 'When this operation ended.'
+- key: table:backups_restore_run;field:status
+  summary: 'The recorded progress or final state of this operation.'
+- key: table:backups_restore_run;field:mode
+  summary: 'How the restore handles new and existing destination records.'
+  content: |
+    Restore mode controls how destination records are handled, while matching and identifier strategy control how saved records connect to them. Review these choices before execution: replacement can delete existing records and requires explicit confirmation. This row records the choices made for that run. Inspect the table results and any pre-restore backup before responding to an unexpected recovery outcome.
+- key: table:backups_restore_run;field:idStrategy
+  summary: 'Whether original record identifiers are preserved or replaced during recovery.'
+  content: |
+    Restore mode controls how destination records are handled, while matching and identifier strategy control how saved records connect to them. Review these choices before execution: replacement can delete existing records and requires explicit confirmation. This row records the choices made for that run. Inspect the table results and any pre-restore backup before responding to an unexpected recovery outcome.
+- key: table:backups_restore_run;field:matchOn
+  summary: 'The key used to match saved records to destination records.'
+  content: |
+    Restore mode controls how destination records are handled, while matching and identifier strategy control how saved records connect to them. Review these choices before execution: replacement can delete existing records and requires explicit confirmation. This row records the choices made for that run. Inspect the table results and any pre-restore backup before responding to an unexpected recovery outcome.
+- key: table:backups_restore_run;field:targetInstanceName
+  summary: 'The application instance receiving the restored data.'
+- key: table:backups_restore_run;field:scopeSummary
+  summary: 'A description of the tables and records selected for recovery.'
+- key: table:backups_restore_run;field:preRestoreBackupRunId
+  summary: 'The backup taken of the destination before this restore.'
+- key: table:backups_restore_run;field:rowsInserted
+  summary: 'The number of records inserted by the restore.'
+- key: table:backups_restore_run;field:rowsUpdated
+  summary: 'The number of existing records updated by the restore.'
+- key: table:backups_restore_run;field:rowsUnchanged
+  summary: 'The number of matched records that needed no changes.'
+- key: table:backups_restore_run;field:rowsDeleted
+  summary: 'The number of records deleted by the restore.'
+- key: table:backups_restore_run;field:rowsSkipped
+  summary: 'The number of records skipped by the restore.'
+- key: table:backups_restore_run;field:rowsErrored
+  summary: 'The number of records that encountered restore errors.'
+- key: table:backups_restore_run;field:filesRestored
+  summary: 'The number of referenced files restored from backup storage.'
+- key: table:backups_restore_run;field:filesAlreadyPresent
+  summary: 'The number of files already present at the destination.'
+- key: table:backups_restore_run;field:filesConflicting
+  summary: 'The number of files that conflict with existing destination content.'
+- key: table:backups_restore_run;field:filesMissingFromBackup
+  summary: 'The number of required files absent from the backup.'
+- key: table:backups_restore_run;field:fileBytesRestored
+  summary: 'The total bytes of file content restored.'
+- key: table:backups_restore_run;field:sequencesResynced
+  summary: 'The number of automatic record-number sequences adjusted after recovery.'
+- key: table:backups_restore_run;field:automationsSuppressed
+  summary: 'Whether record automations were suppressed during this restore.'
+  content: |
+    These recorded flags explain whether restoring records was allowed to trigger automations or publish change events. Restore options normally suppress automations and do not publish events, but the selected options determine the actual run. Use these values when investigating missing or unexpected downstream activity, and inspect the **Restore Run** results before manually repeating any follow-up work.
+- key: table:backups_restore_run;field:esbSuppressed
+  summary: 'Whether change-event publication was suppressed during this restore.'
+  content: |
+    These recorded flags explain whether restoring records was allowed to trigger automations or publish change events. Restore options normally suppress automations and do not publish events, but the selected options determine the actual run. Use these values when investigating missing or unexpected downstream activity, and inspect the **Restore Run** results before manually repeating any follow-up work.
+- key: table:backups_restore_run;field:confirmationText
+  summary: 'The confirmation entered for a replacement restore.'
+  content: |
+    Restore mode controls how destination records are handled, while matching and identifier strategy control how saved records connect to them. Review these choices before execution: replacement can delete existing records and requires explicit confirmation. This row records the choices made for that run. Inspect the table results and any pre-restore backup before responding to an unexpected recovery outcome.
+- key: table:backups_restore_run;field:errorMessage
+  summary: 'The error message recorded for this operation.'
+- key: table:backups_restore_run_table;field:backupRestoreRunId
+  summary: 'The restore run this table result belongs to.'
+- key: table:backups_restore_run_table;field:tableName
+  summary: 'The source table described by this record.'
+- key: table:backups_restore_run_table;field:targetTableName
+  summary: 'The destination table that received the restored records.'
+- key: table:backups_restore_run_table;field:rowsInserted
+  summary: 'The number of records inserted by the restore.'
+- key: table:backups_restore_run_table;field:rowsUpdated
+  summary: 'The number of existing records updated by the restore.'
+- key: table:backups_restore_run_table;field:rowsUnchanged
+  summary: 'The number of matched records that needed no changes.'
+- key: table:backups_restore_run_table;field:rowsDeleted
+  summary: 'The number of records deleted by the restore.'
+- key: table:backups_restore_run_table;field:rowsSkipped
+  summary: 'The number of records skipped by the restore.'
+- key: table:backups_restore_run_table;field:rowsErrored
+  summary: 'The number of records that encountered restore errors.'
+- key: table:backups_restore_run_table;field:schemaStatus
+  summary: 'How the saved table structure compares with the destination structure.'
+  content: |
+    The comparison states are {{values:backups_schema_status}}. Differences can arise when the application’s fields change after a backup was taken. Review **{{label:backups_restore_run_table.unmappedFields}}** and the restore’s record counts before assuming all saved data was recovered. The recorded comparison concerns this table; inspect the parent **Restore Run** for the overall outcome and any errors.
+- key: table:backups_restore_run_table;field:unmappedFields
+  summary: 'The saved fields that could not be mapped to destination fields.'
+- key: table:backups_restore_run_table;field:sequenceResynced
+  summary: 'Whether automatic record numbering was adjusted for this table.'
+- key: table:backups_restore_run_table;field:status
+  summary: 'The recorded progress or final state of this operation.'
+- key: table:backups_file_object;field:sha256
+  summary: 'The checksum identifying this saved file’s content.'
+- key: table:backups_file_object;field:sizeBytes
+  summary: 'The file’s recorded size in bytes.'
+- key: table:backups_file_object;field:contentType
+  summary: 'The media type recorded for this file.'
+- key: table:backups_file_object;field:storageBucketAlias
+  summary: 'The configured storage alias holding this backup content.'
+- key: table:backups_file_object;field:objectKey
+  summary: 'The path identifying the object within its storage bucket.'
+- key: table:backups_file_object;field:encrypted
+  summary: 'Whether this stored object is recorded as encrypted.'
+- key: table:backups_file_object;field:encryptionKeyFingerprint
+  summary: 'A fingerprint identifying the encryption key without exposing it.'
+- key: table:backups_file_object;field:firstRunId
+  summary: 'The first backup run recorded as using this file.'
+- key: table:backups_file_object;field:lastReferencedRunId
+  summary: 'The latest backup run recorded as using this file.'
+- key: table:backups_file_object;field:referenceCount
+  summary: 'The number of backup references recorded for this file.'
+- key: table:backups_file_object;field:lastVerifiedDate
+  summary: 'When this file was last verified.'
+- key: table:backups_file_object;field:lastVerifyStatus
+  summary: 'The latest recorded verification outcome for this file.'
+- key: table:backups_file_object;field:lockedUntil
+  summary: 'The time until which storage retention protects this content from deletion.'
+- key: table:backups_file_object;field:deletedDate
+  summary: 'When this file’s deletion was recorded.'
+- key: table:backups_id_map;field:restoreRunId
+  summary: 'The restore run that assigned the new identifier.'
+- key: table:backups_id_map;field:tableName
+  summary: 'The table containing the remapped record.'
+- key: table:backups_id_map;field:oldId
+  summary: 'The record identifier in the source backup.'
+- key: table:backups_id_map;field:newId
+  summary: 'The record identifier assigned at the destination.'
+- key: process:backupRunPlan;slot:help
+  summary: 'Preview a backup plan’s scope and run it when you continue.'
+  content: |
+    Use **Run backup now** on a **Backup Plan** when you need a fresh copy before a change or want to check the plan. Review the resolved scope and any warnings first; nothing is written until you continue. The run writes its output to the configured destination and records progress in **Backup Run**. Inspect its result and verify the saved content before relying on it for recovery.
+- key: process:backupVerify;slot:help
+  summary: 'Check a backup’s stored content against its manifest.'
+  content: |
+    Use **Verify backup** on a **Backup Run** to check that its objects are present and their contents match the recorded backup. The content check reads stored bytes, decodes them and compares counts and checksums with the manifest. It records a **Verification** result without changing the backup. Review mismatches and warnings before using that run as a recovery source.
+- key: process:backupCheckSla;slot:help
+  summary: 'Check whether backup plans meet their freshness and verification expectations.'
+  content: |
+    This monitoring action checks backup health against each plan’s expected backup and verification ages. Use **Backup Plan** and the Backups home page to inspect its assessment, then open the latest **Backup Run** or **Verification** for the underlying evidence. A health problem may require a new run, a storage repair or investigation of a failed check rather than simply changing the monitoring threshold.
+- key: process:backupRestore;slot:help
+  summary: 'Preview and restore data from a saved backup run.'
+  content: |
+    Use **Restore...** on a **Backup Run** to recover its data. Choose the tables, matching rule, record handling and file options, then review the proposed changes. Nothing is written until you continue. Replacement mode can delete destination records and requires confirmation. Review the option to back up the target first. After execution, inspect **Restore Run** and its table results for skipped records, warnings or failures.
+- key: process:backupRestoreTable;slot:help
+  summary: 'Preview and restore table data from a selected backup artifact.'
+  content: |
+    Use the table restore action when you need data from a particular backup artifact rather than the entire run. Check the source table, destination, matching rule and restore mode in the options and review screen. Nothing is written until you continue. Replacement can remove destination data. Inspect **Restore Run** and its table results afterward to confirm which records and files were recovered.
+- key: process:backupImportCatalog;slot:help
+  summary: 'Find saved backups in storage and add their metadata to the local catalog.'
+  content: |
+    Use **Import backups from storage** when backup objects exist in storage but their run records are absent from this application. Enter the backup bucket alias and folder prefix, then review the discovered backups before continuing. This imports catalog information; it does not restore application records. Once imported, inspect **Backup Run**, verify the saved content and choose a restore action when needed.
+- key: process:backupDownload;slot:help
+  summary: 'Create download links for the stored artifacts of selected backup runs.'
+  content: |
+    Use **Download** when you need the files belonging to a backup run outside the application. Review the files offered before continuing to obtain their links. The action does not restore records or change the saved backup. Treat the downloaded content according to the data it contains, and use **Artifacts** or **Verification** when you need to inspect completeness and integrity.
+- key: process:backupDownloadArtifact;slot:help
+  summary: 'Create a download link for a selected backup artifact.'
+  content: |
+    Use the artifact download action when you need a particular stored part of a backup. Review the selected artifact and its related **Backup Run** so you know which table or supporting file you are retrieving. The action returns a download link and does not change application records. A single part may not contain everything needed for a complete restore.
+- key: process:backupRunScheduledPlan;slot:help
+  summary: 'Execute a backup plan on behalf of its configured schedule.'
+  content: |
+    This background action runs a plan when invoked by scheduling. It checks the plan owner before starting the backup and records the resulting **Backup Run**. Use the plan’s latest run and error information to investigate a missed or failed execution. For an interactive preview and manual start, use **Run backup now** on the plan instead.
+- key: process:backupCancelRun;slot:help
+  summary: 'Request cancellation of a backup run that is still in progress.'
+  content: |
+    Use **Cancel** on a running backup when it should stop. Review the selected runs before continuing. Cancellation is a request to the running process, so check **Backup Run** afterward for its final state. It does not turn partial output into a complete backup or undo a previous successful run. Use a successfully completed and verified backup for recovery.
+- key: process:backupCancelRestoreRun;slot:help
+  summary: 'Request cancellation of a restore that is still in progress.'
+  content: |
+    Use **Cancel** on a restore when further recovery work should stop. Review the selection and then check **Restore Run** for its final status and table results. Cancellation does not imply that records already changed have been rolled back. Inspect the counts and any target backup before deciding how to recover from a partially completed restore.
+- key: process:backupResumeRun;slot:help
+  summary: 'Preview where an interrupted backup can continue and resume eligible work.'
+  content: |
+    Use **Resume** on an interrupted backup when you want to continue eligible work from its recorded progress. Review the proposed continuation and any reasons the run cannot resume before proceeding. Nothing is written during the preview. After execution, inspect **Backup Run** and its artifacts, then verify the completed backup before treating it as a recovery source.
+- key: process:backupTestDestination;slot:help
+  summary: 'Check whether a plan’s configured backup storage can be used.'
+  content: |
+    Use **Test destination** after configuring a **Backup Plan** destination. After review, the test writes a small probe object, reads it back, lists it and deletes it. For a plan using object locking, it only reads and writes no probe. Review each check and the capability report. This tests destination access, not the integrity of an existing backup; use **Verify backup** for that.
+- key: process:backupPrune;slot:help
+  summary: 'Preview and delete backups that a plan’s retention rules no longer keep.'
+  content: |
+    Use **Prune per retention** on a **Backup Plan** to remove expired backup content. Review exactly what will be deleted before continuing; the preview deletes nothing. Retention periods, minimum retained backups and storage locks affect eligibility. Pruning removes recovery material, so review the retained backups and their verification results before proceeding. Inspect the run catalog afterward for the recorded pruning result.
+- key: process:backupPruneRuns;slot:help
+  summary: 'Apply retention pruning to the plans associated with selected backup runs.'
+  content: |
+    Use **Prune per retention** on selected runs to evaluate the retention rules of their plans. The action is not simply a command to delete each selected row. Review the proposed deletions before continuing. Minimum retention and storage locks can protect backups from deletion. Pruning removes saved recovery content, so inspect the backups that will remain and their verification results first.
+- key: widget:backupsHome;slot:label
+  summary: 'Backup plan health, recent runs and verification information.'
+- key: widget:backupsHealthCard;slot:label
+  summary: 'A summary of backup health and plans needing attention.'
+- key: widget:backupsPlanScopeJoinPlan;slot:label
+  summary: 'Inclusion and exclusion rules for this backup plan.'
+- key: widget:backupsRunJoinPlan;slot:label
+  summary: 'Backup runs produced by this plan.'
+- key: widget:backupsArtifactJoinRun;slot:label
+  summary: 'Stored artifacts belonging to this backup run.'
+- key: widget:backupsVerificationJoinRun;slot:label
+  summary: 'Verification checks performed on this backup run.'
+- key: widget:backupsRestoreRunJoinRun;slot:label
+  summary: 'Restore runs using this backup as their source.'
+- key: widget:backupsRestoreRunTableJoinRestoreRun;slot:label
+  summary: 'Per-table results for this restore run.'
+
+- key: app:backups;slot:help
+  summary: "Back up, verify and restore the application's data to S3-compatible storage."
+  content: |
+    Use the **Setup** card on the home to review what a working backup needs (a plan, a storage bucket, a schedule, verification) with the next step for each. A new installation has no plan: run Create default plan on Backup plans, which asks only for the storage bucket alias, or add a plan with New. Everything else (runs, artifacts, verifications, restores) is written by the backups themselves.
+
+- key: table:backups_plan;slot:help
+  summary: "A backup plan: what is backed up, where it goes, when it runs, how long it is kept and how it is verified."
+  content: |
+    **How rows get created:** add a plan with **New**, or run **Create default plan**, which adds "Nightly full" (the whole instance, data and audit tables, JSONL and gzip, every night at 02:00 UTC, verified on Sundays) and asks only for the storage bucket alias. The alias names a bucket your application provides to Backups; no key or endpoint is ever stored here. With the scheduler integration installed, an active plan with a cron expression runs on schedule; Run backup now runs it by hand. A few settings (PARQUET, ZSTD, incremental, native dump, read replica) are not available yet and are refused when the plan is saved.
+
+- key: table:backups_plan_scope;slot:help
+  summary: "One include or exclude rule of a plan: an app, a qbit, a table or a table role."
+  content: |
+    **How rows get created:** add **Scope rules** on a plan record. A plan whose scope is Instance backs up everything allowed by its roles and needs no rules; a plan limited to apps, qbits or tables is made of these rules, and an Exclude rule always wins over an Include.
+
+- key: table:backups_run;slot:help
+  summary: "One execution of a plan, or an ad-hoc export: its status, size, rows, files and storage location."
+  content: |
+    **How rows get created:** only the backups write it. Run backup now, a plan's schedule, a pre-restore safety backup, Resume and Import backups from storage each add a row. Nothing is added by hand; Verify backup, Download, Restore and Prune are actions on a run.
+
+- key: table:backups_artifact;slot:help
+  summary: "One stored object of a run: a table part, its manifest, checksums or a file index, with its hash and size."
+  content: |
+    **How rows get created:** only the backups write it, one row per object a run stores. Open **Artifacts** on a run record to use **Restore this table** or **Download**.
+
+- key: table:backups_verification;slot:help
+  summary: "The result of checking a backup: that it is in storage, that it decodes, or that it restores into scratch."
+  content: |
+    **How rows get created:** only verification writes it. Run Verify backup on a run (storage, content or test restore), or give a plan a verify cron expression so it verifies its latest backup on a schedule. Inspect the newest result on the plan and the home.
+
+- key: table:backups_restore_run;slot:help
+  summary: "One restore of a backup into the application: mode, id strategy, row and file counts and its safety backup."
+  content: |
+    **How rows get created:** only Restore writes it. Run Restore on a backup run, or Restore this table on an artifact: Review previews what would change before anything is written. A restore that replaces data takes a safety backup first and links it here.
+
+- key: table:backups_restore_run_table;slot:help
+  summary: "Per-table results of a restore: rows inserted, updated and skipped and how the schema matched."
+  content: |
+    **How rows get created:** only Restore writes it, one row per table restored. Open **Tables restored** on a restore record to inspect these results.
+
+- key: table:backups_file_object;slot:help
+  summary: "The content-addressed store of copied files, shared by runs, with reference counts and verification."
+  content: |
+    **How rows get created:** only backups write it. When a plan copies the files its rows reference, each distinct file (by SHA-256) is stored once and every run that uses it adds a reference; Prune lowers the counts and removes a file when no run needs it.
+
+- key: table:backups_id_map;slot:help
+  summary: "Old id to new id pairs recorded while restoring with new ids, so related rows can be rewritten."
+  content: |
+    **How rows get created:** only Restore writes it, and only when a restore uses the Remap new ids strategy. The rows are working data for that restore and are cleared when it finishes, so you will usually find this table empty.
+
+- key: table:backups_plan;field:storageBucketAlias
+  summary: The configured storage alias where you want this plan to write backups.
+  content: |
+    Enter the alias of a bucket your application provides to Backups, never credentials or an endpoint. Use **Test destination** to check access before running the plan. You can select a separate **Prune bucket alias** for deletion access.
+- key: table:backups_plan;field:cronExpression
+  summary: The schedule expression for running this plan automatically.
+  content: |
+    Set the schedule and **{{label:backups_plan.cronTimeZoneId}}** together. Your application must install the scheduler integration, and the plan must be active. Leave the expression blank for manual runs. Use **Run backup now** to preview and start a backup yourself.
+- key: process:backupsPlanCreateDefault;slot:help
+  summary: Create a starter backup plan for a storage bucket you choose.
+  content: |
+    Use **Create default plan** on **Backup plans** and enter a **Storage bucket alias** provided by your application. Review the proposed plan before continuing. The action adds **Nightly full** for data and audit tables with JSONL, gzip and copied files, scheduled at 02:00 UTC with content verification on Sundays at 04:00 UTC.
+
+    You can edit the settings afterward. The action creates a plan only when its plan key does not already exist; it does not change an existing plan. Scheduled execution requires your application's scheduler integration.
+- key: process:backupVerifyPlan;slot:help
+  summary: Verify the latest successful backup of a scheduled plan.
+  content: |
+    Use the plan's verification schedule and level to control automatic checks. This background action selects its latest successful backup and records the check in **Verification**. If no successful backup exists, there is nothing to verify. Inspect the plan and its latest verification result when investigating missed or failed checks.
+- key: process:*.backupExportTable;slot:help
+  summary: Preview and back up selected table records using your current access.
+  content: |
+    Use **Back up these records** to export records from a table. Review the selected scope before continuing; the preview writes nothing. Your record access applies to this export. The destination comes from the supplied storage alias or an active plan. Inspect the resulting **Backup Run** and verify it before relying on it for recovery.
+
+- key: possibleValueSource:backups_scope_type;slot:help
+  summary: The available scope type choices for backups.
+  content: |
+    You can see these choices: {{values:backups_scope_type}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_table_role;slot:help
+  summary: The available table role choices for backups.
+  content: |
+    You can see these choices: {{values:backups_table_role}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_scope_action;slot:help
+  summary: The available scope action choices for backups.
+  content: |
+    You can see these choices: {{values:backups_scope_action}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_selector_type;slot:help
+  summary: The available selector type choices for backups.
+  content: |
+    You can see these choices: {{values:backups_selector_type}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_format;slot:help
+  summary: The available format choices for backups.
+  content: |
+    You can see these choices: {{values:backups_format}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_compression;slot:help
+  summary: The available compression choices for backups.
+  content: |
+    You can see these choices: {{values:backups_compression}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_encryption_mode;slot:help
+  summary: The available encryption mode choices for backups.
+  content: |
+    You can see these choices: {{values:backups_encryption_mode}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_sensitive_policy;slot:help
+  summary: The available sensitive policy choices for backups.
+  content: |
+    You can see these choices: {{values:backups_sensitive_policy}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_heavy_policy;slot:help
+  summary: The available heavy policy choices for backups.
+  content: |
+    You can see these choices: {{values:backups_heavy_policy}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_file_policy;slot:help
+  summary: The available file policy choices for backups.
+  content: |
+    You can see these choices: {{values:backups_file_policy}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_file_missing_policy;slot:help
+  summary: The available file missing policy choices for backups.
+  content: |
+    You can see these choices: {{values:backups_file_missing_policy}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_consistency;slot:help
+  summary: The available consistency choices for backups.
+  content: |
+    You can see these choices: {{values:backups_consistency}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_consistency_achieved;slot:help
+  summary: The available consistency achieved choices for backups.
+  content: |
+    You can see these choices: {{values:backups_consistency_achieved}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_incremental_mode;slot:help
+  summary: The available incremental mode choices for backups.
+  content: |
+    You can see these choices: {{values:backups_incremental_mode}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_native_dump;slot:help
+  summary: The available native dump choices for backups.
+  content: |
+    You can see these choices: {{values:backups_native_dump}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_lock_mode;slot:help
+  summary: The available lock mode choices for backups.
+  content: |
+    You can see these choices: {{values:backups_lock_mode}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_sla_status;slot:help
+  summary: The available sla status choices for backups.
+  content: |
+    You can see these choices: {{values:backups_sla_status}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_run_trigger;slot:help
+  summary: The available run trigger choices for backups.
+  content: |
+    You can see these choices: {{values:backups_run_trigger}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_run_status;slot:help
+  summary: The available run status choices for backups.
+  content: |
+    You can see these choices: {{values:backups_run_status}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_retention_class;slot:help
+  summary: The available retention class choices for backups.
+  content: |
+    You can see these choices: {{values:backups_retention_class}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_verification_status;slot:help
+  summary: The available verification status choices for backups.
+  content: |
+    You can see these choices: {{values:backups_verification_status}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_artifact_type;slot:help
+  summary: The available artifact type choices for backups.
+  content: |
+    You can see these choices: {{values:backups_artifact_type}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_artifact_status;slot:help
+  summary: The available artifact status choices for backups.
+  content: |
+    You can see these choices: {{values:backups_artifact_status}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_verify_level;slot:help
+  summary: The available verify level choices for backups.
+  content: |
+    You can see these choices: {{values:backups_verify_level}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_verify_result;slot:help
+  summary: The available verify result choices for backups.
+  content: |
+    You can see these choices: {{values:backups_verify_result}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_restore_status;slot:help
+  summary: The available restore status choices for backups.
+  content: |
+    You can see these choices: {{values:backups_restore_status}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_restore_mode;slot:help
+  summary: The available restore mode choices for backups.
+  content: |
+    You can see these choices: {{values:backups_restore_mode}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_id_strategy;slot:help
+  summary: The available id strategy choices for backups.
+  content: |
+    You can see these choices: {{values:backups_id_strategy}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_schema_status;slot:help
+  summary: The available schema status choices for backups.
+  content: |
+    You can see these choices: {{values:backups_schema_status}}. Check the related field help for supported settings and how the choice affects your backup or restore.
+
+- key: possibleValueSource:backups_plan;slot:help
+  summary: The backup plans available for linking related records.
+
+- key: possibleValueSource:backups_run;slot:help
+  summary: The backup runs available for linking related records.
+
+- key: possibleValueSource:backups_restore_run;slot:help
+  summary: The restore runs available for linking related records.
+
+- key: process:BackupPlanSyncToScheduledJobProcess;slot:help
+  summary: Keep scheduled backup jobs aligned with your plan settings.
+  content: |
+    Set the plan's schedule, time zone and active status to control its scheduled backup job. This background integration synchronizes those settings when your application configures a scheduler. It does not start a backup immediately. Use **Run backup now** for a manual backup and inspect the plan's latest run to confirm execution.
+- key: process:BackupPlanVerifySyncToScheduledJobProcess;slot:help
+  summary: Keep scheduled verification jobs aligned with your plan settings.
+  content: |
+    Set the plan's verification schedule to request automatic checks when your application configures a scheduler. This background integration keeps the verification job aligned with the plan. Inspect **Verification** for completed checks; synchronizing a schedule does not verify a backup. Use **Verify backup** on a saved run for a manual check.

src/main/resources/qbit-backups.properties

@@ -0,0 +1 @@
+version=${project.version}

src/test/java/com/kingsrook/qbits/backups/BackupsClusterCachesTest.java

@@ -0,0 +1,164 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups;
+
+
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.function.Consumer;
+import com.kingsrook.qbits.backups.metrics.BackupsMetrics;
+import com.kingsrook.qbits.backups.model.BackupsTables;
+import com.kingsrook.qbits.backups.run.BackupCatalog;
+import com.kingsrook.qqq.backend.core.cluster.ClusterBus;
+import com.kingsrook.qqq.backend.core.cluster.ClusterBusInterface;
+import com.kingsrook.qqq.backend.core.cluster.ClusterBusMetaData;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCacheAudit;
+import com.kingsrook.qqq.backend.core.cluster.ClusterEvent;
+import com.kingsrook.qqq.backend.core.cluster.QNodeRoles;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** The Backups caches are correct on several nodes: every static cache is
+ ** registered with ClusterCaches, and the cached metrics are cleared on every
+ ** node when a plan changes.
+ *******************************************************************************/
+class BackupsClusterCachesTest
+{
+   /** the "wire": events every node's transport sees */
+   static final List<Consumer<ClusterEvent>> WIRE = new CopyOnWriteArrayList<>();
+
+
+
+   /*******************************************************************************
+    ** A transport over the static WIRE, so a "remote" publish reaches us.
+    *******************************************************************************/
+   public static class WireBus implements ClusterBusInterface
+   {
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public void publish(ClusterEvent event)
+      {
+         WIRE.forEach(listener -> listener.accept(event));
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public AutoCloseable subscribe(Consumer<ClusterEvent> listener)
+      {
+         WIRE.add(listener);
+         return (() -> WIRE.remove(listener));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** A backed-up test instance, connected to the wire as node-a.
+    *******************************************************************************/
+   @BeforeEach
+   void setUp() throws Exception
+   {
+      VerifyTestSupport.setUp(1);
+      QInstance instance = QContext.getQInstance();
+      instance.withSupplementalMetaData(new ClusterBusMetaData().withBus(new QCodeReference(WireBus.class)));
+      QNodeRoles.set(new QNodeRoles.Settings(EnumSet.allOf(QNodeRoles.Role.class), true, true, true, null, "node-a"));
+      ClusterBus.connect(instance);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @AfterEach
+   void afterEach()
+   {
+      ClusterBus.close(QContext.getQInstance());
+      WIRE.clear();
+      QNodeRoles.set(null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Every static Memoization in the QBit is registered (shared or node-local).
+    *******************************************************************************/
+   @Test
+   void testNoUnregisteredStaticCaches() throws Exception
+   {
+      assertThat(ClusterCacheAudit.unregisteredStaticCaches("com.kingsrook.qbits.backups")).isEmpty();
+   }
+
+
+
+   /*******************************************************************************
+    ** Another node's invalidation clears this node's cached metrics.
+    *******************************************************************************/
+   @Test
+   void testRemoteInvalidationClearsMetricsCache() throws Exception
+   {
+      String cached = BackupsMetrics.renderCached();
+      assertThat(BackupsMetrics.renderCached()).isSameAs(cached);
+
+      ClusterEvent remote = new ClusterEvent(ClusterEvent.TYPE_MEMO_INVALIDATE, BackupsMetrics.CACHE_NAME, null);
+      remote.setOriginNodeId("node-b");
+      WIRE.forEach(listener -> listener.accept(remote));
+
+      assertThat(BackupsMetrics.renderCached()).isNotSameAs(cached);
+   }
+
+
+
+   /*******************************************************************************
+    ** A plan edit here clears the metrics here and tells the other nodes.
+    *******************************************************************************/
+   @Test
+   void testPlanEditPublishesMetricsInvalidation() throws Exception
+   {
+      QRecord plan = VerifyTestSupport.plan(Map.of());
+      String  cached = BackupsMetrics.renderCached();
+
+      List<ClusterEvent> sent = new CopyOnWriteArrayList<>();
+      WIRE.add(sent::add);
+
+      new BackupCatalog().update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("slaHours", 48));
+
+      assertThat(BackupsMetrics.renderCached()).isNotSameAs(cached);
+      assertThat(sent).anySatisfy(event ->
+      {
+         assertThat(event.getType()).isEqualTo(ClusterEvent.TYPE_MEMO_INVALIDATE);
+         assertThat(event.getName()).isEqualTo(BackupsMetrics.CACHE_NAME);
+         assertThat(event.getOriginNodeId()).isEqualTo("node-a");
+      });
+   }
+}

src/test/java/com/kingsrook/qbits/backups/BackupsQBitProducerTest.java

@@ -18,6 +18,7 @@ package com.kingsrook.qbits.backups;
 
 
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundles;
 import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfigValidationException;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -40,12 +41,16 @@ class BackupsQBitProducerTest
       BackupsQBitConfig config    = new BackupsQBitConfig().withBackendName("rdbms");
       BackupsQBitProducer producer  = new BackupsQBitProducer().withQBitConfig(config).withNamespace("backups");
 
+      String artifactVersion = System.getProperty("qbit.artifactVersion");
+      assertThat(artifactVersion).isNotBlank();
+      assertThat(producer.getQBitMetaData().getVersion()).isEqualTo(artifactVersion).isEqualTo(BackupsQBitProducer.VERSION);
       assertThat(producer.isEnabled()).isTrue();
       assertThat(producer.getQBitConfig()).isSameAs(config);
       assertThat(producer.getNamespace()).isEqualTo("backups");
       assertThat(config.getDefaultBackendNameForTables()).isEqualTo("rdbms");
 
       producer.produce(qInstance);
+      assertThat(HelpBundles.of(qInstance).getBundles().get(0).getVersion()).isEqualTo(artifactVersion).isEqualTo(BackupsQBitProducer.VERSION);
       assertThat(qInstance.getQBits()).hasSize(1);
       assertThat(qInstance.getQBits().values().iterator().next().getArtifactId()).isEqualTo("qbit-backups");
    }
@@ -79,4 +84,26 @@ class BackupsQBitProducerTest
       assertThat(BackupsQBitConfig.PERMISSION_BASE_NAME).isEqualTo("backups");
    }
 
+
+
+   /*******************************************************************************
+    ** Long operations always enqueue, with no unsafe automatic external replay.
+    *******************************************************************************/
+   @Test
+   void testDurableProcesses() throws Exception
+   {
+      BackupsTestUtils.init();
+      for(String name : java.util.List.of("backupRunPlan", "backupRunScheduledPlan", "backupResumeRun", "backupRestore", "backupRestoreTable", "backupVerify"))
+      {
+         var process = com.kingsrook.qqq.backend.core.context.QContext.getQInstance().getProcess(name);
+         assertThat(process).as(name).isNotNull();
+         var durability = com.kingsrook.qqq.backend.core.jobs.DurableProcessMetaData.of(process);
+         assertThat(durability).as(name).isNotNull();
+         assertThat(durability.getAlwaysDurable()).isTrue();
+         assertThat(durability.getEligible()).isTrue();
+         assertThat(durability.getIdempotency()).isEqualTo(com.kingsrook.qqq.backend.core.jobs.model.JobIdempotency.NONE);
+         assertThat(durability.getChunkSize()).isEqualTo(1);
+      }
+   }
+
 }

src/test/java/com/kingsrook/qbits/backups/BackupsTestUtils.java

@@ -106,7 +106,7 @@ public final class BackupsTestUtils
       new BackupsQBitProducer().withQBitConfig(config).withNamespace("backups").produce(qInstance);
 
       new QInstanceValidator().validate(qInstance);
-      QContext.init(qInstance, new QSession());
+      QContext.init(qInstance, new QSession().withPermissions("backups.restore"));
       return (qInstance);
    }
 

src/test/java/com/kingsrook/qbits/backups/alerts/AlertsQBitAlertSinkTest.java

@@ -18,16 +18,24 @@ package com.kingsrook.qbits.backups.alerts;
 
 
 import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 import com.kingsrook.qbits.backups.BackupsRuntime;
 import com.kingsrook.qbits.backups.VerifyTestSupport;
-import com.kingsrook.qbits.backups.alerts.fakeapi.AlertsInstanceMetaData;
-import com.kingsrook.qbits.backups.alerts.fakeapi.QAlertTypeMetaData;
-import com.kingsrook.qbits.backups.alerts.fakeapi.QAlerts;
 import com.kingsrook.qbits.backups.sla.SlaChecker;
+import com.kingsrook.qqq.backend.core.alerts.AlertResolution;
+import com.kingsrook.qqq.backend.core.alerts.AlertSignal;
+import com.kingsrook.qqq.backend.core.alerts.AlertSinkInterface;
+import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData;
+import com.kingsrook.qqq.backend.core.alerts.QAlerts;
 import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -35,14 +43,10 @@ import static org.assertj.core.api.Assertions.assertThat;
 
 
 /*******************************************************************************
- ** The Alerts qbit adapter, bound reflectively.  The SPI is not in QQQ 4.1.0-RC.4,
- ** so these tests run it against a test double with the same shape as the real
- ** com.kingsrook.qqq.backend.core.alerts package (QRun-IO/qqq PR #1108).
+ ** The backup adapter against the released core alerts API and a registered sink.
  *******************************************************************************/
 class AlertsQBitAlertSinkTest
 {
-   private static final String FAKE_API = "com.kingsrook.qbits.backups.alerts.fakeapi";
-
 
 
    /*******************************************************************************
@@ -52,9 +56,11 @@ class AlertsQBitAlertSinkTest
    void setUp() throws Exception
    {
       VerifyTestSupport.setUp(3);
-      QAlerts.reset();
-      QAlertTypeMetaData.ADDED.clear();
-      AlertsInstanceMetaData.installed = true;
+      RecordingSink.raised.clear();
+      RecordingSink.resolved.clear();
+      RecordingSink.heartbeats.clear();
+      RecordingSink.failWith = null;
+      AlertsInstanceMetaData.ofOrWithNew(QContext.getQInstance()).withSink(new QCodeReference(RecordingSink.class));
    }
 
 
@@ -65,8 +71,8 @@ class AlertsQBitAlertSinkTest
    @AfterEach
    void tearDown()
    {
-      AlertsInstanceMetaData.installed = false;
-      QAlerts.reset();
+      BackupsRuntime.reset();
+      QContext.clear();
    }
 
 
@@ -78,7 +84,7 @@ class AlertsQBitAlertSinkTest
    @Test
    void testRaiseResolveHeartbeat()
    {
-      AlertsQBitAlertSink sink = new AlertsQBitAlertSink(FAKE_API);
+      AlertsQBitAlertSink sink = new AlertsQBitAlertSink();
       Map<String, String> detail = new LinkedHashMap<>();
       detail.put("plan", "nightly");
       detail.put("slaHours", "26");
@@ -87,60 +93,90 @@ class AlertsQBitAlertSinkTest
       sink.resolve("backups.plan.missed", "backups.plan.missed:nightly");
       sink.heartbeat("backups.plan:nightly", 93600, 0);
 
-      assertThat(QAlerts.RAISED).hasSize(1);
-      var signal = QAlerts.RAISED.get(0);
-      assertThat(signal.alertTypeName).isEqualTo("backups.plan.missed");
-      assertThat(signal.dedupKey).isEqualTo("backups.plan.missed:nightly");
-      assertThat(signal.severity.name()).isEqualTo("CRITICAL");
-      assertThat(signal.title).isEqualTo("A backup did not run in time");
-      assertThat(signal.summary).isEqualTo("plan=nightly, slaHours=26");
-      assertThat(signal.attributes).containsEntry("plan", "nightly").containsEntry("slaHours", "26");
-      assertThat(QAlerts.RESOLVED).containsExactly("backups.plan.missed|backups.plan.missed:nightly|Cleared by the Backups qbit");
-      assertThat(QAlerts.HEARTBEATS).containsExactly("backups.plan:nightly");
+      assertThat(RecordingSink.raised).hasSize(1);
+      var signal = RecordingSink.raised.get(0);
+      assertThat(signal.getAlertTypeName()).isEqualTo("backups.plan.missed");
+      assertThat(signal.getDedupKey()).isEqualTo("backups.plan.missed:nightly");
+      assertThat(signal.getSeverity().name()).isEqualTo("CRITICAL");
+      assertThat(signal.getTitle()).isEqualTo("A backup did not run in time");
+      assertThat(signal.getSummary()).isEqualTo("plan=nightly, slaHours=26");
+      assertThat(signal.getAttributes()).containsEntry("plan", "nightly").containsEntry("slaHours", "26");
+      assertThat(RecordingSink.resolved).singleElement().satisfies(resolution ->
+      {
+         assertThat(resolution.getAlertTypeName()).isEqualTo("backups.plan.missed");
+         assertThat(resolution.getDedupKey()).isEqualTo("backups.plan.missed:nightly");
+         assertThat(resolution.getNote()).isEqualTo("Cleared by the Backups qbit");
+      });
+      assertThat(RecordingSink.heartbeats).containsExactly("backups.plan:nightly");
 
       sink.raise("backups.storage.debris", "k", BackupAlertSink.Severity.INFO, "no detail", null);
-      assertThat(QAlerts.RAISED.get(1).summary).isNull();
+      assertThat(RecordingSink.raised.get(1).getSummary()).isNull();
    }
 
 
 
    /*******************************************************************************
-    ** Without the Alerts qbit installed (no alerts meta-data on the instance) every
+    ** Without a configured sink, even with alert type metadata present, every
     ** call is a no-op.
     *******************************************************************************/
    @Test
    void testNoOpWithoutTheAlertsQBit()
    {
-      AlertsInstanceMetaData.installed = false;
-      AlertsQBitAlertSink sink = new AlertsQBitAlertSink(FAKE_API);
-      sink.raise("t", "k", BackupAlertSink.Severity.INFO, "x", Map.of());
-      sink.resolve("t", "k");
-      sink.heartbeat("m", 1, 1);
-      assertThat(QAlerts.RAISED).isEmpty();
-      assertThat(QAlerts.RESOLVED).isEmpty();
-      assertThat(QAlerts.HEARTBEATS).isEmpty();
+      AlertsInstanceMetaData.of(QContext.getQInstance()).withSink(null);
+      AlertsQBitAlertSink sink = new AlertsQBitAlertSink();
+      long failures = QAlerts.getProducerFailureCount();
+      sink.raise(null, "k", BackupAlertSink.Severity.INFO, "x", Map.of());
+      sink.resolve(null, "k");
+      sink.heartbeat(null, 1, 1);
+      assertThat(QAlerts.getProducerFailureCount()).isEqualTo(failures);
+      assertThat(RecordingSink.raised).isEmpty();
+      assertThat(RecordingSink.resolved).isEmpty();
+      assertThat(RecordingSink.heartbeats).isEmpty();
    }
 
 
 
    /*******************************************************************************
-    ** A broken or missing SPI never throws into the backup.
+    ** The same adapter follows the current instance and a sink installed later.
+    *******************************************************************************/
+   @Test
+   void testLateInstallationAndInstanceSwitch()
+   {
+      var original = QContext.getQInstance();
+      QContext.setQInstance(new QInstance());
+      AlertsQBitAlertSink sink = new AlertsQBitAlertSink();
+      assertThat(sink.isAlertsQBitInstalled()).isFalse();
+      sink.raise("backups.plan.missed", "k", BackupAlertSink.Severity.INFO, "x", Map.of());
+      assertThat(RecordingSink.raised).isEmpty();
+      sink.registerAlertTypes(QContext.getQInstance());
+      assertThat(sink.isAlertsQBitInstalled()).isFalse();
+      AlertsInstanceMetaData.of(QContext.getQInstance()).withSink(new QCodeReference(RecordingSink.class));
+      sink.raise("backups.plan.missed", "k", BackupAlertSink.Severity.INFO, "x", Map.of());
+      assertThat(RecordingSink.raised).hasSize(1);
+      assertThat(RecordingSink.raised.get(0).getSummary()).isNull();
+      QContext.setQInstance(original);
+      assertThat(sink.isAlertsQBitInstalled()).isTrue();
+   }
+
+
+
+   /*******************************************************************************
+    ** Sink and adapter failures never throw into the backup.
     *******************************************************************************/
    @Test
    void testFailuresAreSwallowed()
    {
-      AlertsQBitAlertSink missing = new AlertsQBitAlertSink("no.such.package");
-      missing.raise("t", "k", BackupAlertSink.Severity.INFO, "x", Map.of());
-      missing.resolve("t", "k");
-      missing.heartbeat("m", 1, 1);
-      assertThat(missing.registerAlertTypes(QContext.getQInstance())).isZero();
-
-      QAlerts.failWith = new IllegalStateException("alerts down");
-      AlertsQBitAlertSink broken = new AlertsQBitAlertSink(FAKE_API);
+      long failures = QAlerts.getProducerFailureCount();
+      RecordingSink.failWith = new IllegalStateException("alerts down");
+      AlertsQBitAlertSink broken = new AlertsQBitAlertSink();
       broken.raise("t", "k", BackupAlertSink.Severity.INFO, "x", Map.of());
       broken.resolve("t", "k");
       broken.heartbeat("m", 1, 1);
-      assertThat(QAlerts.RAISED).isEmpty();
+      assertThat(RecordingSink.raised).isEmpty();
+      assertThat(QAlerts.getProducerFailureCount()).isEqualTo(failures + 3);
+      broken.raise("t", "k", null, "bad severity", Map.of());
+      broken.raise("t", "k", null, "bad severity again", Map.of());
+      assertThat(broken.registerAlertTypes(null)).isZero();
    }
 
 
@@ -149,41 +185,41 @@ class AlertsQBitAlertSinkTest
     ** The backup alert types are declared with the Alerts qbit.
     *******************************************************************************/
    @Test
-   void testRegisterAlertTypes()
+   void testRegisterAlertTypes() throws Exception
    {
-      int added = new AlertsQBitAlertSink(FAKE_API).registerAlertTypes(QContext.getQInstance());
+      QInstance instance = new QInstance();
+      AlertsQBitAlertSink sink = new AlertsQBitAlertSink();
+      int added = sink.registerAlertTypes(instance);
+      var types = new ArrayList<>(AlertsInstanceMetaData.of(instance).getAlertTypes().values());
       assertThat(added).isEqualTo(BackupAlerts.alertTypes().size()).isEqualTo(9);
-      assertThat(QAlertTypeMetaData.ADDED).extracting(t -> t.name).containsExactly(
-         "backups.plan.missed", "backups.run.failed", "backups.run.missing_in_storage", "backups.files.missing", "backups.verify.failed",
-         "backups.verify.overdue", "backups.plan.owner_invalid", "backups.storage.debris", "backups.restore.completed");
-      assertThat(QAlertTypeMetaData.ADDED).allSatisfy(t ->
+      assertThat(types).extracting(t -> t.getName()).containsExactly(
+         "backups.plan.missed", "backups.run.failed", "backups.run.missing-in-storage", "backups.files.missing", "backups.verify.failed",
+         "backups.verify.overdue", "backups.plan.owner-invalid", "backups.storage.debris", "backups.restore.completed");
+      assertThat(types).allSatisfy(t ->
       {
-         assertThat(t.groupName).isEqualTo("Backups");
-         assertThat(t.howToFix).isNotBlank();
-         assertThat(t.label).isNotBlank();
+         assertThat(t.getGroupName()).isEqualTo("Backups");
+         assertThat(t.getHowToFix()).isNotBlank();
+         assertThat(t.getLabel()).isNotBlank();
       });
-      assertThat(QAlertTypeMetaData.ADDED.get(2).defaultSeverity.name()).isEqualTo("CRITICAL");
+      assertThat(types.get(2).getDefaultSeverity().name()).isEqualTo("CRITICAL");
+      assertThat(sink.registerAlertTypes(instance)).isZero();
+      new QInstanceValidator().validate(QContext.getQInstance());
    }
 
 
 
    /*******************************************************************************
-    ** Detection: the SPI is present or not by class name; the real SPI is not in
-    ** RC.4, so nothing is bound by default there.
+    ** Default binding preserves a host-provided backup sink.
     *******************************************************************************/
    @Test
    void testDetectionAndBinding()
    {
-      assertThat(AlertsQBitAlertSink.isPresent(FAKE_API)).isTrue();
-      assertThat(AlertsQBitAlertSink.isPresent("no.such.package")).isFalse();
-
-      boolean realSpiPresent = AlertsQBitAlertSink.isPresent(AlertsQBitAlertSink.DEFAULT_API_PACKAGE);
-      assertThat(AlertsQBitAlertSink.bindIfPresent()).isEqualTo(realSpiPresent);
-      if(realSpiPresent)
-      {
-         assertThat(BackupsRuntime.getAlertSink()).isInstanceOf(AlertsQBitAlertSink.class);
-         assertThat(AlertsQBitAlertSink.bindIfPresent()).as("an already bound sink is not replaced").isFalse();
-      }
+      assertThat(AlertsQBitAlertSink.bindIfAbsent()).isFalse();
+      assertThat(BackupsRuntime.getAlertSink()).isSameAs(VerifyTestSupport.sink);
+      BackupsRuntime.setAlertSink(null);
+      assertThat(AlertsQBitAlertSink.bindIfAbsent()).isTrue();
+      assertThat(BackupsRuntime.getAlertSink()).isInstanceOf(AlertsQBitAlertSink.class);
+      assertThat(AlertsQBitAlertSink.bindIfAbsent()).isFalse();
    }
 
 
@@ -194,22 +230,80 @@ class AlertsQBitAlertSinkTest
    @Test
    void testSlaAlertsThroughTheAdapter() throws Exception
    {
-      BackupsRuntime.setAlertSink(new AlertsQBitAlertSink(FAKE_API));
+      BackupsRuntime.setAlertSink(new AlertsQBitAlertSink());
       QRecord plan = VerifyTestSupport.plan(Map.of());
       VerifyTestSupport.backup(plan);
-      assertThat(QAlerts.HEARTBEATS).contains("backups.plan:nightly");
+      assertThat(RecordingSink.heartbeats).contains("backups.plan:nightly");
 
       VerifyTestSupport.clock.advance(Duration.ofHours(30));
       new SlaChecker().check(VerifyTestSupport.reload(plan));
-      assertThat(QAlerts.RAISED).anySatisfy(s ->
+      assertThat(RecordingSink.raised).anySatisfy(s ->
       {
-         assertThat(s.alertTypeName).isEqualTo("backups.plan.missed");
-         assertThat(s.dedupKey).isEqualTo("backups.plan.missed:nightly");
-         assertThat(s.severity.name()).isEqualTo("ERROR");
+         assertThat(s.getAlertTypeName()).isEqualTo("backups.plan.missed");
+         assertThat(s.getDedupKey()).isEqualTo("backups.plan.missed:nightly");
+         assertThat(s.getSeverity().name()).isEqualTo("ERROR");
       });
 
       VerifyTestSupport.backup(plan);
       new SlaChecker().check(VerifyTestSupport.reload(plan));
-      assertThat(QAlerts.RESOLVED).anyMatch(r -> r.startsWith("backups.plan.missed|backups.plan.missed:nightly|"));
+      assertThat(RecordingSink.resolved).anyMatch(r -> r.getAlertTypeName().equals("backups.plan.missed") && r.getDedupKey().equals("backups.plan.missed:nightly"));
+   }
+
+
+
+   /*******************************************************************************
+    ** Sink loaded by the real QCodeLoader through AlertsInstanceMetaData.
+    *******************************************************************************/
+   public static class RecordingSink implements AlertSinkInterface
+   {
+      static final List<AlertSignal> raised = new ArrayList<>();
+      static final List<AlertResolution> resolved = new ArrayList<>();
+      static final List<String> heartbeats = new ArrayList<>();
+      static RuntimeException failWith;
+
+
+
+      /*******************************************************************************
+       ** {@inheritDoc}
+       *******************************************************************************/
+      @Override
+      public void raise(AlertSignal signal)
+      {
+         if(failWith != null)
+         {
+            throw (failWith);
+         }
+         raised.add(signal);
+      }
+
+
+
+      /*******************************************************************************
+       ** {@inheritDoc}
+       *******************************************************************************/
+      @Override
+      public void resolve(AlertResolution resolution)
+      {
+         if(failWith != null)
+         {
+            throw (failWith);
+         }
+         resolved.add(resolution);
+      }
+
+
+
+      /*******************************************************************************
+       ** {@inheritDoc}
+       *******************************************************************************/
+      @Override
+      public void heartbeat(String heartbeatKey, Instant at)
+      {
+         if(failWith != null)
+         {
+            throw (failWith);
+         }
+         heartbeats.add(heartbeatKey);
+      }
    }
 }

src/test/java/com/kingsrook/qbits/backups/alerts/BackupAlertsTest.java

@@ -70,8 +70,8 @@ class BackupAlertsTest
    void testTheSpecTable()
    {
       Set<String> names = BackupAlerts.alertTypes().stream().map(BackupAlerts.AlertType::name).collect(Collectors.toSet());
-      assertThat(names).containsExactlyInAnyOrder("backups.plan.missed", "backups.run.failed", "backups.run.missing_in_storage", "backups.files.missing",
-         "backups.verify.failed", "backups.verify.overdue", "backups.plan.owner_invalid", "backups.storage.debris", "backups.restore.completed");
+      assertThat(names).containsExactlyInAnyOrder("backups.plan.missed", "backups.run.failed", "backups.run.missing-in-storage", "backups.files.missing",
+         "backups.verify.failed", "backups.verify.overdue", "backups.plan.owner-invalid", "backups.storage.debris", "backups.restore.completed");
       assertThat(BackupAlerts.alertTypes()).allSatisfy(t -> assertThat(t.howToFix()).isNotBlank());
       assertThat(BackupAlerts.alertTypes().stream().filter(t -> t.name().equals(BackupAlerts.RUN_MISSING)).findFirst().orElseThrow().defaultSeverity())
          .isEqualTo(BackupAlertSink.Severity.CRITICAL);

src/test/java/com/kingsrook/qbits/backups/alerts/fakeapi/AlertSeverity.java

@@ -1,30 +0,0 @@
-/*
- * Copyright 2026 Kingsrook, LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kingsrook.qbits.backups.alerts.fakeapi;
-
-
-/*******************************************************************************
- ** Test double of the Alerts producer SPI (QRun-IO/qqq PR #1108): the same shape as
- ** com.kingsrook.qqq.backend.core.alerts.AlertSeverity.
- *******************************************************************************/
-public enum AlertSeverity
-{
-   INFO,
-   WARNING,
-   ERROR,
-   CRITICAL
-}

src/test/java/com/kingsrook/qbits/backups/alerts/fakeapi/AlertsInstanceMetaData.java

@@ -1,40 +0,0 @@
-/*
- * Copyright 2026 Kingsrook, LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kingsrook.qbits.backups.alerts.fakeapi;
-
-
-import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
-
-
-/*******************************************************************************
- ** Test double of the Alerts producer SPI AlertsInstanceMetaData: of(instance) is
- ** non-null only when a test says the Alerts qbit is installed.
- *******************************************************************************/
-public class AlertsInstanceMetaData
-{
-   public static boolean installed = false;
-
-
-
-   /*******************************************************************************
-    ** The alerts meta-data of an instance, or null.
-    *******************************************************************************/
-   public static AlertsInstanceMetaData of(QInstance qInstance)
-   {
-      return (installed ? new AlertsInstanceMetaData() : null);
-   }
-}

src/test/java/com/kingsrook/qbits/backups/alerts/fakeapi/QAlertTypeMetaData.java

@@ -1,103 +0,0 @@
-/*
- * Copyright 2026 Kingsrook, LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kingsrook.qbits.backups.alerts.fakeapi;
-
-
-import java.util.ArrayList;
-import java.util.List;
-import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
-
-
-/*******************************************************************************
- ** Test double of the Alerts producer SPI QAlertTypeMetaData: records the types
- ** added to an instance.
- *******************************************************************************/
-public class QAlertTypeMetaData
-{
-   public static final List<QAlertTypeMetaData> ADDED = new ArrayList<>();
-
-   public String        name;
-   public String        label;
-   public String        groupName;
-   public String        howToFix;
-   public AlertSeverity defaultSeverity;
-
-
-
-   /*******************************************************************************
-    ** Fluent.
-    *******************************************************************************/
-   public QAlertTypeMetaData withName(String name)
-   {
-      this.name = name;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent.
-    *******************************************************************************/
-   public QAlertTypeMetaData withLabel(String label)
-   {
-      this.label = label;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent.
-    *******************************************************************************/
-   public QAlertTypeMetaData withGroupName(String groupName)
-   {
-      this.groupName = groupName;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent.
-    *******************************************************************************/
-   public QAlertTypeMetaData withHowToFix(String howToFix)
-   {
-      this.howToFix = howToFix;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent.
-    *******************************************************************************/
-   public QAlertTypeMetaData withDefaultSeverity(AlertSeverity defaultSeverity)
-   {
-      this.defaultSeverity = defaultSeverity;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Add to the instance (recorded).
-    *******************************************************************************/
-   public void addSelfToInstance(QInstance qInstance)
-   {
-      ADDED.add(this);
-   }
-}

src/test/java/com/kingsrook/qbits/backups/alerts/fakeapi/QAlerts.java

@@ -1,104 +0,0 @@
-/*
- * Copyright 2026 Kingsrook, LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kingsrook.qbits.backups.alerts.fakeapi;
-
-
-import java.util.ArrayList;
-import java.util.List;
-
-
-/*******************************************************************************
- ** Test double of the Alerts producer SPI QAlerts: static calls that record what
- ** they were given.
- *******************************************************************************/
-public final class QAlerts
-{
-   public static final List<AlertSignal> RAISED     = new ArrayList<>();
-   public static final List<String>      RESOLVED   = new ArrayList<>();
-   public static final List<String>      HEARTBEATS = new ArrayList<>();
-
-   /** When set, every call throws it (the SPI is meant not to, but the adapter must survive). */
-   public static RuntimeException failWith;
-
-
-
-   /*******************************************************************************
-    ** Utility class.
-    *******************************************************************************/
-   private QAlerts()
-   {
-   }
-
-
-
-   /*******************************************************************************
-    ** Forget everything.
-    *******************************************************************************/
-   public static void reset()
-   {
-      RAISED.clear();
-      RESOLVED.clear();
-      HEARTBEATS.clear();
-      failWith = null;
-   }
-
-
-
-   /*******************************************************************************
-    ** Raise.
-    *******************************************************************************/
-   public static void raise(AlertSignal signal)
-   {
-      maybeFail();
-      RAISED.add(signal);
-   }
-
-
-
-   /*******************************************************************************
-    ** Resolve.
-    *******************************************************************************/
-   public static void resolve(String alertTypeName, String dedupKey, String note)
-   {
-      maybeFail();
-      RESOLVED.add(alertTypeName + "|" + dedupKey + "|" + note);
-   }
-
-
-
-   /*******************************************************************************
-    ** Heartbeat.
-    *******************************************************************************/
-   public static void heartbeat(String heartbeatKey)
-   {
-      maybeFail();
-      HEARTBEATS.add(heartbeatKey);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static void maybeFail()
-   {
-      if(failWith != null)
-      {
-         throw (failWith);
-      }
-   }
-}

src/test/java/com/kingsrook/qbits/backups/app/BackupsHomeTest.java

@@ -88,7 +88,7 @@ class BackupsHomeTest
    void testEmptyStates()
    {
       String html = BackupsHomeView.render(new BackupsHomeData(List.of(), List.of(), List.of(), List.of(), List.of(), Instant.now()), "/app");
-      assertThat(html).contains("No backup plans yet").contains("Nothing to check yet").contains("No test restore yet")
+      assertThat(html).contains("No backup plans yet").contains("Setup: <a").contains("No plan: nothing is backed up.").doesNotContain("Nothing needs attention").contains("No test restore yet")
          .contains("Nothing has run yet").contains("Trends appear after the first successful run").contains("Retention starts once runs exist");
       assertThat(BackupsHomeView.renderHealthCard(new BackupsHomeData(List.of(), List.of(), List.of(), List.of(), List.of(), Instant.now()), "/app")).contains("No backup plans yet");
    }
@@ -121,6 +121,45 @@ class BackupsHomeTest
 
 
 
+   /*******************************************************************************
+    ** The Setup card and the Needs attention card never contradict each other: a plan
+    ** whose storage aliases do not resolve is a Setup fault, and it is listed under
+    ** Needs attention too, with the same count.
+    *******************************************************************************/
+   @Test
+   void testUnresolvedAliasAppearsInSetupAndNeedsAttention() throws Exception
+   {
+      BackupsRuntime.getConfig().withStorageResolver(alias -> BackupsTestUtils.BACKUPS);
+      new BackupCatalog().insert(BackupsTables.PLAN, new LinkedHashMap<>(Map.of("name", "Offsite", "planKey", "offsite", "isActive", true, "scopeType", "INSTANCE",
+         "storageBucketAlias", "backupsAws", "pruneBucketAlias", "backupsAwsPrune")));
+
+      //////////////////////////////////////////////////////////////////////////////
+      // the aliases resolved when the plan was saved; now the bucket is gone      //
+      //////////////////////////////////////////////////////////////////////////////
+      BackupsRuntime.getConfig().withStorageResolver(alias -> "backups".equals(alias) ? BackupsTestUtils.BACKUPS : null);
+
+      BackupsHomeData data = BackupsHomeData.load();
+      List<BackupsSetupStatus.Item> setupTodo = BackupsSetupStatus.load(data.plans()).stream().filter(i -> i.state() == BackupsSetupStatus.State.TODO).toList();
+      assertThat(setupTodo).extracting(BackupsSetupStatus.Item::label).contains("Storage");
+
+      String html = BackupsHomeView.render(data, "/app");
+      String attention = html.substring(html.indexOf(">Needs attention<"));
+      attention = attention.substring(0, attention.indexOf("All runs"));
+      assertThat(attention).doesNotContain("Nothing needs attention").contains("backupsAws").contains("backupsAwsPrune");
+
+      ///////////////////////////////////////////////////////////////////////////////
+      // every Setup TODO is listed under Needs attention (counts agree: 2 and 2)   //
+      ///////////////////////////////////////////////////////////////////////////////
+      assertThat(html).contains(setupTodo.size() + " items need attention.");
+      for(BackupsSetupStatus.Item item : setupTodo)
+      {
+         assertThat(attention).contains("Setup: <a").contains(item.label());
+      }
+      assertThat(attention.split("<li>", -1).length - 1).isEqualTo(setupTodo.size() + data.plansNeedingAttention().size() + data.failedRuns().size());
+   }
+
+
+
    /*******************************************************************************
     ** The links follow the configured table path prefix.
     *******************************************************************************/

src/test/java/com/kingsrook/qbits/backups/app/BackupsSetupStatusTest.java

@@ -0,0 +1,172 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.app;
+
+
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qbits.backups.BackupsTestUtils;
+import com.kingsrook.qbits.backups.app.BackupsSetupStatus.Item;
+import com.kingsrook.qbits.backups.app.BackupsSetupStatus.State;
+import com.kingsrook.qbits.backups.model.BackupsTables;
+import com.kingsrook.qbits.backups.processes.DefaultPlanProcess;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** The Setup card of the Backups home: with no plan it tells the user to run Create
+ ** default plan; with a plan it checks the storage alias, the schedule, the
+ ** verification and the prune bucket, and links each to its table.
+ *******************************************************************************/
+class BackupsSetupStatusTest
+{
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   void setUp() throws Exception
+   {
+      BackupsTestUtils.init();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static Item item(List<Item> items, String label)
+   {
+      return (items.stream().filter(i -> i.label().equals(label)).findFirst().orElseThrow());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static QRecord plan(Map<String, Object> values)
+   {
+      QRecord plan = new QRecord().withValue("name", "Nightly").withValue("isActive", true).withValue("scopeType", "INSTANCE").withValue("storageBucketAlias", "backups");
+      values.forEach((k, v) -> plan.setValue(k, (java.io.Serializable) v));
+      return (plan);
+   }
+
+
+
+   /*******************************************************************************
+    ** No plans: the first step is the Create default plan action.
+    *******************************************************************************/
+   @Test
+   void noPlansPointsAtTheDefaultAction()
+   {
+      List<Item> items = BackupsSetupStatus.load(List.of());
+      assertThat(item(items, "Backup plans").state()).isEqualTo(State.TODO);
+      assertThat(item(items, "Backup plans").nextStep()).contains("Create default plan");
+      assertThat(item(items, "Plan scope rules").state()).isEqualTo(State.OPTIONAL);
+
+      String html = BackupsHomeView.render(new BackupsHomeData(List.of(), List.of(), List.of(), List.of(), List.of(), Instant.now()), "/app");
+      assertThat(html).contains("Setup").contains("1 item needs attention").contains("Create default plan").contains("href=\"/app/backups_plan\"");
+   }
+
+
+
+   /*******************************************************************************
+    ** A bare plan: storage resolves, but no schedule, no verification and no prune bucket.
+    *******************************************************************************/
+   @Test
+   void aBarePlanNeedsASchedule()
+   {
+      List<Item> items = BackupsSetupStatus.load(List.of(plan(Map.of())), true);
+      assertThat(item(items, "Backup plans").state()).isEqualTo(State.OK);
+      assertThat(item(items, "Storage").state()).isEqualTo(State.OK);
+      assertThat(item(items, "Schedule").state()).isEqualTo(State.TODO);
+      assertThat(item(items, "Verification").state()).isEqualTo(State.OPTIONAL);
+      assertThat(item(items, "Prune bucket").state()).isEqualTo(State.DEFAULT);
+   }
+
+
+
+   /*******************************************************************************
+    ** A complete plan: everything is in place.
+    *******************************************************************************/
+   @Test
+   void aCompletePlanIsSetUp()
+   {
+      List<Item> items = BackupsSetupStatus.load(List.of(plan(Map.of("cronExpression", DefaultPlanProcess.CRON, "verifyCronExpression", DefaultPlanProcess.VERIFY_CRON, "pruneBucketAlias", "backups"))), true);
+      assertThat(items).extracting(Item::state).doesNotContain(State.TODO, State.DEFAULT);
+      assertThat(items).extracting(Item::tableName).contains(BackupsTables.PLAN, BackupsTables.PLAN_SCOPE, BackupsTables.VERIFICATION);
+   }
+
+
+
+   /*******************************************************************************
+    ** Without a scheduler the crons never fire: Schedule and Verification are TODO, and a
+    ** plan whose isActive is null is not counted active (as at runtime).
+    *******************************************************************************/
+   @Test
+   void noSchedulerMeansCronsNeverFire()
+   {
+      QRecord complete = plan(Map.of("cronExpression", DefaultPlanProcess.CRON, "verifyCronExpression", DefaultPlanProcess.VERIFY_CRON, "pruneBucketAlias", "backups"));
+      List<Item> items = BackupsSetupStatus.load(List.of(complete), false);
+      assertThat(item(items, "Schedule").state()).isEqualTo(State.TODO);
+      assertThat(item(items, "Schedule").detail()).contains("No scheduler is configured");
+      assertThat(item(items, "Verification").state()).isEqualTo(State.TODO);
+      assertThat(item(items, "Verification").nextStep()).contains("schedulerName");
+
+      QRecord unset = new QRecord().withValue("name", "N").withValue("storageBucketAlias", "backups");
+      assertThat(item(BackupsSetupStatus.load(List.of(unset), true), "Backup plans").state()).isEqualTo(State.TODO);
+      assertThat(item(BackupsSetupStatus.load(List.of(complete)), "Schedule").state()).as("the test instance has no scheduler").isEqualTo(State.TODO);
+   }
+
+
+
+   /*******************************************************************************
+    ** A host that ticks plans itself is not told it needs a scheduler; a verify cron its
+    ** tick does not run is called out, and one it does run is fine.
+    *******************************************************************************/
+   @Test
+   void hostSchedulerCountsButUnrunVerifyIsCalledOut()
+   {
+      QRecord complete = plan(Map.of("cronExpression", DefaultPlanProcess.CRON, "verifyCronExpression", DefaultPlanProcess.VERIFY_CRON, "pruneBucketAlias", "backups"));
+      List<Item> items = BackupsSetupStatus.load(List.of(complete), false, "the website ticker", false);
+      assertThat(item(items, "Schedule").state()).isEqualTo(State.OK);
+      assertThat(item(items, "Schedule").detail()).contains("the website ticker");
+      assertThat(item(items, "Verification").state()).isEqualTo(State.OPTIONAL);
+      assertThat(item(items, "Verification").detail()).contains("Not run by the website ticker");
+
+      assertThat(item(BackupsSetupStatus.load(List.of(complete), false, "the website ticker", true), "Verification").state()).isEqualTo(State.OK);
+   }
+
+
+
+   /*******************************************************************************
+    ** An alias that does not resolve, an inactive plan and a scoped plan are called out.
+    *******************************************************************************/
+   @Test
+   void problemsAreCalledOut()
+   {
+      List<Item> items = BackupsSetupStatus.load(List.of(plan(Map.of("storageBucketAlias", "nowhere", "isActive", false, "scopeType", "TABLES"))), true);
+      assertThat(item(items, "Backup plans").state()).isEqualTo(State.TODO);
+      assertThat(item(items, "Storage").state()).isEqualTo(State.TODO);
+      assertThat(item(items, "Storage").detail()).contains("nowhere");
+      assertThat(item(items, "Plan scope rules").detail()).contains("limited by scope rules");
+   }
+}

src/test/java/com/kingsrook/qbits/backups/health/BackupsHealthChecksTest.java

@@ -0,0 +1,495 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.health;
+
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import com.kingsrook.qbits.backups.BackupsRuntime;
+import com.kingsrook.qbits.backups.BackupsTestUtils;
+import com.kingsrook.qbits.backups.VerifyTestSupport;
+import com.kingsrook.qbits.backups.model.BackupsTables;
+import com.kingsrook.qbits.backups.plan.DestinationProbe;
+import com.kingsrook.qbits.backups.run.BackupCatalog;
+import com.kingsrook.qbits.backups.storage.InMemoryBackupStorage;
+import com.kingsrook.qbits.backups.storage.StoredObjectInfo;
+import com.kingsrook.qbits.backups.storage.WriteOptions;
+import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.health.HealthCategory;
+import com.kingsrook.qqq.backend.core.health.HealthCheck;
+import com.kingsrook.qqq.backend.core.health.HealthCheckInput;
+import com.kingsrook.qqq.backend.core.health.HealthCheckKind;
+import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData;
+import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome;
+import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData;
+import com.kingsrook.qqq.backend.core.health.HealthStatus;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** Central registration, migrated SLA rules, read-only probes and secret safety.
+ *******************************************************************************/
+class BackupsHealthChecksTest
+{
+   private static final String SECRET = "S3NT1NEL-secret";
+
+   private final BackupCatalog catalog = new BackupCatalog();
+   private ReadOnlyStorage storage;
+
+
+
+   /*******************************************************************************
+    ** All probes use memory storage; the resolver represents credential injection.
+    *******************************************************************************/
+   @BeforeEach
+   void setUp() throws Exception
+   {
+      VerifyTestSupport.setUp(3);
+      storage = new ReadOnlyStorage(SECRET);
+      BackupsRuntime.getConfig().withStorageResolver(alias ->
+      {
+         assertThat(alias).as("never resolve prune credentials").isNotEqualTo("prune");
+         return (storage);
+      });
+   }
+
+
+
+   /*******************************************************************************
+    ** Use the actual producer registry, including its reflective supplier loading.
+    *******************************************************************************/
+   @Test
+   void testRegistrationAndDynamicPlans() throws Exception
+   {
+      HealthChecksMetaData registry = HealthChecksMetaData.ofOrWithNew(QContext.getQInstance());
+      assertThat(registry.getChecks()).extracting(HealthCheckMetaData::getName).containsExactly("backups.sla");
+      assertThat(registry.getSuppliers()).singleElement().satisfies(supplier ->
+      {
+         assertThat(supplier.getNamePrefix()).isEqualTo("backups.destination");
+         assertThat(supplier.getComponent()).isEqualTo("backups");
+      });
+      QRecord active = VerifyTestSupport.plan(Map.of());
+      VerifyTestSupport.plan(Map.of("name", "Off", "planKey", "off", "isActive", false));
+      List<HealthCheckMetaData> checks = registry.resolveAll(QContext.getQInstance());
+      assertThat(checks).hasSize(2).allSatisfy(check ->
+      {
+         assertThat(check.getComponent()).isEqualTo("backups");
+         assertThat(check.getKind()).isEqualTo(HealthCheckKind.DEPENDENCY);
+         assertThat(check.getCategory()).isEqualTo(HealthCategory.STORAGE);
+         assertThat(check.getIncludeInReadiness()).isFalse();
+         assertThat(check.getName()).matches("[a-z0-9]+([._-][a-z0-9]+)*");
+      });
+      assertThat(checks.get(0).getIntervalSeconds()).isEqualTo(900);
+      assertThat(checks.get(0).getTimeoutMs()).isEqualTo(5000);
+      assertThat(QCodeLoader.getAdHoc(HealthCheck.class, checks.get(0).getCheck()).run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
+      HealthCheckMetaData destination = checks.get(1);
+      assertThat(destination.getName()).isEqualTo("backups.destination." + active.getValueInteger("id"));
+      assertThat(destination.getIntervalSeconds()).isEqualTo(86400);
+      assertThat(destination.getTimeoutMs()).isEqualTo(30000);
+      assertThat(destination.getCheckInstance().run(input()).getStatus()).isEqualTo(HealthStatus.UP);
+      catalog.update(BackupsTables.PLAN, active.getValueInteger("id"), Map.of("isActive", false));
+      assertThat(destination.getCheckInstance().run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
+      assertThat(registry.resolveAll(QContext.getQInstance())).hasSize(1);
+      VerifyTestSupport.plan(Map.of("name", "New", "planKey", "new"));
+      assertThat(registry.resolveAll(QContext.getQInstance())).hasSize(2);
+   }
+
+
+
+   /*******************************************************************************
+    ** Carry over the indicator's empty, critical, ordinary and inactive cases.
+    *******************************************************************************/
+   @Test
+   void testSlaUpDownAndDegraded() throws Exception
+   {
+      BackupsSlaHealthCheck check = new BackupsSlaHealthCheck();
+      HealthCheckOutcome empty = check.run(input());
+      assertThat(empty.getStatus()).isEqualTo(HealthStatus.UP);
+      assertThat(empty.getDetails().getValues()).containsEntry("activePlans", 0);
+      QRecord critical = VerifyTestSupport.plan(Map.of("critical", true));
+      QRecord ordinary = VerifyTestSupport.plan(Map.of("name", "Other", "planKey", "other"));
+      QRecord inactive = VerifyTestSupport.plan(Map.of("name", "Off", "planKey", "off", "isActive", false, "critical", true));
+      setStatus(inactive, "LATE");
+      setStatus(critical, "OK");
+      setStatus(ordinary, "OK");
+      assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UP);
+      for(String status : List.of("LATE", "FAILING", "MISSING_IN_STORAGE"))
+      {
+         setStatus(ordinary, status);
+         assertThat(check.run(input()).getStatus()).as(status).isEqualTo(HealthStatus.DEGRADED);
+         setStatus(critical, status);
+         HealthCheckOutcome down = check.run(input());
+         assertThat(down.getStatus()).as(status).isEqualTo(HealthStatus.DOWN);
+         assertThat(down.getDetails().getValues()).containsEntry("activePlans", 2).containsEntry("slaStatus." + status, 2);
+         assertClean(down);
+         setStatus(critical, "OK");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Unknown or unverified plans do not become a false UP or a critical DOWN.
+    *******************************************************************************/
+   @Test
+   void testSlaUnknownAndMixedStates() throws Exception
+   {
+      QRecord plan = VerifyTestSupport.plan(Map.of("critical", true));
+      BackupsSlaHealthCheck check = new BackupsSlaHealthCheck();
+      assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
+      for(String status : List.of("UNVERIFIED", "UNKNOWN"))
+      {
+         setStatus(plan, status);
+         assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
+      }
+      QRecord other = VerifyTestSupport.plan(Map.of("name", "Other", "planKey", "other"));
+      setStatus(other, "UNVERIFIED");
+      assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
+      setStatus(other, "OK");
+      assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UP);
+   }
+
+
+
+   /*******************************************************************************
+    ** Preserve the old indicator's no-context behavior and existing sessions.
+    *******************************************************************************/
+   @Test
+   void testWithoutContext() throws Exception
+   {
+      QRecord plan = VerifyTestSupport.plan(Map.of());
+      QInstance instance = QContext.getQInstance();
+      QSession session = QContext.getQSession();
+      new BackupsSlaHealthCheck().run(input());
+      assertThat(QContext.getQSession()).isSameAs(session);
+      QContext.clear();
+      try
+      {
+         HealthCheckInput input = new HealthCheckInput().withQInstance(instance).withDeadline(Instant.now().plusSeconds(5));
+         assertThat(new BackupsSlaHealthCheck().run(input).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
+         assertThat(QContext.getQInstance()).isNull();
+         assertThat(new BackupsDestinationHealthChecks().supply(instance)).hasSize(1);
+         assertThat(QContext.getQInstance()).isNull();
+         BackupsRuntime.getConfig().withStorageResolver(alias ->
+         {
+            assertThat(QContext.getQSession()).isInstanceOf(com.kingsrook.qqq.backend.core.model.session.QSystemUserSession.class);
+            return (storage);
+         });
+         assertThat(destination(plan).run(input).getStatus()).isEqualTo(HealthStatus.UP);
+         assertThat(QContext.getQInstance()).isNull();
+      }
+      finally
+      {
+         QContext.init(instance, session);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Locked and unlocked plans only list/head; neither reads object contents.
+    *******************************************************************************/
+   @Test
+   void testDestinationReadOnlyAndMissingManifest() throws Exception
+   {
+      for(String lockMode : List.of("NONE", "GOVERNANCE"))
+      {
+         BackupsRuntime.getConfig().withStorageResolver(alias -> BackupsTestUtils.BACKUPS);
+         QRecord plan = VerifyTestSupport.plan(Map.of("name", lockMode, "planKey", lockMode.toLowerCase(), "objectLockMode", lockMode, "objectLockDays", 3, "pruneBucketAlias", "prune"));
+         BackupsRuntime.getConfig().withStorageResolver(alias ->
+         {
+            assertThat(alias).as("never resolve prune credentials").isNotEqualTo("prune");
+            return (storage);
+         });
+         Integer runId = catalog.insert(BackupsTables.RUN, Map.of("manifestKey", "manifest.json"));
+         catalog.update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("lastSuccessRunId", runId));
+         storage.putBytes("manifest.json", new byte[] { 1 });
+         assertThat(destination(plan).run(input()).getStatus()).isEqualTo(HealthStatus.UP);
+         storage.clear();
+         assertThat(destination(plan).run(input()).getStatus()).isEqualTo(HealthStatus.DOWN);
+      }
+      assertThat(storage.lists).isEqualTo(4);
+      assertThat(storage.heads).isEqualTo(4);
+   }
+
+
+
+   /*******************************************************************************
+    ** A deleted plan is UNKNOWN and does not touch storage.
+    *******************************************************************************/
+   @Test
+   void testDeletedPlan() throws Exception
+   {
+      QRecord plan = VerifyTestSupport.plan(Map.of());
+      catalog.deleteByIds(BackupsTables.PLAN, List.of(plan.getValueInteger("id")));
+      assertThat(destination(plan).run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
+      assertThat(storage.lists).isZero();
+   }
+
+
+
+   /*******************************************************************************
+    ** No work starts with an expired deadline, including direct probe calls.
+    *******************************************************************************/
+   @Test
+   void testExpiredDeadline() throws Exception
+   {
+      QRecord plan = VerifyTestSupport.plan(Map.of());
+      HealthCheckInput expired = input().withDeadline(Instant.now().minusSeconds(1));
+      assertThat(new BackupsSlaHealthCheck().run(expired).getStatus()).isEqualTo(HealthStatus.DOWN);
+      assertThat(destination(plan).run(expired).getStatus()).isEqualTo(HealthStatus.DOWN);
+      assertThat(DestinationProbe.probeReadOnly(plan, expired.getDeadline()).ok()).isFalse();
+      assertThat(storage.lists).isZero();
+      assertThat(storage.heads).isZero();
+   }
+
+
+
+   /*******************************************************************************
+    ** Runner cancellation after list prevents the following manifest head.
+    *******************************************************************************/
+   @Test
+   void testCancellationBetweenStorageCalls() throws Exception
+   {
+      QRecord plan = VerifyTestSupport.plan(Map.of());
+      Integer runId = catalog.insert(BackupsTables.RUN, Map.of("manifestKey", "manifest.json"));
+      catalog.update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("lastSuccessRunId", runId));
+      storage.interruptAfterList = true;
+      try
+      {
+         assertThat(destination(plan).run(input()).getStatus()).isEqualTo(HealthStatus.DOWN);
+         assertThat(storage.lists).isEqualTo(1);
+         assertThat(storage.heads).isZero();
+      }
+      finally
+      {
+         Thread.interrupted();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Poison configuration, persisted state and provider errors. sanitize is
+    ** package-private in core; assert raw outcomes already contain no secrets.
+    *******************************************************************************/
+   @Test
+   void testNoSecretsInSuccessAndFailureOutcomes() throws Exception
+   {
+      QRecord plan = VerifyTestSupport.plan(Map.of("name", SECRET, "storageBucketAlias", SECRET, "critical", true));
+      Integer runId = catalog.insert(BackupsTables.RUN, Map.of("manifestKey", SECRET));
+      catalog.update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("lastSuccessRunId", runId));
+      storage.putBytes(SECRET, new byte[] { 1 });
+      for(String status : List.of("OK", "LATE", "UNVERIFIED", SECRET))
+      {
+         setStatus(plan, status);
+         assertClean(new BackupsSlaHealthCheck().run(input()));
+         assertClean(destination(plan).run(input()));
+      }
+      storage.failList = true;
+      HealthCheckOutcome failure = destination(plan).run(input());
+      assertThat(failure.getStatus()).isEqualTo(HealthStatus.DOWN);
+      assertClean(failure);
+      storage.failList = false;
+      storage.failHead = true;
+      failure = destination(plan).run(input());
+      assertThat(failure.getStatus()).isEqualTo(HealthStatus.DOWN);
+      assertClean(failure);
+      BackupsRuntime.getConfig().withStorageResolver(alias ->
+      {
+         throw (new IllegalStateException(SECRET));
+      });
+      failure = destination(plan).run(input());
+      assertThat(failure.getStatus()).isEqualTo(HealthStatus.DOWN);
+      assertClean(failure);
+      assertClean(new BackupsSlaHealthCheck().run(input()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Catalog failures are DOWN with fixed messages, even if metadata is poisoned.
+    *******************************************************************************/
+   @Test
+   void testCatalogFailureDoesNotExposeSecrets()
+   {
+      QContext.getQInstance().getTable(BackupsTables.PLAN).setBackendName(SECRET);
+      HealthCheckOutcome sla = new BackupsSlaHealthCheck().run(input());
+      HealthCheckOutcome destination = new BackupsDestinationHealthCheck(1).run(input());
+      assertThat(sla.getStatus()).isEqualTo(HealthStatus.DOWN);
+      assertThat(destination.getStatus()).isEqualTo(HealthStatus.DOWN);
+      assertClean(sla);
+      assertClean(destination);
+   }
+
+
+
+   /*******************************************************************************
+    ** Update persisted SLA state without running the scheduled calculator.
+    *******************************************************************************/
+   private void setStatus(QRecord plan, String status) throws Exception
+   {
+      catalog.update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("slaStatus", status));
+   }
+
+
+
+   /*******************************************************************************
+    ** Fresh bounded input for each call.
+    *******************************************************************************/
+   private HealthCheckInput input()
+   {
+      return (new HealthCheckInput().withQInstance(QContext.getQInstance()).withDeadline(Instant.now().plusSeconds(5)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Construct the same per-plan check as the supplier.
+    *******************************************************************************/
+   private BackupsDestinationHealthCheck destination(QRecord plan)
+   {
+      return (new BackupsDestinationHealthCheck(plan.getValueInteger("id")));
+   }
+
+
+
+   /*******************************************************************************
+    ** Validate the raw public message and flattened detail representation.
+    *******************************************************************************/
+   private void assertClean(HealthCheckOutcome outcome)
+   {
+      assertThat(outcome.getMessage()).doesNotContain(SECRET, "\n", "\r");
+      assertThat(outcome.getDetails().toJson()).doesNotContain(SECRET);
+   }
+
+
+
+   /*******************************************************************************
+    ** Existing memory fake with read failures and assertions against side effects.
+    *******************************************************************************/
+   private static class ReadOnlyStorage extends InMemoryBackupStorage
+   {
+      private final String credential;
+      private boolean failList;
+      private boolean failHead;
+      private boolean interruptAfterList;
+      private int lists;
+      private int heads;
+
+
+
+      /*******************************************************************************
+       ** Mimic the credential held by a host-provided storage adapter.
+       *******************************************************************************/
+      ReadOnlyStorage(String credential)
+      {
+         this.credential = credential;
+      }
+
+
+
+      /*******************************************************************************
+       ** Record list calls and inject credential-bearing failures or cancellation.
+       *******************************************************************************/
+      @Override
+      public List<StoredObjectInfo> list(String prefix)
+      {
+         lists++;
+         if(failList)
+         {
+            throw (new IllegalStateException(credential));
+         }
+         if(interruptAfterList)
+         {
+            Thread.currentThread().interrupt();
+         }
+         return (super.list(prefix));
+      }
+
+
+
+      /*******************************************************************************
+       ** Record head calls and inject credential-bearing failures.
+       *******************************************************************************/
+      @Override
+      public Optional<StoredObjectInfo> head(String key)
+      {
+         heads++;
+         if(failHead)
+         {
+            throw (new IllegalStateException(credential));
+         }
+         return (super.head(key));
+      }
+
+
+
+      /*******************************************************************************
+       ** Health probes must not write, even to an unlocked bucket.
+       *******************************************************************************/
+      @Override
+      public OutputStream openWrite(String key, WriteOptions options)
+      {
+         throw (new AssertionError("Health probe wrote an object"));
+      }
+
+
+
+      /*******************************************************************************
+       ** Health probes must not fetch backup contents.
+       *******************************************************************************/
+      @Override
+      public InputStream openRead(String key)
+      {
+         throw (new AssertionError("Health probe read object contents"));
+      }
+
+
+
+      /*******************************************************************************
+       ** Health probes must not delete.
+       *******************************************************************************/
+      @Override
+      public DeleteResult deletePermanent(String key)
+      {
+         throw (new AssertionError("Health probe deleted an object"));
+      }
+
+
+
+      /*******************************************************************************
+       ** Capability diagnostics can be active probes; health must not invoke them.
+       *******************************************************************************/
+      @Override
+      public Set<StorageCapability> capabilities()
+      {
+         throw (new AssertionError("Health probe requested capabilities"));
+      }
+   }
+}

src/test/java/com/kingsrook/qbits/backups/health/BackupsHealthIndicatorTest.java

@@ -1,119 +0,0 @@
-/*
- * Copyright 2026 Kingsrook, LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kingsrook.qbits.backups.health;
-
-
-import java.util.Map;
-import com.kingsrook.qbits.backups.VerifyTestSupport;
-import com.kingsrook.qbits.backups.model.BackupsTables;
-import com.kingsrook.qbits.backups.run.BackupCatalog;
-import com.kingsrook.qqq.backend.core.context.QContext;
-import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.middleware.health.model.metadata.HealthCheckResult;
-import com.kingsrook.qqq.middleware.health.model.metadata.HealthStatus;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import static org.assertj.core.api.Assertions.assertThat;
-
-
-/*******************************************************************************
- ** The health indicator: DOWN only when a critical, active plan is LATE, FAILING or
- ** MISSING_IN_STORAGE.
- *******************************************************************************/
-class BackupsHealthIndicatorTest
-{
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @BeforeEach
-   void setUp() throws Exception
-   {
-      VerifyTestSupport.setUp(3);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private void setStatus(QRecord plan, String status) throws Exception
-   {
-      new BackupCatalog().update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("slaStatus", status));
-   }
-
-
-
-   /*******************************************************************************
-    ** UP with details when nothing critical is wrong, including a non-critical plan
-    ** that is LATE; DOWN when a critical plan is LATE, FAILING or MISSING_IN_STORAGE.
-    *******************************************************************************/
-   @Test
-   void testUpAndDown() throws Exception
-   {
-      BackupsHealthIndicator indicator = new BackupsHealthIndicator();
-      assertThat(indicator.getName()).isEqualTo("backups");
-
-      HealthCheckResult empty = indicator.check(QContext.getQInstance());
-      assertThat(empty.getStatus()).isEqualTo(HealthStatus.UP);
-      assertThat(empty.getDetails()).containsEntry("activePlans", 0);
-
-      QRecord critical = VerifyTestSupport.plan(Map.of("critical", true));
-      QRecord ordinary = VerifyTestSupport.plan(Map.of("name", "Other", "planKey", "other"));
-      VerifyTestSupport.plan(Map.of("name", "Off", "planKey", "off", "isActive", false, "critical", true));
-
-      setStatus(critical, "OK");
-      setStatus(ordinary, "LATE");
-      HealthCheckResult up = indicator.check(QContext.getQInstance());
-      assertThat(up.getStatus()).isEqualTo(HealthStatus.UP);
-      assertThat(up.getDetails()).containsEntry("activePlans", 2).doesNotContainKey("criticalPlansWithProblems");
-      assertThat(up.getDurationMs()).isNotNull();
-
-      for(String status : new String[] { "LATE", "FAILING", "MISSING_IN_STORAGE" })
-      {
-         setStatus(critical, status);
-         HealthCheckResult down = indicator.check(QContext.getQInstance());
-         assertThat(down.getStatus()).as(status).isEqualTo(HealthStatus.DOWN);
-         assertThat(down.getDetails().get("criticalPlansWithProblems").toString()).contains("nightly: " + status);
-      }
-
-      setStatus(critical, "UNVERIFIED");
-      assertThat(indicator.check(QContext.getQInstance()).getStatus()).as("not verified is not down").isEqualTo(HealthStatus.UP);
-   }
-
-
-
-   /*******************************************************************************
-    ** On a thread with no QQQ context (a health request) the indicator makes its own
-    ** and cleans up.
-    *******************************************************************************/
-   @Test
-   void testWorksWithoutAContext() throws Exception
-   {
-      QInstance qInstance = QContext.getQInstance();
-      QContext.clear();
-      try
-      {
-         assertThat(new BackupsHealthIndicator().check(qInstance).getStatus()).isEqualTo(HealthStatus.UP);
-         assertThat(QContext.getQInstance()).isNull();
-      }
-      finally
-      {
-         QContext.init(qInstance, new com.kingsrook.qqq.backend.core.model.session.QSession());
-      }
-   }
-}

src/test/java/com/kingsrook/qbits/backups/manifest/TableFingerprintsTest.java

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.manifest;
+
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.kingsrook.qbits.backups.BackupsTestUtils;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** Fingerprint algorithm dispatch is explicit; a marker cannot be ignored just
+ ** because its hash happens to match a different supported algorithm.
+ *******************************************************************************/
+class TableFingerprintsTest
+{
+   /*******************************************************************************
+    ** A frozen old hash matches only the legacy algorithm, and new hashes only
+    ** match the declared core algorithm. Cosmetic changes preserve both.
+    *******************************************************************************/
+   @Test
+   void testAlgorithmDispatch() throws Exception
+   {
+      BackupsTestUtils.init();
+      var table = QContext.getQInstance().getTable("blogPost");
+      ObjectMapper mapper = new ObjectMapper();
+      ObjectNode entry;
+      try(var in = getClass().getResourceAsStream("/manifests/legacy-schema.json"))
+      {
+         entry = (ObjectNode) mapper.readTree(in).get("blogPost");
+      }
+      String oldHash = entry.get("fingerprint").asText();
+      String coreHash = TableFingerprints.of(table, "memory");
+      assertThat(coreHash).isNotEqualTo(oldHash);
+      table.setLabel("Cosmetic label");
+      assertThat(TableFingerprints.matches(mapper.treeToValue(entry, BackupManifest.TableEntry.class), table, "memory")).isTrue();
+      entry.put("fingerprintAlgorithm", TableFingerprints.LEGACY_ALGORITHM);
+      assertThat(TableFingerprints.matches(mapper.treeToValue(entry, BackupManifest.TableEntry.class), table, "memory")).isTrue();
+      entry.put("fingerprintAlgorithm", TableFingerprints.CORE_ALGORITHM);
+      assertThat(TableFingerprints.matches(mapper.treeToValue(entry, BackupManifest.TableEntry.class), table, "memory")).isFalse();
+      entry.put("fingerprint", coreHash);
+      assertThat(TableFingerprints.matches(mapper.treeToValue(entry, BackupManifest.TableEntry.class), table, "memory")).isTrue();
+      entry.put("fingerprintAlgorithm", "unknown-v2");
+      assertThat(TableFingerprints.matches(mapper.treeToValue(entry, BackupManifest.TableEntry.class), table, "memory")).isFalse();
+      entry.remove("fingerprint");
+      assertThat(TableFingerprints.matches(mapper.treeToValue(entry, BackupManifest.TableEntry.class), table, "memory")).isFalse();
+   }
+}

src/test/java/com/kingsrook/qbits/backups/metrics/BackupsMetricsTest.java

@@ -87,10 +87,15 @@ class BackupsMetricsTest
       String empty = BackupsMetrics.renderCached();
       assertThat(empty).doesNotContain("{instance_name");
 
-      VerifyTestSupport.backup(VerifyTestSupport.plan(Map.of()));
       VerifyTestSupport.clock.advance(Duration.ofSeconds(30));
-      assertThat(BackupsMetrics.renderCached()).as("cached").isEqualTo(empty);
+      assertThat(BackupsMetrics.renderCached()).as("cached").isSameAs(empty);
       VerifyTestSupport.clock.advance(Duration.ofSeconds(31));
+      assertThat(BackupsMetrics.renderCached()).as("expired").isNotSameAs(empty);
+
+      /////////////////////////////////////////////////////////////
+      // a saved plan and a finished run clear the cache at once //
+      /////////////////////////////////////////////////////////////
+      VerifyTestSupport.backup(VerifyTestSupport.plan(Map.of()));
       assertThat(BackupsMetrics.renderCached()).contains("plan=\"nightly\"");
 
       assertThat(BackupsMetrics.escape("a\"b\\c\nd")).isEqualTo("a\\\"b\\\\c\\nd");

src/test/java/com/kingsrook/qbits/backups/restore/LegacyManifestRestoreTest.java

@@ -0,0 +1,123 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.restore;
+
+
+import java.util.Map;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.kingsrook.qbits.backups.BackupsTestUtils;
+import com.kingsrook.qbits.backups.codec.HashingOutputStream;
+import com.kingsrook.qbits.backups.manifest.BackupManifest;
+import com.kingsrook.qbits.backups.model.BackupsTables;
+import com.kingsrook.qbits.backups.run.BackupCatalog;
+import com.kingsrook.qbits.backups.run.BackupRunResult;
+import com.kingsrook.qbits.backups.verify.BackupVerifier;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** Exercise pre-RC.6 manifest fingerprints through verification and restore.
+ ** The schema fixture was frozen from the original stand-in, not recomputed
+ ** with the implementation being tested.
+ *******************************************************************************/
+class LegacyManifestRestoreTest
+{
+   /*******************************************************************************
+    ** Load the blog and grant the permission required by the core restore flag.
+    *******************************************************************************/
+   @BeforeEach
+   void setUp() throws Exception
+   {
+      BackupsTestUtils.init();
+      BackupsTestUtils.loadBlog(30);
+   }
+
+
+
+   /*******************************************************************************
+    ** Build an old-format manifest around real backup parts and re-seal its hash.
+    *******************************************************************************/
+   private BackupRunResult legacyBackup() throws Exception
+   {
+      BackupRunResult run = RestoreTestSupport.backup(Map.of());
+      ObjectMapper mapper = new ObjectMapper();
+      ObjectNode manifest = (ObjectNode) mapper.readTree(BackupsTestUtils.BACKUPS.getBytes(run.getManifestKey()));
+      JsonNode legacy;
+      try(var in = getClass().getResourceAsStream("/manifests/legacy-schema.json"))
+      {
+         legacy = mapper.readTree(in);
+      }
+      for(JsonNode table : manifest.get("tables"))
+      {
+         ObjectNode entry = (ObjectNode) table;
+         JsonNode old = legacy.get(entry.get("name").asText());
+         entry.remove("fingerprintAlgorithm");
+         entry.set("schema", old.get("schema"));
+         entry.set("fingerprint", old.get("fingerprint"));
+      }
+      byte[] bytes = mapper.writeValueAsBytes(manifest);
+      BackupsTestUtils.BACKUPS.putBytes(run.getManifestKey(), bytes);
+      new BackupCatalog().update(BackupsTables.RUN, run.getRunId(), Map.of("manifestSha256", HashingOutputStream.sha256Hex(bytes)));
+      assertThat(BackupManifest.read(bytes).table("blogPost").fingerprintAlgorithm()).isNull();
+      return (run);
+   }
+
+
+
+   /*******************************************************************************
+    ** Old hashes still receive exact codec verification and restore all data,
+    ** including original timestamps, after deletes, edits and additions.
+    *******************************************************************************/
+   @Test
+   void testLegacyManifestVerifiesAndRestores() throws Exception
+   {
+      String before = RestoreTestSupport.contentHash("blogPost");
+      BackupRunResult run = legacyBackup();
+      var verified = new BackupVerifier().verify(run.getRunId(), BackupVerifier.CONTENT, "MANUAL", "tester");
+      assertThat(verified.status()).isEqualTo("PASSED");
+      assertThat(verified.messages()).noneMatch(message -> message.contains("has changed since"));
+      RestoreTestSupport.mutate();
+      RestoreOutcome outcome = new RestoreEngine().execute(RestoreTestSupport.request(run, RestoreRequest.MODE_REPLACE, "blogPost").withConfirmationText("blogPost"));
+      assertThat(outcome.getErrors()).isEmpty();
+      assertThat(outcome.getTables().get("blogPost").getSchemaStatus()).isEqualTo("MATCH");
+      assertThat(RestoreTestSupport.contentHash("blogPost")).isEqualTo(before);
+   }
+
+
+
+   /*******************************************************************************
+    ** Legacy field maps and core field lists also compare correctly when hashes
+    ** differ: optional additions are compatible and type changes are rejected.
+    *******************************************************************************/
+   @Test
+   void testLegacySchemaChangesAreAnalyzed() throws Exception
+   {
+      BackupRunResult run = legacyBackup();
+      QContext.getQInstance().getTable("author").withField(new QFieldMetaData("nickname", QFieldType.STRING));
+      QContext.getQInstance().getTable("blogPost").getField("title").setType(QFieldType.INTEGER);
+      RestoreOutcome review = new RestoreEngine().review(RestoreTestSupport.request(run, RestoreRequest.MODE_UPSERT, "author", "blogPost"));
+      assertThat(review.getTables().get("author").getSchemaStatus()).isEqualTo("COMPATIBLE");
+      assertThat(review.getTables().get("blogPost").getSchemaStatus()).isEqualTo("MISMATCH");
+   }
+}

src/test/java/com/kingsrook/qbits/backups/restore/RestoreSafetyTest.java

@@ -187,4 +187,42 @@ class RestoreSafetyTest
       QContext.setQSession(new QSession().withPermissions("backups.view", "backups.restore"));
       assertThat(new RestoreEngine().plan(request).errors()).isEmpty();
    }
+
+
+   /*******************************************************************************
+    ** The core restore permission is required even in anonymous demo mode, and
+    ** Review reports it before a restore can create a run or write any data.
+    *******************************************************************************/
+   @Test
+   void testCoreRestorePermissionIsRequiredWhenAnonymous() throws Exception
+   {
+      BackupRunResult run = RestoreTestSupport.backup(Map.of());
+      QContext.setQSession(new QSession());
+      RestoreRequest request = RestoreTestSupport.request(run, RestoreRequest.MODE_UPSERT, "blogPost");
+      assertThat(new RestoreEngine().plan(request).errors()).anyMatch(e -> e.contains("backups.restore"));
+      assertThatThrownBy(() -> new RestoreEngine().execute(request)).hasMessageContaining("backups.restore");
+      assertThatThrownBy(() -> new StandardRestoreWriter().insert(QContext.getQInstance().getTable("blogPost"),
+         List.of(new com.kingsrook.qqq.backend.core.model.data.QRecord().withValue("id", 99)), request, null)).hasMessageContaining("backups.restore");
+   }
+
+
+
+   /*******************************************************************************
+    ** Job-page cancellation reaches the restore worker and its catalog status.
+    *******************************************************************************/
+   @Test
+   void testDurableCancellation() throws Exception
+   {
+      BackupRunResult run = RestoreTestSupport.backup(Map.of());
+      RestoreRequest request = RestoreTestSupport.request(run, RestoreRequest.MODE_UPSERT).withPreRestoreBackup(false);
+      try(var scope = com.kingsrook.qqq.backend.core.jobs.plans.JobChunkScope.open(null, () ->
+      {
+         throw (new com.kingsrook.qqq.backend.core.jobs.plans.JobInterruptedException(com.kingsrook.qqq.backend.core.jobs.plans.JobInterruptedException.Reason.CANCEL));
+      }))
+      {
+         assertThatThrownBy(() -> new RestoreEngine().execute(request)).isInstanceOf(com.kingsrook.qqq.backend.core.jobs.plans.JobInterruptedException.class);
+      }
+      assertThat(new BackupCatalog().where(BackupsTables.RESTORE_RUN, "status", "CANCELLED")).hasSize(1);
+   }
+
 }

src/test/java/com/kingsrook/qbits/backups/run/BackupEngineTest.java

@@ -20,6 +20,7 @@ package com.kingsrook.qbits.backups.run;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.zip.GZIPInputStream;
@@ -28,11 +29,25 @@ import com.kingsrook.qbits.backups.manifest.BackupManifest;
 import com.kingsrook.qbits.backups.model.BackupsTables;
 import com.kingsrook.qbits.backups.scope.PlanSettings;
 import com.kingsrook.qbits.backups.scope.ScopeResolver;
+import com.kingsrook.qbits.backups.scope.TableRole;
+import com.kingsrook.qbits.backups.scope.TableRoles;
+import com.kingsrook.qqq.backend.core.actions.async.AsyncJobCallback;
 import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.jobs.plans.JobChunkScope;
+import com.kingsrook.qqq.backend.core.jobs.plans.JobInterruptedException;
+import com.kingsrook.qqq.backend.core.jobs.progress.JobLogLevel;
+import com.kingsrook.qqq.backend.core.jobs.progress.JobPhase;
+import com.kingsrook.qqq.backend.core.jobs.progress.JobProgress;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
 
 /*******************************************************************************
@@ -109,7 +124,9 @@ class BackupEngineTest
       BackupManifest.TableEntry posts = manifest.table("blogPost");
       assertThat(posts.rowCount()).isEqualTo(7);
       assertThat(posts.excludedFields()).extracting(BackupManifest.ExcludedField::name).containsExactly("secretNote");
-      assertThat(posts.fingerprint()).startsWith("sha256:");
+      assertThat(posts.fingerprintAlgorithm()).isEqualTo(com.kingsrook.qbits.backups.manifest.TableFingerprints.CORE_ALGORITHM);
+      assertThat(posts.fingerprint()).isEqualTo("sha256:" + com.kingsrook.qqq.backend.core.model.metadata.tables.TableSchemaFingerprint.of(QContext.getQInstance().getTable("blogPost"), "memory").getHash());
+      assertThat(posts.schema()).isEqualTo(com.kingsrook.qqq.backend.core.model.metadata.tables.TableSchemaFingerprint.of(QContext.getQInstance().getTable("blogPost"), "memory").getDescriptor());
 
       BackupManifest.Part part = posts.parts().get(0);
       byte[] stored = BackupsTestUtils.BACKUPS.getBytes(result.getRunPrefix() + part.key());
@@ -185,7 +202,9 @@ class BackupEngineTest
       assertThat(gunzip(BackupsTestUtils.BACKUPS.getBytes(refOnly.getRunPrefix() + "files/index-00001.jsonl.gz"))).contains("REFERENCE_ONLY");
 
       BackupsTestUtils.MEDIA.removeIgnoringLock("media/photo-1.jpg");
-      BackupRunResult missing = run(new BackupCatalog().where(BackupsTables.PLAN, "planKey", "nightly").get(0));
+      RecordingProgress progress = new RecordingProgress();
+      BackupRunResult missing = new BackupEngine().execute(BackupRunFactory.forPlan(new BackupCatalog().where(BackupsTables.PLAN, "planKey", "nightly").get(0), "MANUAL").withProgress(progress));
+      assertThat(progress.logs).anyMatch(l -> l.startsWith("WARN:") && l.contains("missing"));
       assertThat(missing.getFilesMissing()).isGreaterThan(0);
       assertThat(missing.getWarnings()).anyMatch(w -> w.contains("missing"));
       assertThat(missing.getStatus()).isEqualTo("SUCCEEDED_WITH_WARNINGS");
@@ -206,7 +225,7 @@ class BackupEngineTest
          BackupRunResult result = run(plan(Map.of()));
          assertThat(result.getStatus()).isEqualTo("FAILED");
          assertThat(result.isUsable()).isFalse();
-         assertThat(result.getErrorMessage()).contains("injected write failure");
+         assertThat(result.getErrorMessage()).contains("Backup failed");
          assertThat(BackupsTestUtils.BACKUPS.list(result.getRunPrefix() + "manifest.json")).isEmpty();
          assertThat(new BackupCatalog().get(BackupsTables.RUN, result.getRunId()).orElseThrow().getValueString("status")).isEqualTo("FAILED");
       }
@@ -218,6 +237,38 @@ class BackupEngineTest
 
 
 
+   /*******************************************************************************
+    ** A table on a backend with no query interface (a storage-only module, like
+    ** qbit-edi's edi_preview_download) is left out of the scope, even when it is
+    ** selected by name, and the run succeeds.  Before the fix the reader queried it
+    ** and the run FAILED with "Query is not implemented in this module".
+    *******************************************************************************/
+   @Test
+   void testUnqueryableTableIsExcluded() throws Exception
+   {
+      QInstance qInstance = QContext.getQInstance();
+      qInstance.addBackend(new QBackendMetaData().withName("storageOnly").withBackendType(StorageOnlyBackendModule.class));
+      qInstance.addTable(new QTableMetaData().withName("previewDownload").withBackendName("storageOnly").withPrimaryKeyField("id")
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER)));
+
+      BackupRunResult result = run(plan(Map.of()));
+      assertThat(result.getErrorMessage()).isNull();
+      assertThat(result.getStatus()).isEqualTo("SUCCEEDED_WITH_WARNINGS");
+      BackupManifest manifest = BackupManifest.read(BackupsTestUtils.BACKUPS.getBytes(result.getManifestKey()));
+      assertThat(manifest.table("previewDownload")).isNull();
+      assertThat(manifest.table("blogPost")).isNotNull();
+
+      assertThat(TableRoles.roleOf(qInstance, qInstance.getTable("previewDownload"))).isEqualTo(TableRole.EXTERNAL);
+      assertThat(TableRoles.isQueryable(qInstance, qInstance.getTable("blogPost"))).isTrue();
+
+      ScopeResolver.ResolvedScope scope = new ScopeResolver(qInstance).resolve(PlanSettings.adHoc().with("scopeType", "TABLES").with("includeRoles", "DATA,EXTERNAL"),
+         List.of(new ScopeResolver.ScopeRule(true, "TABLE", "previewDownload", null, false, null), new ScopeResolver.ScopeRule(true, "ROLE", "EXTERNAL", null, false, null)));
+      assertThat(scope.names()).doesNotContain("previewDownload");
+      assertThat(scope.excluded()).contains(new ScopeResolver.ExcludedTable("previewDownload", ScopeResolver.NOT_QUERYABLE));
+   }
+
+
+
    /*******************************************************************************
     ** Required snapshots fail before anything is written (no backend can open one yet).
     *******************************************************************************/
@@ -279,4 +330,261 @@ class BackupEngineTest
       assertThat(scope.tables()).isEmpty();
       assertThat(scope.excluded()).extracting(ScopeResolver.ExcludedTable::reason).containsExactly("BACKUPS");
    }
+
+
+   /*******************************************************************************
+    ** A real plan reports work in order, relative artifact names and no row data.
+    *******************************************************************************/
+   @Test
+   void testJobProgress() throws Exception
+   {
+      RecordingProgress progress = new RecordingProgress();
+      BackupRunResult result = new BackupRunCoordinator().runPlan(plan(Map.of("prefix", "private-prefix")), "MANUAL", progress, null);
+      assertThat(result.isUsable()).isTrue();
+      assertThat(progress.phases).containsExactly("prepare", "tables", "media", "upload");
+      assertThat(progress.getSnapshot().getPhases()).extracting(JobPhase::getWeight).containsExactly(2.0, 75.0, 18.0, 5.0);
+      assertThat(progress.getSnapshot().getPhases()).extracting(JobPhase::getStatus)
+         .containsExactly(JobPhase.Status.DONE, JobPhase.Status.DONE, JobPhase.Status.DONE, JobPhase.Status.DONE);
+      assertThat(progress.counts).contains("tables:0/2 tables", "tables:1/2 tables", "tables:2/2 tables", "media:0/4 files", "media:4/4 files");
+      assertThat(progress.bytes).isSorted();
+      assertThat(progress.bytes.getLast()).isGreaterThan(0);
+      assertThat(progress.getSnapshot().getBytesTotal()).isNull();
+      assertThat(progress.messages).contains("Backing up author (1 of 2)", "Backing up blogPost (2 of 2)");
+      assertThat(progress.logs).anyMatch(l -> l.matches("INFO:author: 3 rows, .* MB in .* s"));
+      assertThat(progress.logs).anyMatch(l -> l.matches("INFO:blogPost: 7 rows, .* MB in .* s"));
+      assertThat(progress.logs).contains("INFO:Media: 4 files, 3 copied, 1 already stored, 0 missing", "INFO:Wrote checksums.sha256", "INFO:Wrote manifest.json", "INFO:Wrote files/index-00001.jsonl.gz");
+      assertThat(progress.logs).anyMatch(l -> l.startsWith("INFO:Wrote tables/blogPost/"));
+      for(String warning : result.getWarnings())
+      {
+         assertThat(progress.logs).contains("WARN:" + warning);
+      }
+      assertThat(String.join("\n", progress.logs)).doesNotContain("private-prefix", "do-not-leak", "Body of", "media/photo-");
+   }
+
+
+
+   /*******************************************************************************
+    ** Repeated backups report deduplicated media and complete all four phases.
+    *******************************************************************************/
+   @Test
+   void testProgressWithDeduplication() throws Exception
+   {
+      QRecord plan = plan(Map.of());
+      run(plan);
+      RecordingProgress progress = new RecordingProgress();
+      BackupRunResult result = new BackupEngine().execute(BackupRunFactory.forPlan(plan, "MANUAL").withProgress(progress));
+      assertThat(result.isUsable()).isTrue();
+      assertThat(progress.phases).containsExactly("prepare", "tables", "media", "upload");
+      assertThat(progress.getSnapshot().getPhases()).allMatch(p -> p.getStatus() == JobPhase.Status.DONE);
+      assertThat(progress.logs).contains("INFO:Media: 4 files, 0 copied, 4 already stored, 0 missing");
+   }
+
+
+
+   /*******************************************************************************
+    ** Arbitrary provider failures must not put credentials or row values in jobs.
+    *******************************************************************************/
+   @Test
+   void testProgressFailureDoesNotLeakProviderText() throws Exception
+   {
+      RecordingProgress progress = new RecordingProgress();
+      BackupRunRequest request = BackupRunFactory.forPlan(plan(Map.of()), "MANUAL").withProgress(progress)
+         .withReader((table, filter, start, size, transaction, sink) ->
+         {
+            throw (new com.kingsrook.qqq.backend.core.exceptions.QException("https://storage-user:storage-password@host/?secret=storage-secret accessKey=storage-access Detail: Key (email)=(private-row) already exists."));
+         });
+      BackupRunResult result = new BackupEngine().execute(request);
+      assertThat(result.getStatus()).isEqualTo("FAILED");
+      assertThat(progress.logs).contains("ERROR:" + result.getErrorMessage());
+      assertThat(progress.getSnapshot().getPhases()).anyMatch(p -> p.getStatus() == JobPhase.Status.FAILED);
+      assertThat(result.getErrorMessage()).startsWith("Backup failed: QException: https://[REDACTED]@host/");
+      assertThat(String.join("\n", progress.logs) + result.getErrorMessage() + result.getErrorDetail()).doesNotContain("storage-user", "storage-password", "storage-secret", "storage-access", "private-row");
+   }
+
+
+
+   /*******************************************************************************
+    ** The job page's cancel check must escape the engine to the durable worker.
+    *******************************************************************************/
+   @Test
+   void testJobCancellationBetweenPages() throws Exception
+   {
+      BackupRunRequest request = BackupRunFactory.forPlan(plan(Map.of("pageSize", 1)), "MANUAL");
+      int[] checks = { 0 };
+      try(JobChunkScope ignored = JobChunkScope.open(null, () ->
+      {
+         if(++checks[0] == 3)
+         {
+            throw (new JobInterruptedException(JobInterruptedException.Reason.CANCEL));
+         }
+      }))
+      {
+         assertThatThrownBy(() -> new BackupEngine().execute(request)).isInstanceOf(JobInterruptedException.class);
+      }
+      assertThat(new BackupCatalog().where(BackupsTables.RUN, "status", "CANCELLED")).hasSize(1);
+   }
+
+
+
+   /*******************************************************************************
+    ** Storage providers can include endpoint user-info and raw keys in exceptions.
+    *******************************************************************************/
+   @Test
+   void testStorageFailureDoesNotLeakCredentials() throws Exception
+   {
+      RecordingProgress progress = new RecordingProgress();
+      var storage = new com.kingsrook.qbits.backups.storage.InMemoryBackupStorage()
+      {
+         /*******************************************************************************
+          ** Deliberately unsafe provider diagnostics must stay out of job output.
+          *******************************************************************************/
+         @Override
+         public java.io.OutputStream openWrite(String key, com.kingsrook.qbits.backups.storage.WriteOptions options)
+         {
+            throw (new com.kingsrook.qbits.backups.storage.BackupStorageException(com.kingsrook.qbits.backups.storage.BackupStorageException.Reason.ACCESS_DENIED,
+               "https://endpoint-user:endpoint-password@storage accessKey=access-secret secretKey=secret-secret"));
+         }
+      };
+      BackupRunResult result = new BackupEngine().execute(BackupRunFactory.forPlan(plan(Map.of()), "MANUAL").withProgress(progress).withStorage(storage));
+      assertThat(result.getStatus()).isEqualTo("FAILED");
+      assertThat(progress.logs).contains("ERROR:" + result.getErrorMessage());
+      assertThat(String.join("\n", progress.logs) + result.getErrorMessage() + result.getErrorDetail()).doesNotContain("endpoint-user", "endpoint-password", "access-secret", "secret-secret");
+   }
+
+
+
+   /*******************************************************************************
+    ** A failed put records what an operator needs on the run row, with no secrets:
+    ** error_message is "Backup failed: Class: message" and error_detail has the
+    ** class, the cause chain and the root cause frames; the presigned signature,
+    ** the access key id and the URL user-info are redacted, the object key is kept.
+    *******************************************************************************/
+   @Test
+   void testFailedPutRecordsSanitizedMessageAndDetail() throws Exception
+   {
+      var storage = new com.kingsrook.qbits.backups.storage.InMemoryBackupStorage()
+      {
+         /*******************************************************************************
+          ** A MinIO-style failure with a presigned URL in the message.
+          *******************************************************************************/
+         @Override
+         public java.io.OutputStream openWrite(String key, com.kingsrook.qbits.backups.storage.WriteOptions options)
+         {
+            throw (new com.kingsrook.qbits.backups.storage.BackupStorageException(com.kingsrook.qbits.backups.storage.BackupStorageException.Reason.ACCESS_DENIED,
+               "AccessDenied on PUT https://minio-user:minio-pass@minio.galaxy.lan/kof22-backups/" + key
+                  + "?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20261016%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Signature=deadbeefcafe0123",
+               new IOException("XMinioStorageFull: Storage backend has reached its minimum free drive threshold. Authorization: AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/x, Signature=feedface")));
+         }
+      };
+      BackupRunResult result = new BackupEngine().execute(BackupRunFactory.forPlan(plan(Map.of()), "MANUAL").withStorage(storage));
+      assertThat(result.getStatus()).isEqualTo("FAILED");
+
+      QRecord run     = new BackupCatalog().get(BackupsTables.RUN, result.getRunId()).orElseThrow();
+      String  message = run.getValueString("errorMessage");
+      String  detail  = run.getValueString("errorDetail");
+      assertThat(message).isEqualTo(result.getErrorMessage());
+      assertThat(detail).isEqualTo(result.getErrorDetail());
+
+      assertThat(message).startsWith("Backup failed: BackupStorageException: AccessDenied on PUT https://[REDACTED]@minio.galaxy.lan/kof22-backups/qqq-backups/testapp/nightly/")
+         .doesNotContain("see server logs");
+      assertThat(detail).startsWith("com.kingsrook.qbits.backups.storage.BackupStorageException: AccessDenied on PUT")
+         .contains("\nCaused by: java.io.IOException: XMinioStorageFull: Storage backend has reached its minimum free drive threshold.")
+         .contains("X-Amz-Signature=[REDACTED]")
+         .contains("Root cause stack (top ")
+         .contains("\tat ");
+      assertThat(detail.length()).isLessThanOrEqualTo(FailureText.MAX_DETAIL_CHARS);
+      assertThat(message + detail).doesNotContain("deadbeefcafe0123", "AKIAIOSFODNN7EXAMPLE", "minio-user", "minio-pass", "feedface");
+   }
+
+
+
+   /*******************************************************************************
+    ** Failed backups must fail the durable chunk, not just add a summary line.
+    *******************************************************************************/
+   @Test
+   void testDurableLoadPropagatesFailure() throws Exception
+   {
+      var input = new com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput();
+      input.setRecords(List.of(plan(Map.of())));
+      BackupsTestUtils.BACKUPS.withFailWritesFor(key -> key.contains("/tables/"));
+      try(JobChunkScope scope = JobChunkScope.open(null).withProgress(new RecordingProgress()))
+      {
+         assertThatThrownBy(() -> new com.kingsrook.qbits.backups.processes.BackupsProcesses.RunPlanLoad().runOnePage(input,
+            new com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput()))
+            .isInstanceOf(com.kingsrook.qqq.backend.core.exceptions.QException.class).hasMessageContaining("Backup failed");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Record values at call time: JobProgress's live snapshot is mutable.
+    *******************************************************************************/
+   static class RecordingProgress extends JobProgress
+   {
+      final List<String> phases = new ArrayList<>();
+      final List<String> counts = new ArrayList<>();
+      final List<Long> bytes = new ArrayList<>();
+      final List<String> logs = new ArrayList<>();
+      final List<String> messages = new ArrayList<>();
+
+      /*******************************************************************************
+       ** Use the core's safe standalone destination.
+       *******************************************************************************/
+      RecordingProgress()
+      {
+         super((AsyncJobCallback) null);
+      }
+
+      /*******************************************************************************
+       ** Record transitions.
+       *******************************************************************************/
+      @Override
+      public JobProgress phase(String name)
+      {
+         phases.add(name);
+         return (super.phase(name));
+      }
+
+      /*******************************************************************************
+       ** Record counts and their units.
+       *******************************************************************************/
+      @Override
+      public JobProgress done(long done)
+      {
+         counts.add(getSnapshot().getCurrentPhase() + ":" + done + "/" + getSnapshot().getTotal() + " " + getSnapshot().getUnitLabel());
+         return (super.done(done));
+      }
+
+      /*******************************************************************************
+       ** Record byte counts.
+       *******************************************************************************/
+      @Override
+      public JobProgress bytesDone(long done)
+      {
+         bytes.add(done);
+         return (super.bytesDone(done));
+      }
+
+      /*******************************************************************************
+       ** Capture text before core redaction as well as exercising core publication.
+       *******************************************************************************/
+      @Override
+      public JobProgress log(JobLogLevel level, String message)
+      {
+         logs.add(level + ":" + message);
+         return (super.log(level, message));
+      }
+
+      /*******************************************************************************
+       ** Record live messages.
+       *******************************************************************************/
+      @Override
+      public JobProgress message(String message)
+      {
+         messages.add(message);
+         return (super.message(message));
+      }
+   }
+
 }

src/test/java/com/kingsrook/qbits/backups/run/BackupRunCoordinatorTest.java

@@ -24,10 +24,10 @@ import java.util.List;
 import java.util.Map;
 import com.kingsrook.qbits.backups.BackupsRuntime;
 import com.kingsrook.qbits.backups.BackupsTestUtils;
-import com.kingsrook.qbits.backups.LifecycleTestSupport;
 import com.kingsrook.qbits.backups.LifecycleTestSupport.EtlSummaries;
 import com.kingsrook.qbits.backups.LifecycleTestSupport.MutableClock;
 import com.kingsrook.qbits.backups.LifecycleTestSupport.RecordingAlertSink;
+import com.kingsrook.qbits.backups.LifecycleTestSupport;
 import com.kingsrook.qbits.backups.manifest.BackupManifest;
 import com.kingsrook.qbits.backups.model.BackupsTables;
 import com.kingsrook.qbits.backups.processes.CancelRunProcess;
@@ -296,7 +296,7 @@ class BackupRunCoordinatorTest
       QRecord         plan   = LifecycleTestSupport.plan(perTableTinyParts);
       BackupRunResult failed = new BackupRunCoordinator().runPlan(plan, "MANUAL", request -> request.withReader(failingAfter(request.getReader(), "blogPost", 4)));
       assertThat(failed.getStatus()).isEqualTo("FAILED");
-      assertThat(failed.getErrorMessage()).contains("injected failure in blogPost");
+      assertThat(failed.getErrorMessage()).contains("Backup failed");
       assertThat(BackupsTestUtils.BACKUPS.list(failed.getRunPrefix() + "manifest.json")).isEmpty();
 
       QRecord failedRow = catalog.get(BackupsTables.RUN, failed.getRunId()).orElseThrow();
@@ -384,4 +384,59 @@ class BackupRunCoordinatorTest
       assertThat(manifest.table("blogPost").parts()).hasSize(7);
       assertThat(catalog.where(BackupsTables.ARTIFACT, "backupRunId", failed.getRunId()).stream().filter(a -> "blogPost".equals(a.getValueString("tableName")))).hasSize(7);
    }
+
+
+   /*******************************************************************************
+    ** Explicit retry of a durable chunk must reuse its backup and completed parts.
+    *******************************************************************************/
+   @Test
+   void testDurableRetryResumesSameRun() throws Exception
+   {
+      QRecord plan = LifecycleTestSupport.plan(Map.of());
+      BackupEngineTest.RecordingProgress progress = new BackupEngineTest.RecordingProgress();
+      var identity = new com.kingsrook.qqq.backend.core.jobs.plans.JobChunkScope.ChunkIdentity(42, 8, 0, "backupRunPlan");
+      try(var scope = com.kingsrook.qqq.backend.core.jobs.plans.JobChunkScope.open(null, null, identity).withProgress(progress))
+      {
+         BackupsTestUtils.BACKUPS.withFailWritesFor(key -> key.contains("/tables/blogPost/"));
+         BackupRunResult failed = new BackupRunCoordinator().runPlan(plan, "MANUAL");
+         assertThat(failed.getStatus()).isEqualTo("FAILED");
+         BackupsTestUtils.BACKUPS.withFailWritesFor(key -> false);
+         // A worker death can leave RUNNING rather than FAILED; identity, not age,
+         // ties this recovery to the same durable job.
+         catalog.update(BackupsTables.RUN, failed.getRunId(), Map.of("status", "RUNNING"));
+         BackupRunResult resumed = new BackupRunCoordinator().runPlan(plan, "MANUAL");
+         assertThat(resumed.isUsable()).isTrue();
+         assertThat(resumed.getRunId()).isEqualTo(failed.getRunId());
+         assertThat(resumed.getRunPrefix()).isEqualTo(failed.getRunPrefix());
+         assertThat(progress.logs).contains("INFO:Resuming run " + failed.getRunId() + ": 1 tables already done");
+         assertThat(catalog.where(BackupsTables.RUN, "backupPlanId", plan.getValueInteger("id"))).hasSize(1);
+         int artifacts = catalog.where(BackupsTables.ARTIFACT, "backupRunId", failed.getRunId()).size();
+         BackupRunResult again = new BackupRunCoordinator().runPlan(plan, "MANUAL");
+         assertThat(again.getRunId()).isEqualTo(failed.getRunId());
+         assertThat(catalog.where(BackupsTables.ARTIFACT, "backupRunId", failed.getRunId())).hasSize(artifacts);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** A durable job refused for capacity must not escape into the inline queue.
+    *******************************************************************************/
+   @Test
+   void testDurableCapacityRefusalDoesNotQueue() throws Exception
+   {
+      QRecord occupied = LifecycleTestSupport.plan(Map.of());
+      liveRun(occupied.getValueInteger("id"), "RUNNING", clock.instant());
+      QRecord requested = LifecycleTestSupport.plan(Map.of("planKey", "second", "name", "Second"));
+      var identity = new com.kingsrook.qqq.backend.core.jobs.plans.JobChunkScope.ChunkIdentity(43, 9, 0, "backupRunPlan");
+      try(var scope = com.kingsrook.qqq.backend.core.jobs.plans.JobChunkScope.open(null, null, identity))
+      {
+         org.assertj.core.api.Assertions.assertThatThrownBy(() -> new BackupRunCoordinator().runPlan(requested, "MANUAL"))
+            .isInstanceOf(com.kingsrook.qqq.backend.core.exceptions.QException.class).hasMessageContaining("could not start");
+      }
+      assertThat(catalog.queuedRuns()).isEmpty();
+      assertThat(catalog.where(BackupsTables.RUN, "runKey", "job-43-plan-" + requested.getValueInteger("id")))
+         .singleElement().satisfies(run -> assertThat(run.getValueString("status")).isEqualTo("FAILED"));
+   }
+
 }

src/test/java/com/kingsrook/qbits/backups/run/FailureTextTest.java

@@ -0,0 +1,186 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.run;
+
+
+import java.io.IOException;
+import java.sql.SQLException;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** Unit tests of the failure text sanitizer, summary and detail.
+ *******************************************************************************/
+class FailureTextTest
+{
+
+   /*******************************************************************************
+    ** Null and plain text pass through; object keys and bucket names are kept.
+    *******************************************************************************/
+   @Test
+   void testPlainTextIsKept()
+   {
+      assertThat(FailureText.sanitize(null)).isNull();
+      String plain = "Query is not implemented in this module: PreviewDownloadBackend (bucket kof22-backups, key qqq-backups/kof22/nightly/2026/10/16/run/tables/a/part-00001.jsonl.gz)";
+      assertThat(FailureText.sanitize(plain)).isEqualTo(plain);
+      assertThat(FailureText.sanitize("objectKey=qqq-backups/x/y.gz bucket=kof22-backups")).isEqualTo("objectKey=qqq-backups/x/y.gz bucket=kof22-backups");
+   }
+
+
+
+   /*******************************************************************************
+    ** URL user-info and presigned query parameters are redacted; host and path kept.
+    *******************************************************************************/
+   @Test
+   void testUrls()
+   {
+      assertThat(FailureText.sanitize("PUT https://user:pass@minio.lan/bucket/a/b.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2Fx&X-Amz-Signature=abc123&partNumber=2 failed"))
+         .isEqualTo("PUT https://[REDACTED]@minio.lan/bucket/a/b.gz?X-Amz-Algorithm=[REDACTED]&X-Amz-Credential=[REDACTED]&X-Amz-Signature=[REDACTED]&partNumber=2 failed");
+      assertThat(FailureText.sanitize("jdbc:postgresql://app:hunter2@db:5432/app")).isEqualTo("jdbc:postgresql://[REDACTED]@db:5432/app");
+      assertThat(FailureText.sanitize("https://h/p?Signature=s1&Expires=1&AWSAccessKeyId=k&access_token=t&key=v"))
+         .isEqualTo("https://h/p?Signature=[REDACTED]&Expires=1&AWSAccessKeyId=[REDACTED]&access_token=[REDACTED]&key=[REDACTED]");
+   }
+
+
+
+   /*******************************************************************************
+    ** key=value and JSON secrets, Authorization and Bearer values.
+    *******************************************************************************/
+   @Test
+   void testAssignmentsAndHeaders()
+   {
+      assertThat(FailureText.sanitize("password=p1 secret=\"s 2\" token='t3' key=k4 accessKey=a5 secretKey=s6 apiKey=x7 db.password = p8, other=ok"))
+         .isEqualTo("password=[REDACTED] secret=[REDACTED] token=[REDACTED] key=[REDACTED] accessKey=[REDACTED] secretKey=[REDACTED] apiKey=[REDACTED] db.password = [REDACTED], other=ok");
+      assertThat(FailureText.sanitize("{\"clientSecret\": \"abc\", \"name\": \"keep\"}")).isEqualTo("{\"clientSecret\": \"[REDACTED]\", \"name\": \"keep\"}");
+      assertThat(FailureText.sanitize("Authorization: Bearer eyJhbGciOi.payload.sig\nnext line")).isEqualTo("Authorization: [REDACTED]\nnext line");
+      assertThat(FailureText.sanitize("sent Bearer abcdefghijk to the API")).isEqualTo("sent Bearer [REDACTED] to the API");
+      assertThat(FailureText.sanitize("id AKIAIOSFODNN7EXAMPLE and ASIAIOSFODNN7EXAMPLE")).isEqualTo("id [REDACTED] and [REDACTED]");
+   }
+
+
+
+   /*******************************************************************************
+    ** PEM blocks (terminated or not) and Postgres row values.
+    *******************************************************************************/
+   @Test
+   void testPemAndRowValues()
+   {
+      assertThat(FailureText.sanitize("bad key -----BEGIN PRIVATE KEY-----\nMIIabc\n-----END PRIVATE KEY----- here")).isEqualTo("bad key [REDACTED PEM] here");
+      assertThat(FailureText.sanitize("cut -----BEGIN RSA PRIVATE KEY-----\nMIIabc")).isEqualTo("cut [REDACTED PEM]");
+      assertThat(FailureText.sanitize("duplicate key value violates unique constraint \"u_email\" Key (email)=(ada@example.com) already exists."))
+         .isEqualTo("duplicate key value violates unique constraint \"u_email\" Key (email)=([REDACTED]) already exists.");
+      assertThat(FailureText.sanitize("ERROR: null value in column \"x\"\n  Detail: Failing row contains (1, ada, secret-row).\n  Where: x"))
+         .isEqualTo("ERROR: null value in column \"x\"\n  Detail: [REDACTED]\n  Where: x");
+   }
+
+
+
+   /*******************************************************************************
+    ** The summary is short, sanitized and bounded; no message falls back to the class.
+    *******************************************************************************/
+   @Test
+   void testSummary()
+   {
+      assertThat(FailureText.summary(new IllegalStateException("Query is not implemented in this module: PreviewDownloadBackend")))
+         .isEqualTo("Backup failed: IllegalStateException: Query is not implemented in this module: PreviewDownloadBackend");
+      assertThat(FailureText.summary(new NullPointerException())).isEqualTo("Backup failed: NullPointerException");
+      assertThat(FailureText.summary("Could not create the run record", new IOException(" password=x "))).isEqualTo("Could not create the run record: IOException: password=[REDACTED]");
+      assertThat(FailureText.summary(new IOException("x".repeat(2000)))).hasSize("Backup failed: IOException: ".length() + FailureText.MAX_SUMMARY_MESSAGE_CHARS + 3);
+   }
+
+
+
+   /*******************************************************************************
+    ** The detail has the class, the cause chain and the root cause frames, bounded.
+    *******************************************************************************/
+   @Test
+   void testDetail()
+   {
+      SQLException root   = new SQLException("Key (id)=(4242xyz) already exists.");
+      IOException  middle = new IOException("write failed password=pwsecret", root);
+      RuntimeException top = new RuntimeException("Backup of t failed", middle);
+      String detail = FailureText.detail(top);
+      assertThat(detail).startsWith("java.lang.RuntimeException: Backup of t failed\nCaused by: java.io.IOException: write failed password=[REDACTED]\nCaused by: java.sql.SQLException: Key (id)=([REDACTED]) already exists.\nRoot cause stack (top ");
+      assertThat(detail).contains("\tat com.kingsrook.qbits.backups.run.FailureTextTest.testDetail(FailureTextTest.java:");
+      assertThat(detail.split("\n\tat ")).hasSize(FailureText.ROOT_CAUSE_FRAMES + 1);
+      assertThat(detail).doesNotContain("4242xyz", "pwsecret");
+
+      IOException noFrames = new IOException((String) null);
+      noFrames.setStackTrace(new StackTraceElement[0]);
+      assertThat(FailureText.detail(noFrames)).isEqualTo("java.io.IOException");
+
+      String huge = FailureText.detail(new IOException("y".repeat(20_000)));
+      assertThat(huge).hasSize(FailureText.MAX_DETAIL_CHARS).endsWith("... (truncated)");
+   }
+
+
+
+   /*******************************************************************************
+    ** A cause cycle does not loop.
+    *******************************************************************************/
+   @Test
+   void testCauseCycle()
+   {
+      CyclicException a = new CyclicException("a");
+      CyclicException b = new CyclicException("b");
+      a.setCause(b);
+      b.setCause(a);
+      assertThat(FailureText.detail(a)).startsWith("com.kingsrook.qbits.backups.run.FailureTextTest$CyclicException: a\nCaused by: com.kingsrook.qbits.backups.run.FailureTextTest$CyclicException: b\nRoot cause stack");
+   }
+
+
+
+   /***************************************************************************
+    ** An exception whose cause can be set to form a cycle.
+    ***************************************************************************/
+   private static class CyclicException extends Exception
+   {
+      private Throwable cause;
+
+
+
+      /*******************************************************************************
+       ** Constructor.
+       *******************************************************************************/
+      CyclicException(String message)
+      {
+         super(message);
+      }
+
+
+
+      /*******************************************************************************
+       ** Set the cause.
+       *******************************************************************************/
+      void setCause(Throwable cause)
+      {
+         this.cause = cause;
+      }
+
+
+
+      /*******************************************************************************
+       ** The cause.
+       *******************************************************************************/
+      @Override
+      public synchronized Throwable getCause()
+      {
+         return (cause);
+      }
+   }
+}

src/test/java/com/kingsrook/qbits/backups/run/StorageOnlyBackendModule.java

@@ -14,83 +14,73 @@
  * limitations under the License.
  */
 
-package com.kingsrook.qbits.backups.alerts.fakeapi;
+package com.kingsrook.qbits.backups.run;
 
 
-import java.util.LinkedHashMap;
-import java.util.Map;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface;
+import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
+import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
+import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 
 
 /*******************************************************************************
- ** Test double of the Alerts producer SPI AlertSignal (only what the adapter uses).
+ ** A storage-only backend module (like qbit-edi's PreviewDownloadBackend): it
+ ** streams objects but has no query interface, so a table on it cannot be read.
  *******************************************************************************/
-public class AlertSignal
+public class StorageOnlyBackendModule implements QBackendModuleInterface, QStorageInterface
 {
-   public final String              alertTypeName;
-   public       String              dedupKey;
-   public       AlertSeverity       severity;
-   public       String              title;
-   public       String              summary;
-   public final Map<String, String> attributes = new LinkedHashMap<>();
+   public static final String TYPE = "test-storage-only";
 
-
-
-   /*******************************************************************************
-    ** Constructor.
-    *******************************************************************************/
-   public AlertSignal(String alertTypeName)
+   static
    {
-      this.alertTypeName = alertTypeName;
+      QBackendModuleDispatcher.registerBackendModule(new StorageOnlyBackendModule());
    }
 
 
 
    /*******************************************************************************
-    ** Setter.
+    ** The backend type.
     *******************************************************************************/
-   public void setDedupKey(String dedupKey)
+   @Override
+   public String getBackendType()
    {
-      this.dedupKey = dedupKey;
+      return (TYPE);
    }
 
 
 
    /*******************************************************************************
-    ** Setter.
+    ** Storage only.
     *******************************************************************************/
-   public void setSeverity(AlertSeverity severity)
+   @Override
+   public QStorageInterface getStorageInterface()
    {
-      this.severity = severity;
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter.
-    *******************************************************************************/
-   public void setTitle(String title)
-   {
-      this.title = title;
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter.
-    *******************************************************************************/
-   public void setSummary(String summary)
-   {
-      this.summary = summary;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent attribute.
-    *******************************************************************************/
-   public AlertSignal withAttribute(String key, String value)
-   {
-      attributes.put(key, value);
       return (this);
    }
+
+
+
+   /*******************************************************************************
+    ** Nothing is stored.
+    *******************************************************************************/
+   @Override
+   public InputStream getInputStream(StorageInput input)
+   {
+      return (new ByteArrayInputStream(new byte[0]));
+   }
+
+
+
+   /*******************************************************************************
+    ** Writes are discarded.
+    *******************************************************************************/
+   @Override
+   public OutputStream createOutputStream(StorageInput input)
+   {
+      return (new ByteArrayOutputStream());
+   }
 }

src/test/java/com/kingsrook/qbits/backups/run/TableReaderTest.java

@@ -0,0 +1,163 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.run;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qbits.backups.BackupsTestUtils;
+import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.model.actions.consistentread.ConsistentReadInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+
+/*******************************************************************************
+ ** Integration coverage for the core pager adapter's ordering, bounded fetches,
+ ** raw fields and transaction lifetime.
+ *******************************************************************************/
+class TableReaderTest
+{
+   private QTableMetaData table;
+
+
+
+   /*******************************************************************************
+    ** Seed more records than fit in a page.
+    *******************************************************************************/
+   @BeforeEach
+   void setUp() throws Exception
+   {
+      BackupsTestUtils.init();
+      BackupsTestUtils.loadBlog(7);
+      table = QContext.getQInstance().getTable("blogPost");
+      table.getField("body").setIsHeavy(true);
+      table.getField("secretNote").setIsHidden(true);
+      table.withCustomizer(TableCustomizers.POST_QUERY_RECORD, new QCodeReference(CaptureQueries.class));
+      CaptureQueries.inputs.clear();
+   }
+
+
+
+   /*******************************************************************************
+    ** An OR filter and a resume cursor are ANDed, with ascending keys even when
+    ** the caller requests descending order. Every query has the requested bound.
+    *******************************************************************************/
+   @Test
+   void testFilterResumeOrderingAndRawFields() throws Exception
+   {
+      String filter = "{\"booleanOperator\":\"OR\",\"criteria\":[{\"fieldName\":\"id\",\"operator\":\"LESS_THAN_OR_EQUALS\",\"values\":[3]},{\"fieldName\":\"id\",\"operator\":\"GREATER_THAN\",\"values\":[5]}],\"orderBys\":[{\"fieldName\":\"id\",\"isAscending\":false}]}";
+      List<QRecord> rows = new ArrayList<>();
+      new TableReader.Keyset().read(table, filter, 2, 2, null, rows::add);
+      assertThat(rows).extracting(r -> r.getValueInteger("id")).containsExactly(3, 6, 7);
+      assertThat(rows.get(0).getValueString("body")).isEqualTo("Body of 3, with a comma");
+      assertThat(rows.get(0).getValueString("secretNote")).isEqualTo("do-not-leak-3");
+      assertThat(CaptureQueries.inputs).hasSize(2).allSatisfy(input ->
+      {
+         assertThat(input.getFilter().getLimit()).isEqualTo(2);
+         assertThat(input.getFilter().getSkip()).isNull();
+         assertThat(input.getShouldOmitHiddenFields()).isFalse();
+         assertThat(input.getShouldMaskPasswords()).isFalse();
+         assertThat(input.getShouldFetchHeavyFields()).isTrue();
+         assertThat(input.getShouldTranslatePossibleValues()).isFalse();
+         assertThat(input.getShouldGenerateDisplayValues()).isFalse();
+      });
+   }
+
+
+
+   /*******************************************************************************
+    ** Changes made after a snapshot was opened are invisible on every page. The
+    ** reader leaves the transaction open, with its commit callbacks unexecuted.
+    *******************************************************************************/
+   @Test
+   void testConsistentSnapshotIsRetainedAcrossPages() throws Exception
+   {
+      List<Boolean> commits = new ArrayList<>();
+      try(QBackendTransaction transaction = QBackendTransaction.openConsistentReadFor(new ConsistentReadInput().withBackendName(table.getBackendName())))
+      {
+         transaction.addAfterCommitCallback(() -> commits.add(true));
+         new UpdateAction().execute(new UpdateInput("blogPost").withRecords(List.of(new QRecord().withValue("id", 7).withValue("title", "Changed after snapshot"))));
+         CaptureQueries.inputs.clear();
+         List<QRecord> rows = new ArrayList<>();
+         new TableReader.Keyset().read(table, null, null, 2, transaction, rows::add);
+         assertThat(rows).extracting(r -> r.getValueInteger("id")).containsExactly(1, 2, 3, 4, 5, 6, 7);
+         assertThat(rows.get(6).getValueString("title")).isEqualTo("Post 7");
+         assertThat(CaptureQueries.inputs).hasSize(4).allSatisfy(input -> assertThat(input.getTransaction()).isSameAs(transaction));
+         assertThat(commits).isEmpty();
+         transaction.commit();
+         assertThat(commits).containsExactly(true);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Exact page multiples, empty scans, invalid bounds and unsupported key types
+    ** must terminate or fail explicitly rather than drop rows or stream unordered.
+    *******************************************************************************/
+   @Test
+   void testBoundariesAndInvalidPaging() throws Exception
+   {
+      List<QRecord> rows = new ArrayList<>();
+      new TableReader.Keyset().read(table, null, 3, 2, null, rows::add);
+      assertThat(rows).extracting(r -> r.getValueInteger("id")).containsExactly(4, 5, 6, 7);
+      rows.clear();
+      new TableReader.Keyset().read(table, null, 7, 2, null, rows::add);
+      assertThat(rows).isEmpty();
+      assertThatThrownBy(() -> new TableReader.Keyset().read(table, null, null, 0, null, rows::add)).hasMessageContaining("Page size");
+      assertThatThrownBy(() -> new TableReader.Keyset().read(table, "{\"limit\":1}", null, 2, null, rows::add)).hasMessageContaining("skip or limit");
+      assertThatThrownBy(() -> new TableReader.Keyset().read(table, "{", null, 2, null, rows::add)).hasMessageContaining("scope filter");
+      table.getField("id").setType(QFieldType.BOOLEAN);
+      assertThatThrownBy(() -> new TableReader.Keyset().read(table, null, null, 2, null, rows::add)).hasMessageContaining("orderable primary key");
+   }
+
+
+
+   /*******************************************************************************
+    ** Observe actual QueryAction inputs without replacing the backend or pager.
+    *******************************************************************************/
+   public static class CaptureQueries implements TableCustomizerInterface
+   {
+      static final List<QueryInput> inputs = new ArrayList<>();
+
+
+
+      /*******************************************************************************
+       ** Record each page and preserve the result unchanged.
+       *******************************************************************************/
+      @Override
+      public List<QRecord> postQuery(QueryOrGetInputInterface input, List<QRecord> records)
+      {
+         inputs.add((QueryInput) input);
+         return (records);
+      }
+   }
+}

src/test/java/com/kingsrook/qbits/backups/schedule/BackupPlanScheduleTest.java

@@ -61,6 +61,7 @@ import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
 import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager;
+import com.kingsrook.qqq.backend.core.testing.HelpBundleAssertions;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -306,7 +307,7 @@ class BackupPlanScheduleTest
       assertThat(catalog.where(BackupsTables.RUN, "backupPlanId", planId)).isEmpty();
       assertThat(BackupsTestUtils.BACKUPS.list("")).isEmpty();
       assertThat(catalog.get(BackupsTables.PLAN, planId).orElseThrow().getValueString("slaStatus")).isEqualTo("OWNER_INVALID");
-      assertThat(alerts.raised).containsExactly("backups.plan.owner_invalid|backups.plan.owner_invalid:nightly");
+      assertThat(alerts.raised).containsExactly("backups.plan.owner-invalid|backups.plan.owner-invalid:nightly");
 
       BackupScheduledRunProcess.setOwnerHasAdminCheck(null);
       Integer noOwner = insertPlan(Map.of("name", "Ownerless", "planKey", "ownerless", "isActive", true));
@@ -320,4 +321,22 @@ class BackupPlanScheduleTest
       assertThat(inactive.getValues().get("status")).isEqualTo("INACTIVE");
       assertThat(catalog.where(BackupsTables.RUN, "backupPlanId", planId)).isEmpty();
    }
+
+
+   /*******************************************************************************
+    ** Cover every qbit target, including optional scheduler integration processes.
+    *******************************************************************************/
+   @Test
+   void helpBundleIsComplete()
+   {
+      QInstance qInstance = QContext.getQInstance();
+      HelpBundleAssertions.forInstance(qInstance)
+         .bundles("help/qbit-backups.yaml")
+         // Core owns the generated bulk CRUD actions; this qbit owns its export actions on host tables.
+         .owning(target -> target.owner() instanceof com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData process
+            ? qInstance.getQBits().containsKey(process.getSourceQBitName())
+            : target.name().startsWith("backups"))
+         .assertComplete();
+   }
+
 }

src/test/java/com/kingsrook/qbits/backups/sla/SlaCheckerTest.java

@@ -180,12 +180,12 @@ class SlaCheckerTest
 
       clock.advance(Duration.ofMinutes(55));
       assertThat(check(plan).status()).isEqualTo(SlaChecker.MISSING_IN_STORAGE);
-      BackupAlertsAssert.assertOpen("backups.run.missing_in_storage:nightly", BackupAlertSink.Severity.CRITICAL);
+      BackupAlertsAssert.assertOpen("backups.run.missing-in-storage:nightly", BackupAlertSink.Severity.CRITICAL);
       assertThat(check(plan).status()).as("a negative answer is never cached").isEqualTo(SlaChecker.MISSING_IN_STORAGE);
 
       BackupsTestUtils.BACKUPS.putBytes(result.getManifestKey(), bytes);
       assertThat(check(plan).status()).isEqualTo(SlaChecker.OK);
-      assertThat(sink.isOpen("backups.run.missing_in_storage:nightly")).isFalse();
+      assertThat(sink.isOpen("backups.run.missing-in-storage:nightly")).isFalse();
    }
 
 
@@ -260,11 +260,11 @@ class SlaCheckerTest
 
       BackupsRuntime.getConfig().withPlanOwnerValidator(owner -> false);
       assertThat(check(plan).status()).isEqualTo(SlaChecker.OWNER_INVALID);
-      BackupAlertsAssert.assertOpen("backups.plan.owner_invalid:nightly", BackupAlertSink.Severity.WARNING);
+      BackupAlertsAssert.assertOpen("backups.plan.owner-invalid:nightly", BackupAlertSink.Severity.WARNING);
 
       BackupsRuntime.getConfig().withPlanOwnerValidator(owner -> "bob".equals(owner));
       assertThat(check(plan).status()).isEqualTo(SlaChecker.OK);
-      assertThat(sink.isOpen("backups.plan.owner_invalid:nightly")).isFalse();
+      assertThat(sink.isOpen("backups.plan.owner-invalid:nightly")).isFalse();
 
       BackupsRuntime.getConfig().withPlanOwnerValidator(owner ->
       {
@@ -287,7 +287,7 @@ class SlaCheckerTest
       clock.advance(Duration.ofDays(2));
       assertThat(check(plan).status()).isEqualTo(SlaChecker.MISSING_IN_STORAGE);
       assertThat(sink.isOpen("backups.plan.missed:nightly")).as("the alerts are independent of the status shown").isTrue();
-      assertThat(sink.isOpen("backups.run.missing_in_storage:nightly")).isTrue();
+      assertThat(sink.isOpen("backups.run.missing-in-storage:nightly")).isTrue();
    }
 
 

src/test/java/com/kingsrook/qbits/backups/tables/BackupsTablesGuardrailTest.java

@@ -0,0 +1,200 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.tables;
+
+
+import java.lang.reflect.Field;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
+import com.kingsrook.qbits.backups.BackupsTestUtils;
+import com.kingsrook.qbits.backups.model.BackupsTables;
+import com.kingsrook.qbits.backups.processes.DefaultPlanProcess;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
+import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundles;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** Guardrail (owner requirement 2026-10-14: "all tables should be populated and
+ ** the qbit should be fully using them"). For every backups_* table:
+ ** - runtime code (not just its model, metadata or home screen) reads or writes it,
+ **   so no table exists only on paper;
+ ** - it is listed here with why it starts empty and, where a user sets it up, how
+ **   (this qbit ships no DDL and its one setup value, the storage bucket alias, is
+ **   the host's, so nothing is seeded by a changelog);
+ ** - it has bundle help that says what it is for and how its rows get created
+ **   (shown on the table screen and when it is empty).
+ ** A new table fails this test until it is wired and explained.
+ *******************************************************************************/
+class BackupsTablesGuardrailTest
+{
+   private static final Path MAIN = Path.of("src", "main", "java", "com", "kingsrook", "qbits", "backups");
+
+   /*******************************************************************************
+    ** Why each table legitimately starts empty, and where its rows come from.
+    *******************************************************************************/
+   static final Map<String, String> STARTS_EMPTY = Map.ofEntries(
+      Map.entry(BackupsTables.PLAN, "host-specific: its storage bucket alias names a bucket the host provides (the Create default plan action adds Nightly full for an alias)"),
+      Map.entry(BackupsTables.PLAN_SCOPE, "user content: include/exclude rules; an INSTANCE plan needs none"),
+      Map.entry(BackupsTables.RUN, "runtime: a plan runs, or Import backups from storage"),
+      Map.entry(BackupsTables.ARTIFACT, "runtime: every object a run stores"),
+      Map.entry(BackupsTables.VERIFICATION, "runtime: Verify backup, or a plan verify schedule"),
+      Map.entry(BackupsTables.RESTORE_RUN, "runtime: a restore"),
+      Map.entry(BackupsTables.RESTORE_RUN_TABLE, "runtime: each table of a restore"),
+      Map.entry(BackupsTables.FILE_OBJECT, "runtime: each distinct file a plan copies"),
+      Map.entry(BackupsTables.ID_MAP, "runtime: working data of a restore with new ids, cleared when it ends"));
+
+   private QInstance qInstance;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   void setUp() throws Exception
+   {
+      qInstance = BackupsTestUtils.init();
+   }
+
+
+
+   /*******************************************************************************
+    ** Every table is used by runtime code.
+    *******************************************************************************/
+   @Test
+   void everyTableIsUsedAtRuntime() throws Exception
+   {
+      List<String> runtimeSource = new ArrayList<>();
+      try(Stream<Path> files = Files.walk(MAIN))
+      {
+         for(Path file : files.filter(p -> p.toString().endsWith(".java")).filter(p -> !p.startsWith(MAIN.resolve("model")) && !p.startsWith(MAIN.resolve("app"))).toList())
+         {
+            runtimeSource.add(Files.readString(file, StandardCharsets.UTF_8));
+         }
+      }
+      for(String table : backupsTables())
+      {
+         String  constant = constantFor(table);
+         Pattern use      = Pattern.compile("\\bBackupsTables\\." + constant + "\\b");
+         assertThat(runtimeSource.stream().anyMatch(source -> use.matcher(source).find()))
+            .as(table + " (BackupsTables." + constant + ") must be read or written by runtime code, not only declared").isTrue();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Every table is explained: where its rows come from.
+    *******************************************************************************/
+   @Test
+   void everyTableIsExplained()
+   {
+      assertThat(backupsTables()).containsExactlyInAnyOrderElementsOf(STARTS_EMPTY.keySet());
+      assertThat(BackupsTables.ALL).containsExactlyInAnyOrderElementsOf(STARTS_EMPTY.keySet());
+      assertThat(STARTS_EMPTY.values()).allMatch(reason -> !reason.isBlank());
+   }
+
+
+
+   /*******************************************************************************
+    ** Every table and the app have bundle help that says how rows get created, and
+    ** the plan help names the action that sets it up.
+    *******************************************************************************/
+   @Test
+   void everyTableHasEmptyStateHelp() throws Exception
+   {
+      var bundle = HelpBundles.of(qInstance).getBundles().get(0);
+      for(String table : backupsTables())
+      {
+         assertThat(bundle.getEntries()).filteredOn(entry -> entry.getKey().equals("table:" + table + ";slot:help"))
+            .singleElement().satisfies(entry -> assertThat(entry.getContent()).contains("How rows get created"));
+      }
+      assertThat(bundle.getEntries()).filteredOn(entry -> entry.getKey().equals("app:backups;slot:help")).hasSize(1);
+      assertThat(bundle.getEntries()).filteredOn(entry -> entry.getKey().equals("table:backups_plan;slot:help"))
+         .singleElement().satisfies(entry -> assertThat(entry.getContent()).contains("Create default plan"));
+      // RC.7 validates supplied widget/value-source entries but does not require their coverage.
+      for(String name : qInstance.getWidgets().keySet().stream().filter(name -> name.startsWith("backups")).toList())
+      {
+         assertThat(bundle.getEntries()).anyMatch(entry -> entry.getKey().startsWith("widget:" + name + ";slot:"));
+      }
+      for(String name : qInstance.getPossibleValueSources().keySet().stream().filter(name -> name.startsWith("backups_")).toList())
+      {
+         assertThat(bundle.getEntries()).anyMatch(entry -> entry.getKey().startsWith("possibleValueSource:" + name + ";slot:"));
+      }
+      assertThat(qInstance.getProcess(DefaultPlanProcess.NAME)).as("the action the plan help names").isNotNull();
+      assertThat(qInstance.getProcess(DefaultPlanProcess.NAME).getLabel()).isEqualTo("Create default plan");
+   }
+
+
+
+   /*******************************************************************************
+    ** A host's own help in the slot is never replaced by the bundle.
+    *******************************************************************************/
+   @Test
+   void hostHelpIsKept() throws Exception
+   {
+      QTableMetaData table = qInstance.getTable(BackupsTables.RUN);
+      table.withHelpContent("help", new QHelpContent("Host help for runs."));
+      qInstance.getApp("backups").withHelpContent("help", new QHelpContent("Host help for the app."));
+      HelpBundles.register(qInstance, "help/qbit-backups.yaml");
+      assertThat(qInstance.getTable("helpContent")).isNull();
+      HelpBundles.syncOnStartup(qInstance);
+      assertThat(table.getHelpContent().get("help")).singleElement().satisfies(h -> assertThat(h.getContent()).isEqualTo("Host help for runs."));
+      assertThat(qInstance.getApp("backups").getHelpContent().get("help")).singleElement().satisfies(h -> assertThat(h.getContent()).isEqualTo("Host help for the app."));
+      assertThat(HelpBundles.of(qInstance).getBundles()).isNotEmpty();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private List<String> backupsTables()
+   {
+      List<String> tables = qInstance.getTables().values().stream().map(QTableMetaData::getName).filter(n -> n.startsWith("backups_")).sorted().toList();
+      assertThat(tables).hasSize(9);
+      return (tables);
+   }
+
+
+
+   /*******************************************************************************
+    ** The BackupsTables constant whose value is the table name.
+    *******************************************************************************/
+   private static String constantFor(String table) throws Exception
+   {
+      for(Field field : BackupsTables.class.getFields())
+      {
+         if(field.getType() == String.class && table.equals(field.get(null)) && !field.getName().equals("PVS_PREFIX"))
+         {
+            return (field.getName());
+         }
+      }
+      throw (new IllegalStateException("no constant for " + table));
+   }
+}

src/test/java/com/kingsrook/qbits/backups/tables/DefaultPlanProcessTest.java

@@ -0,0 +1,205 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.tables;
+
+
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qbits.backups.BackupsTestUtils;
+import com.kingsrook.qbits.backups.model.BackupsTables;
+import com.kingsrook.qbits.backups.permissions.BackupsPermissions;
+import com.kingsrook.qbits.backups.plan.BackupValidatorReset;
+import com.kingsrook.qbits.backups.processes.DefaultPlanProcess;
+import com.kingsrook.qbits.backups.run.BackupCatalog;
+import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** The "Create default plan" table action: it asks for a storage bucket alias,
+ ** creates the Nightly full plan through the normal insert, refuses an alias that
+ ** does not resolve, and is insert-only (a second run changes nothing, an edited
+ ** plan survives).
+ *******************************************************************************/
+class DefaultPlanProcessTest
+{
+   private final BackupCatalog catalog = new BackupCatalog();
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   void setUp() throws Exception
+   {
+      BackupsTestUtils.init();
+      BackupValidatorReset.reset();
+      QContext.getQSession().withPermissions(BackupsPermissions.ADMIN);
+   }
+
+
+
+   /*******************************************************************************
+    ** Run the action through Review and then Result.
+    *******************************************************************************/
+   @SuppressWarnings("unchecked")
+   private List<ProcessSummaryLine> run(String alias, boolean execute) throws QException
+   {
+      RunProcessInput input = new RunProcessInput();
+      input.setProcessName(DefaultPlanProcess.NAME);
+      input.addValue(DefaultPlanProcess.FIELD_ALIAS, alias);
+      //////////////////////////////////////////////////////////////////////////
+      // the first call stops at the input form; the next runs Review (full     //
+      // validation), the last executes and gives the Result                    //
+      //////////////////////////////////////////////////////////////////////////
+      RunProcessOutput output = new RunProcessAction().execute(input);
+      input.addValue("processUUID", output.getProcessUUID());
+      input.setStartAfterStep("input");
+      output = new RunProcessAction().execute(input);
+      if(!execute)
+      {
+         return ((List<ProcessSummaryLine>) output.getValues().get(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY));
+      }
+      input.setStartAfterStep("review");
+      output = new RunProcessAction().execute(input);
+      return ((List<ProcessSummaryLine>) output.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY));
+   }
+
+
+
+   /*******************************************************************************
+    ** The action is a table action that needs no selected records and backups.admin.
+    *******************************************************************************/
+   @Test
+   void isATableActionForAdmins() throws Exception
+   {
+      var process = QContext.getQInstance().getProcess(DefaultPlanProcess.NAME);
+      assertThat(process.getTableName()).isEqualTo(BackupsTables.PLAN);
+      assertThat(process.getMinInputRecords()).isEqualTo(0);
+      assertThat(((com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData) process.getStep("input")).getFormFields().stream().map(f -> f.getName())).containsExactly(DefaultPlanProcess.FIELD_ALIAS);
+   }
+
+
+
+   /*******************************************************************************
+    ** Review previews without writing; Result creates the plan, and the plan works.
+    *******************************************************************************/
+   @Test
+   void createsTheNightlyFullPlan() throws Exception
+   {
+      List<ProcessSummaryLine> review = run("backups", false);
+      assertThat(review).anySatisfy(line -> assertThat(line.getPluralFutureMessage()).contains("will be created"));
+      assertThat(catalog.query(BackupsTables.PLAN, new com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter())).as("Review writes nothing").isEmpty();
+
+      List<ProcessSummaryLine> result = run("backups", true);
+      assertThat(result).singleElement().satisfies(line -> assertThat(line.getStatus().name()).isEqualTo("OK"));
+
+      QRecord plan = catalog.where(BackupsTables.PLAN, "planKey", DefaultPlanProcess.PLAN_KEY).get(0);
+      assertThat(plan.getValueString("name")).isEqualTo("Nightly full");
+      assertThat(plan.getValueString("storageBucketAlias")).isEqualTo("backups");
+      assertThat(plan.getValueBoolean("isActive")).isTrue();
+      assertThat(plan.getValueString("scopeType")).isEqualTo("INSTANCE");
+      assertThat(plan.getValueString("cronExpression")).isEqualTo(DefaultPlanProcess.CRON);
+      assertThat(plan.getValueString("verifyCronExpression")).isEqualTo(DefaultPlanProcess.VERIFY_CRON);
+      assertThat(plan.getValueInteger("slaHours")).isEqualTo(26);
+      assertThat(plan.getValueString("format")).isEqualTo("JSONL");
+   }
+
+
+
+   /*******************************************************************************
+    ** A plan made by the action is a working plan: it runs and backs up the instance.
+    *******************************************************************************/
+   @Test
+   void theCreatedPlanRuns() throws Exception
+   {
+      BackupsTestUtils.loadBlog(3);
+      run("backups", true);
+      QRecord plan = catalog.where(BackupsTables.PLAN, "planKey", DefaultPlanProcess.PLAN_KEY).get(0);
+      var result = new com.kingsrook.qbits.backups.run.BackupRunCoordinator().runPlan(plan, "MANUAL");
+      assertThat(result.getStatus()).isIn("SUCCEEDED", "SUCCEEDED_WITH_WARNINGS");
+   }
+
+
+
+   /*******************************************************************************
+    ** Insert-only: a second run creates nothing, and an edited plan keeps its edit.
+    *******************************************************************************/
+   @Test
+   void isInsertOnly() throws Exception
+   {
+      run("backups", true);
+      Integer id = catalog.where(BackupsTables.PLAN, "planKey", DefaultPlanProcess.PLAN_KEY).get(0).getValueInteger("id");
+      catalog.update(BackupsTables.PLAN, id, Map.of("slaHours", 48, "name", "Our nightly"));
+
+      List<ProcessSummaryLine> review = run("backups", false);
+      assertThat(review).singleElement().satisfies(line -> assertThat(line.getMessage()).contains("already exists"));
+      run("backups", true);
+
+      assertThat(catalog.query(BackupsTables.PLAN, new com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter())).hasSize(1);
+      QRecord plan = catalog.get(BackupsTables.PLAN, id).orElseThrow();
+      assertThat(plan.getValueInteger("slaHours")).isEqualTo(48);
+      assertThat(plan.getValueString("name")).isEqualTo("Our nightly");
+   }
+
+
+
+   /*******************************************************************************
+    ** An alias the host does not provide is refused in Review, and nothing is created.
+    *******************************************************************************/
+   @Test
+   void refusesAnUnknownAlias() throws Exception
+   {
+      List<ProcessSummaryLine> review = run("no-such-bucket", false);
+      assertThat(review).anySatisfy(line ->
+      {
+         assertThat(line.getStatus().name()).isEqualTo("ERROR");
+         assertThat(String.join(" ", line.getBulletsOfText())).contains("no-such-bucket");
+      });
+      run("no-such-bucket", true);
+      assertThat(catalog.query(BackupsTables.PLAN, new com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter())).isEmpty();
+   }
+
+
+
+   /*******************************************************************************
+    ** A plan that already has the name Nightly full (under another key) is refused in
+    ** Review, never "will be created".
+    *******************************************************************************/
+   @Test
+   void reviewRefusesATakenName() throws Exception
+   {
+      catalog.insert(BackupsTables.PLAN, new java.util.LinkedHashMap<>(Map.of("name", DefaultPlanProcess.PLAN_NAME, "planKey", "other-key", "isActive", false, "scopeType", "INSTANCE", "storageBucketAlias", "backups")));
+      List<ProcessSummaryLine> review = run("backups", false);
+      assertThat(review).noneSatisfy(line -> assertThat(line.getPluralFutureMessage()).contains("will be created"));
+      assertThat(review).anySatisfy(line ->
+      {
+         assertThat(line.getStatus().name()).isEqualTo("ERROR");
+         assertThat(String.join(" ", line.getBulletsOfText())).contains("already exists");
+      });
+   }
+}

src/test/java/com/kingsrook/qbits/backups/tables/UnavailableSettingsAndScheduledVerifyTest.java

@@ -0,0 +1,188 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.tables;
+
+
+import java.io.Serializable;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qbits.backups.BackupsTestUtils;
+import com.kingsrook.qbits.backups.model.BackupsTables;
+import com.kingsrook.qbits.backups.plan.BackupValidatorReset;
+import com.kingsrook.qbits.backups.run.BackupCatalog;
+import com.kingsrook.qbits.backups.run.BackupRunCoordinator;
+import com.kingsrook.qbits.backups.schedule.BackupPlanVerifySyncToScheduledJobProcess;
+import com.kingsrook.qbits.backups.schedule.BackupScheduledVerifyProcess;
+import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory;
+import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** Two half-wired settings of backups_plan, made honest or made to work:
+ ** - the settings that are not available yet (read replica, native dump,
+ **   incremental, PARQUET, ZSTD) are refused at save with a clear message and say so
+ **   in their field help;
+ ** - verifyCronExpression creates a scheduled job that runs the process
+ **   backupVerifyPlan; that process now exists, and verifies the plan's latest
+ **   successful backup through the plan's verifyLevel.
+ *******************************************************************************/
+class UnavailableSettingsAndScheduledVerifyTest
+{
+   private final BackupCatalog catalog = new BackupCatalog();
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   void setUp() throws Exception
+   {
+      BackupsTestUtils.init();
+      BackupsTestUtils.loadBlog(3);
+      BackupValidatorReset.reset();
+   }
+
+
+
+   /*******************************************************************************
+    ** Save a plan through the table (validation runs), returning the record.
+    *******************************************************************************/
+   private QRecord save(Map<String, Serializable> overrides) throws QException
+   {
+      Map<String, Serializable> values = new LinkedHashMap<>(Map.of("name", "P" + System.nanoTime(), "planKey", "p" + System.nanoTime(), "isActive", true, "scopeType", "INSTANCE", "storageBucketAlias", "backups"));
+      values.putAll(overrides);
+      QRecord record = new QRecord();
+      record.setValues(values);
+      InsertInput input = new InsertInput(BackupsTables.PLAN);
+      input.setRecords(List.of(record));
+      input.withPermissionBypass("test");
+      return (new InsertAction().execute(input).getRecords().get(0));
+   }
+
+
+
+   /*******************************************************************************
+    ** Every setting that is not available yet is refused, and the field help says so.
+    *******************************************************************************/
+   @Test
+   void unavailableSettingsAreRefusedAndSayWhy() throws Exception
+   {
+      assertThat(save(Map.of()).getErrors()).as("the defaults are fine").isEmpty();
+
+      Map<String, Map<String, Serializable>> refused = Map.of(
+         "Reading from a replica is not available yet", Map.of("useReadReplica", true),
+         "Native database dumps are not available yet", Map.of("nativeDump", "PG_DUMP_CUSTOM"),
+         "Incremental backups are not available yet", Map.of("incrementalMode", "CHANGED_SINCE_LAST"),
+         "PARQUET format is not available yet", Map.of("format", "PARQUET"),
+         "ZSTD compression is not available yet", Map.of("compression", "ZSTD"));
+      for(Map.Entry<String, Map<String, Serializable>> entry : refused.entrySet())
+      {
+         QRecord record = save(entry.getValue());
+         assertThat(record.getErrors()).as(entry.getValue().toString()).isNotEmpty();
+         assertThat(record.getErrors().toString()).contains(entry.getKey());
+      }
+
+      var table = QContext.getQInstance().getTable(BackupsTables.PLAN);
+      for(String field : List.of("useReadReplica", "nativeDump", "incrementalMode", "format", "compression", "scheduledJobId"))
+      {
+         assertThat(table.getField(field).getHelpContents()).as(field + " help").isNotEmpty();
+         assertThat(table.getField(field).getHelpContents().get(0).getContent()).as(field + " help").matches("(?is).*(not available yet|not used).*");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The process a verify cron schedules exists, and verifies the plan's latest
+    ** successful run, as the scheduler, through the plan's verifyLevel.
+    *******************************************************************************/
+   @Test
+   void theVerifyCronRunsARealProcess() throws Exception
+   {
+      assertThat(QContext.getQInstance().getProcess(BackupPlanVerifySyncToScheduledJobProcess.VERIFY_PROCESS_NAME)).as("the process the scheduled verify job names").isNotNull();
+
+      Integer planId = catalog.insert(BackupsTables.PLAN, new LinkedHashMap<>(Map.of("name", "Nightly", "planKey", "nightly", "isActive", true, "scopeType", "INSTANCE",
+         "storageBucketAlias", "backups", "verifyLevel", "STORAGE")));
+      assertThat(run(planId).getValues().get("status")).as("nothing to verify before the first backup").isEqualTo("NO_BACKUP");
+
+      QRecord plan   = catalog.get(BackupsTables.PLAN, planId).orElseThrow();
+      var     result = new BackupRunCoordinator().runPlan(plan, "SCHEDULED");
+      assertThat(result.getStatus()).isIn("SUCCEEDED", "SUCCEEDED_WITH_WARNINGS");
+
+      RunProcessOutput output = run(planId);
+      assertThat(output.getException()).isEmpty();
+      assertThat(output.getValues().get("status")).isEqualTo("PASSED");
+      assertThat(output.getValues().get("level")).isEqualTo("STORAGE");
+
+      List<QRecord> verifications = catalog.where(BackupsTables.VERIFICATION, "backupRunId", result.getRunId());
+      assertThat(verifications).singleElement().satisfies(v ->
+      {
+         assertThat(v.getValueString("level")).isEqualTo("STORAGE");
+         assertThat(v.getValueString("trigger")).isEqualTo("SCHEDULED");
+         assertThat(v.getValueString("requestedByUserId")).isEqualTo(BackupScheduledVerifyProcess.SCHEDULER_USER);
+      });
+      assertThat(catalog.get(BackupsTables.PLAN, planId).orElseThrow().getValue("lastVerifiedDate")).as("a storage check alone does not count as verified").isNull();
+
+      /////////////////////////////////////////////////////////////////////
+      // the plan's level decides how far it goes; an inactive plan is skipped //
+      /////////////////////////////////////////////////////////////////////
+      catalog.update(BackupsTables.PLAN, planId, Map.of("verifyLevel", "CONTENT"));
+      assertThat(run(planId).getValues().get("level")).isEqualTo("CONTENT");
+      assertThat(catalog.get(BackupsTables.PLAN, planId).orElseThrow().getValue("lastVerifiedDate")).isNotNull();
+      catalog.update(BackupsTables.PLAN, planId, Map.of("isActive", false));
+      assertThat(run(planId).getValues().get("status")).isEqualTo("INACTIVE");
+   }
+
+
+
+   /*******************************************************************************
+    ** An unset or unknown level falls back to CONTENT.
+    *******************************************************************************/
+   @Test
+   void levelFallsBackToContent()
+   {
+      assertThat(BackupScheduledVerifyProcess.levelOf(new QRecord())).isEqualTo("CONTENT");
+      assertThat(BackupScheduledVerifyProcess.levelOf(new QRecord().withValue("verifyLevel", "BOGUS"))).isEqualTo("CONTENT");
+      assertThat(BackupScheduledVerifyProcess.levelOf(new QRecord().withValue("verifyLevel", "TEST_RESTORE"))).isEqualTo("TEST_RESTORE");
+   }
+
+
+
+   /*******************************************************************************
+    ** Run the scheduled verify entry point for a plan, as the scheduler would.
+    *******************************************************************************/
+   private RunProcessOutput run(Integer planId) throws QException
+   {
+      RunProcessInput input = new RunProcessInput();
+      input.setProcessName(BackupScheduledVerifyProcess.NAME);
+      input.setCallback(QProcessCallbackFactory.forPrimaryKey("id", planId));
+      input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
+      return (new RunProcessAction().execute(input));
+   }
+}

src/test/java/com/kingsrook/qbits/backups/verify/BackupVerifierTest.java

@@ -591,4 +591,23 @@ class BackupVerifierTest
       }
       return (bytes.toByteArray());
    }
+
+
+   /*******************************************************************************
+    ** A verification must propagate a job interrupt, not report it as corruption.
+    *******************************************************************************/
+   @Test
+   void testDurableCancellation() throws Exception
+   {
+      BackupRunResult run = backup(plan(Map.of()));
+      try(var scope = com.kingsrook.qqq.backend.core.jobs.plans.JobChunkScope.open(null, () ->
+      {
+         throw (new com.kingsrook.qqq.backend.core.jobs.plans.JobInterruptedException(com.kingsrook.qqq.backend.core.jobs.plans.JobInterruptedException.Reason.CANCEL));
+      }))
+      {
+         org.assertj.core.api.Assertions.assertThatThrownBy(() -> new BackupVerifier().verify(run.getRunId(), BackupVerifier.CONTENT, "MANUAL", "test"))
+            .isInstanceOf(com.kingsrook.qqq.backend.core.jobs.plans.JobInterruptedException.class);
+      }
+   }
+
 }

src/test/resources/manifests/legacy-schema.json

@@ -0,0 +1,130 @@
+{
+  "author" : {
+    "schema" : {
+      "name" : "author",
+      "primaryKey" : "id",
+      "backendType" : "memory",
+      "fields" : {
+        "bio" : {
+          "type" : "TEXT",
+          "required" : false,
+          "maxLength" : null,
+          "pvs" : null,
+          "heavy" : false,
+          "sensitive" : false
+        },
+        "id" : {
+          "type" : "INTEGER",
+          "required" : false,
+          "maxLength" : null,
+          "pvs" : null,
+          "heavy" : false,
+          "sensitive" : false
+        },
+        "name" : {
+          "type" : "STRING",
+          "required" : false,
+          "maxLength" : null,
+          "pvs" : null,
+          "heavy" : false,
+          "sensitive" : false
+        }
+      },
+      "uniqueKeys" : [ ],
+      "associations" : [ ]
+    },
+    "fingerprint" : "sha256:c0fce31286898a15a98f9f46515b15c6eee4ffeb1f1eb39d6100833dfe3f760e"
+  },
+  "blogPost" : {
+    "schema" : {
+      "name" : "blogPost",
+      "primaryKey" : "id",
+      "backendType" : "memory",
+      "fields" : {
+        "authorId" : {
+          "type" : "INTEGER",
+          "required" : false,
+          "maxLength" : null,
+          "pvs" : "author",
+          "heavy" : false,
+          "sensitive" : false
+        },
+        "body" : {
+          "type" : "TEXT",
+          "required" : false,
+          "maxLength" : null,
+          "pvs" : null,
+          "heavy" : false,
+          "sensitive" : false
+        },
+        "createDate" : {
+          "type" : "DATE_TIME",
+          "required" : false,
+          "maxLength" : null,
+          "pvs" : null,
+          "heavy" : false,
+          "sensitive" : false
+        },
+        "id" : {
+          "type" : "INTEGER",
+          "required" : false,
+          "maxLength" : null,
+          "pvs" : null,
+          "heavy" : false,
+          "sensitive" : false
+        },
+        "mediaKey" : {
+          "type" : "STRING",
+          "required" : false,
+          "maxLength" : null,
+          "pvs" : null,
+          "heavy" : false,
+          "sensitive" : false
+        },
+        "price" : {
+          "type" : "DECIMAL",
+          "required" : false,
+          "maxLength" : null,
+          "pvs" : null,
+          "heavy" : false,
+          "sensitive" : false
+        },
+        "published" : {
+          "type" : "BOOLEAN",
+          "required" : false,
+          "maxLength" : null,
+          "pvs" : null,
+          "heavy" : false,
+          "sensitive" : false
+        },
+        "publishedOn" : {
+          "type" : "DATE",
+          "required" : false,
+          "maxLength" : null,
+          "pvs" : null,
+          "heavy" : false,
+          "sensitive" : false
+        },
+        "secretNote" : {
+          "type" : "STRING",
+          "required" : false,
+          "maxLength" : null,
+          "pvs" : null,
+          "heavy" : false,
+          "sensitive" : true
+        },
+        "title" : {
+          "type" : "STRING",
+          "required" : false,
+          "maxLength" : null,
+          "pvs" : null,
+          "heavy" : false,
+          "sensitive" : false
+        }
+      },
+      "uniqueKeys" : [ ],
+      "associations" : [ ]
+    },
+    "fingerprint" : "sha256:3a4de7479908cd4b4ce8eaa7a7af43e7cbaabbdbc2b9741bebeb24a6bd41b9ea"
+  }
+}