Compare commits

..
13 Commits
Author SHA1 Message Date
CircleCI fa9c4c1774 Bump version to 0.1.0-RC.7 [skip ci] 2026-10-15 02:21:57 +00:00
james.maesandClaude Opus 5.5 c8ba79f55b Merge develop into release/0.1 for 0.1.0-RC.7 (#26 QQQ RC.8, #27 help bundle)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 21:18:12 -05:00
CircleCI 7830226982 Bump version to 0.1.0-RC.6 [skip ci] 2026-10-14 22:44:27 +00:00
james.maesandClaude Opus 5.5 b1f258a3d5 Merge develop into release/0.1 for 0.1.0-RC.6 (#24)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 17:41:40 -05:00
CircleCI 3df24cda5a Bump version to 0.1.0-RC.5 [skip ci] 2026-10-14 22:22:38 +00:00
james.maesandClaude Opus 5.5 987c20899b Merge develop into release/0.1 for 0.1.0-RC.5 (#25 setup card, default plan, scheduled verify)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 17:19:54 -05:00
CircleCI 14bb6fa1b4 Bump version to 0.1.0-RC.4 [skip ci] 2026-10-14 12:26:09 +00:00
james.maesandClaude Opus 5.5 b1ff70d7d3 Merge develop into release/0.1 for 0.1.0-RC.4 (#21 schema fingerprint, keyset pager, guardrails)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 07:21:32 -05:00
CircleCI cda334f2fe Bump version to 0.1.0-RC.3 [skip ci] 2026-10-14 00:53:05 +00:00
james.maesandClaude Opus 5.5 590b132fb6 Merge develop into release/0.1 for 0.1.0-RC.3 (core alerts, renamed alert types, #20)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 19:50:34 -05:00
CircleCI f0a8c5000e Bump version to 0.1.0-RC.2 [skip ci] 2026-10-13 16:11:19 +00:00
james.maesandClaude Opus 5.5 8f3f9ec60a Merge develop into release/0.1 for 0.1.0-RC.2
qbit-object-storage 0.1.0-RC.2 (AWS SDK v2 2.41.34, GHSA-443w-3rq3-5m5h; #17).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 11:08:28 -05:00
CircleCI cdaf218201 Bump version to 0.1.0-RC.1 [skip ci] 2026-10-13 00:26:41 +00:00
39 changed files with 414 additions and 2118 deletions

No files matched your search

+57 -11
View File
@@ -1,14 +1,60 @@
version: 2.1
setup: true
orbs:
qqq-orb: kingsrook/qqq-orb@0.8
qqq-orb: kingsrook/qqq-orb@0.6.9
commands:
configure-package-install:
steps:
- run:
name: Keep package installation unattended
command: |
# These build machines are discarded after the job.
sudo install -d /etc/needrestart/conf.d
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
workflows:
setup:
jobs:
- qqq-orb/generate_pipeline:
orb_version: '0.8'
filters:
branches:
ignore: automation/qbit-rc-pending
tags:
only: /^v\d+\.\d+\.\d+$/
test_only:
jobs:
- qqq-orb/mvn_test_only:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
filters:
branches:
ignore: /(develop|release\/.*)/
publish_snapshot:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: snapshot
filters:
branches:
only: [develop]
publish_release_candidate:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: release_candidate
filters:
branches:
only: [/release\/.*/]
publish_release:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: release
filters:
branches:
ignore: /.*/
tags:
only: [/^v\d+\.\d+\.\d+$/]
-7
View File
@@ -1,7 +0,0 @@
name: build
on:
push:
jobs:
maven:
uses: self:KofTwentyTwo/ci-workflows/.gitea/workflows/maven.yaml@v1
secrets: inherit
-23
View File
@@ -1,23 +0,0 @@
name: CodeQL
'on':
push:
branches:
- main
- develop
- release/**
- hotfix/**
pull_request: {}
schedule:
- cron: 41 16 * * 0
workflow_dispatch: {}
permissions:
contents: read
actions: read
security-events: write
packages: read
concurrency:
group: codeql-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
analyze:
uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22
+13 -62
View File
@@ -1,74 +1,25 @@
# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
# the release line and the release/* push runs CircleCI publish_release_candidate.
name: Release candidate
'on':
on:
workflow_dispatch:
inputs:
dry_run:
description: Plan a cut without changing release state
description: Print the plan without merging or creating branches
type: boolean
default: false
finalize:
description: Retry finalization of an existing cut; do not cut another RC
type: boolean
default: false
recover:
description: Recover the same published RC after a deployment timeout; never
reupload
type: boolean
default: false
circle_build:
description: Original Circle publisher build number (recovery only)
type: string
default: ''
merge_sha:
description: Existing cut merge_sha (finalize retry only)
type: string
default: ''
bump_sha:
description: Existing cut bump_sha (finalize retry only)
type: string
default: ''
branch:
description: Existing cut branch (finalize retry only)
type: string
default: ''
version:
description: Existing cut version (finalize retry only)
type: string
default: ''
pull_request:
types:
- closed
branches:
- develop
repository_dispatch:
types:
- qbit-rc-published
types: [closed]
branches: [develop]
permissions:
contents: write
actions: read
checks: read
jobs:
release-rc:
if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
|| (github.event_name == 'pull_request' && github.event.pull_request.merged
== true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
'release:rc'))
uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
with:
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
}}
finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
== 'workflow_dispatch' && inputs.finalize) }}
recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
}}
circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
|| '' }}
merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
|| inputs.merge_sha || '' }}
bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
|| inputs.bump_sha || '' }}
branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
|| inputs.branch || '' }}
version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
|| inputs.version || '' }}
secrets: inherit
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
-16
View File
@@ -1,16 +0,0 @@
profile: qbit
orb_version: '0.8'
default_branch: develop
java_version: '21'
browsers: true
api_checks: false
docs_only: false
release_on_main: false
release_tag_policy: stable
test_branch_policy: non-release
publish:
snapshot: true
release_candidate: true
release: true
asciidoc: false
release_completion: true
-9
View File
@@ -5,7 +5,6 @@ All notable changes to this project are documented here. The format follows Keep
## [0.1.0] - Unreleased
### Added
- Central `backups.sla` and per-active-plan `backups.destination.<planId>` health checks, with bounded read-only destination diagnostics and secret-free status details.
- Always-durable backup, resume, scheduled backup, restore and verify processes with conservative `NONE` idempotency and stable backup identities on explicit job retry.
- Weighted backup phases, table/file counts, stored bytes, safe per-table/artifact logs, warnings, failures and resume notices through QQQ `JobProgress`.
- Cooperative job cancellation during backup pages, media copying, restore pages and verification scans; local media-reference spooling keeps phase separation from growing heap usage across tables.
@@ -16,11 +15,6 @@ All notable changes to this project are documented here. The format follows Keep
- Repository scaffold: `BackupsQBitConfig` and `BackupsQBitProducer` (table prefix `backups_`, permissions `backups.*`), build, CI and repository standard files.
### Changed
- Tables whose backend module has no query interface (storage-only modules such as qbit-edi `edi_preview_download`) are classified EXTERNAL and always left out of a backup with the reason `NOT_QUERYABLE`, even when selected by name or role; the Run backup review lists them. Before, such a table was read as DATA and the whole run FAILED with "Query is not implemented in this module".
- Failed runs persist an actionable `error_message` ("Backup failed: Class: message") and fill `error_detail` with the exception class, the cause chain and the top root-cause frames (bounded to 8 KB). Both go through `FailureText.sanitize`, which redacts URL user-info, presigned query parameters, key=value secrets, Authorization/Bearer values, AWS access key ids, PEM blocks and Postgres row values while keeping object keys and bucket names.
- Multi-node deployments: the cached Prometheus text (`BackupsMetrics`) is registered with QQQ `ClusterCaches` as `qbit-backups.metrics` and cleared on every node when a run finishes or a plan is inserted, updated or deleted; the 60 s TTL remains the safety net. `SlaChecker`'s manifest-head and bucket-audit throttles and `BackupsRuntime`'s process-permission map are registered as node-local, with reasons. `BackupsClusterCachesTest` audits the package for unregistered static caches and covers a remote invalidation.
- Builds against QQQ `4.1.0-SNAPSHOT` (`qqq.snapshot.version`) for `ClusterCaches` until the next QQQ RC ships; a release must pin that RC.
- Read the QBit metadata version from the Maven artifact version so release builds report their released version.
- Target QQQ `4.1.0-RC.8` (job progress).
- Persist controlled backup failure messages instead of untrusted provider exception text.
- Pin QQQ to released 4.1.0-RC.7 and replace the temporary help loader with core HelpBundles registration.
@@ -32,6 +26,3 @@ All notable changes to this project are documented here. The format follows Keep
- New manifests identify `qqq-table-schema-v1` and carry core schema descriptors; restore and content verification still accept pre-RC.6 manifests with the original fingerprint algorithm.
- Preserve primary-key ordering, exclusive resume cursors, bounded pages and caller-owned transactions; reject non-orderable primary keys and scope filters with skip/limit.
- Require the core `backups.restore` permission during Review when using a core restore writer, including anonymous-mode hosts; preserve supplied timestamps through `RestoreWriteFlag`.
### Removed
- The legacy `BackupsHealthIndicator` and its optional `qqq-middleware-health` dependency; central health registration replaces manual indicator configuration.
-25
View File
@@ -43,11 +43,6 @@ Follow the QQQ [adoption checklist](https://github.com/QRun-IO/qqq/blob/develop/
See [the RC.6 migration notes](docs/RC6-MIGRATION.md) for fingerprint compatibility,
paging guarantees and restore permissions.
Health checks use the core health API released in QQQ `4.1.0-RC.8`.
The producer registers `backups.sla` and a read-only destination check for each
active plan (`backups.destination.<planId>`). With dependencies cached locally,
run `./mvnw -o verify`; no enforcer skip flags are required.
## In-app help
The producer registers [help/qbit-backups.yaml](src/main/resources/help/qbit-backups.yaml)
@@ -100,23 +95,3 @@ The backup SLA checker still owns expected intervals and grace periods.
RC.5 requires hyphenated alert names: `backups.run.missing-in-storage` and
`backups.plan.owner-invalid` replace the former underscore spellings, including
their deduplication keys. Update host routing rules that used those spellings.
## Health checks
The producer registers these central checks under component `backups`, category `STORAGE`:
| Name | Kind | Interval | Timeout |
| --- | --- | --- | --- |
| `backups.sla` | DEPENDENCY | 900s (15 minutes) | 5000ms |
| `backups.destination.<planId>` | DEPENDENCY | 86400s (24 hours) | 30000ms |
The SLA check reads persisted `slaStatus`: critical active plans that are LATE,
FAILING or MISSING_IN_STORAGE yield DOWN; non-critical failures yield DEGRADED.
Only UNVERIFIED/UNKNOWN plans yield UNKNOWN; no active plans yield UP. Details
contain counts per SLA status. The scheduled `backupCheckSla` still computes SLA state.
Destination checks are supplied for active plans and reuse `DestinationProbe` in
read-only list/head mode, including on unlocked buckets. They never write, delete,
or resolve prune credentials. The Test destination action retains its write tests.
Storage calls are bounded by the central runner, with deadline checks between calls.
Override intervals in `health_check` and disable checks through `health_check.enabled`.
+12 -28
View File
@@ -18,10 +18,9 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<properties>
<qqq.version>4.1.0-RC.26</qqq.version> <!-- feeds qbit-build-parent's qqq-bom-pom import -->
<qbit.guardrails.enforce>true</qbit.guardrails.enforce>
<qbit.guardrails.strictUiApiRoutes>true</qbit.guardrails.strictUiApiRoutes>
<revision>0.1.0-SNAPSHOT</revision>
<revision>0.1.0-RC.7</revision>
<maven.compiler.source>21</maven.compiler.source>
<maven.compiler.target>21</maven.compiler.target>
<jacoco.version>0.8.15</jacoco.version>
@@ -30,7 +29,7 @@
<parent>
<groupId>com.kingsrook</groupId>
<artifactId>qbit-build-parent</artifactId>
<version>2.2.1</version>
<version>2.1.0</version>
</parent>
<groupId>com.kingsrook.qbits</groupId>
@@ -75,6 +74,13 @@
<artifactId>qqq-backend-core</artifactId>
</dependency>
<!-- optional: the BackupsHealthIndicator (apps without qqq-middleware-health never load it) -->
<dependency>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-middleware-health</artifactId>
<optional>true</optional>
</dependency>
<!-- optional: ESB events (apps without qqq-esb never load it) -->
<dependency>
<groupId>com.kingsrook.qqq</groupId>
@@ -87,7 +93,7 @@
<dependency>
<groupId>com.kingsrook.qbits</groupId>
<artifactId>qbit-object-storage</artifactId>
<version>0.1.0-RC.10</version>
<version>0.1.0-RC.2</version>
<optional>true</optional>
</dependency>
@@ -107,7 +113,7 @@
<dependency>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-frontend-next</artifactId>
<version>1.0.0-RC.38</version>
<version>1.0.0-RC.14</version>
<scope>test</scope>
</dependency>
@@ -125,33 +131,11 @@
</dependencies>
<build>
<resources>
<!-- help bundles carry the artifact version (QQQ help bundle provenance) -->
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
<includes>
<include>help/*.yaml</include>
<include>qbit-backups.properties</include>
</includes>
</resource>
<resource>
<directory>src/main/resources</directory>
<filtering>false</filtering>
<excludes>
<exclude>help/*.yaml</exclude>
<exclude>qbit-backups.properties</exclude>
</excludes>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<systemPropertyVariables combine.children="append">
<qbit.artifactVersion>${project.version}</qbit.artifactVersion>
</systemPropertyVariables>
<!-- the MinIO test container default credentials, resolved through ${env.X} refs like production -->
<environmentVariables>
<STORAGE_TEST_ACCESS_KEY>minioadmin</STORAGE_TEST_ACCESS_KEY>
@@ -217,7 +201,7 @@
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<qqq.snapshot.version>4.1.0-RC.26</qqq.snapshot.version>
<qqq.snapshot.version>4.1.0-RC.8</qqq.snapshot.version>
</properties>
<dependencyManagement>
<dependencies>
@@ -17,17 +17,6 @@
package com.kingsrook.qbits.backups;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
import com.kingsrook.qbits.backups.health.BackupsDestinationHealthChecks;
import com.kingsrook.qbits.backups.health.BackupsSlaHealthCheck;
import com.kingsrook.qqq.backend.core.health.HealthCategory;
import com.kingsrook.qqq.backend.core.health.HealthCheckKind;
import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData;
import com.kingsrook.qqq.backend.core.health.HealthCheckSupplierMetaData;
import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundles;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaDataProducer;
@@ -41,41 +30,13 @@ public class BackupsQBitProducer implements QBitMetaDataProducer<BackupsQBitConf
{
public static final String GROUP_ID = "com.kingsrook.qbits";
public static final String ARTIFACT_ID = "qbit-backups";
public static final String VERSION = loadVersion();
public static final String VERSION = "0.1.0";
private BackupsQBitConfig qBitConfig;
private String namespace;
/*******************************************************************************
** Read the artifact version from the Maven-filtered resource.
*******************************************************************************/
private static String loadVersion()
{
try(InputStream inputStream = BackupsQBitProducer.class.getClassLoader().getResourceAsStream("qbit-backups.properties"))
{
if(inputStream == null)
{
throw (new IllegalStateException("Missing qbit-backups.properties; build with Maven to generate the artifact version"));
}
Properties properties = new Properties();
properties.load(inputStream);
String version = properties.getProperty("version");
if(version == null || version.isBlank() || version.contains("${"))
{
throw (new IllegalStateException("Missing or unfiltered version in qbit-backups.properties; build with Maven"));
}
return (version);
}
catch(IOException e)
{
throw (new IllegalStateException("Could not read artifact version from qbit-backups.properties", e));
}
}
/*******************************************************************************
** Fluent setter for the config.
*******************************************************************************/
@@ -203,13 +164,6 @@ public class BackupsQBitProducer implements QBitMetaDataProducer<BackupsQBitConf
// Registration is metadata-only; core guards startup sync when helpContent is absent.
HelpBundles.register(qInstance, "help/qbit-backups.yaml");
HealthChecksMetaData.ofOrWithNew(qInstance)
.withCheck(new HealthCheckMetaData().withName(BackupsSlaHealthCheck.NAME).withLabel("Backup SLA")
.withComponent("backups").withKind(HealthCheckKind.DEPENDENCY).withCategory(HealthCategory.STORAGE)
.withIntervalSeconds(900).withCheck(new QCodeReference(BackupsSlaHealthCheck.class)))
.withSupplier(new HealthCheckSupplierMetaData().withNamePrefix(BackupsDestinationHealthChecks.PREFIX)
.withComponent("backups").withSupplier(new QCodeReference(BackupsDestinationHealthChecks.class)));
// A parent app only lists the apps that are its children, so attach ourselves (for example Admin, then Backups).
com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData parent = qBitConfig.getParentAppName() == null ? null : qInstance.getApp(qBitConfig.getParentAppName());
if(parent != null && (parent.getChildren() == null || parent.getChildren().stream().noneMatch(child -> app.getName().equals(child.getName()))))
@@ -25,7 +25,6 @@ import com.kingsrook.qbits.backups.alerts.BackupAlertSink;
import com.kingsrook.qbits.backups.alerts.NoOpBackupAlertSink;
import com.kingsrook.qbits.backups.storage.BackupStorage;
import com.kingsrook.qbits.backups.storage.BackupStorageException;
import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
/*******************************************************************************
@@ -35,8 +34,7 @@ import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
*******************************************************************************/
public final class BackupsRuntime
{
private static final Map<String, String> PROCESS_PERMISSIONS = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
"built from the qbit config at startup, the same on every node");
private static final Map<String, String> PROCESS_PERMISSIONS = new ConcurrentHashMap<>();
private static volatile BackupsQBitConfig config;
private static volatile Clock clock = Clock.systemUTC();
@@ -69,11 +69,10 @@ public final class BackupsHomeView
public static String render(BackupsHomeData data, String pathPrefix)
{
StringBuilder html = new StringBuilder("<div class=\"backups-home\" style=\"" + ROOT + "\">");
List<BackupsSetupStatus.Item> setupItems = BackupsSetupStatus.load(data.plans());
planStatus(html, data, pathPrefix);
setup(html, setupItems, pathPrefix);
setup(html, data, pathPrefix);
html.append("<div style=\"").append(GRID).append("\">");
needsAttention(html, data, setupItems, pathPrefix);
needsAttention(html, data, pathPrefix);
latestTestRestore(html, data, pathPrefix);
recentActivity(html, data, pathPrefix);
trend(html, data);
@@ -145,8 +144,9 @@ public final class BackupsHomeView
** The Setup card: what a working backup needs, whether each part is in place,
** and the next step, linked to its table.
*******************************************************************************/
private static void setup(StringBuilder html, List<BackupsSetupStatus.Item> items, String prefix)
private static void setup(StringBuilder html, BackupsHomeData data, String prefix)
{
List<BackupsSetupStatus.Item> items = BackupsSetupStatus.load(data.plans());
open(html, "Setup");
long todo = items.stream().filter(i -> i.state() == BackupsSetupStatus.State.TODO).count();
html.append("<p style=\"margin:0 0 .6rem;font-size:.9rem;\">")
@@ -185,27 +185,23 @@ public final class BackupsHomeView
/*******************************************************************************
** Needs attention: every Setup item that needs setup (the same list the Setup
** card counts), plus plans in a problem state and recent failed runs, so the two
** cards never disagree.
** Needs attention.
*******************************************************************************/
private static void needsAttention(StringBuilder html, BackupsHomeData data, List<BackupsSetupStatus.Item> setupItems, String prefix)
private static void needsAttention(StringBuilder html, BackupsHomeData data, String prefix)
{
open(html, "Needs attention");
List<BackupsSetupStatus.Item> setupTodo = setupItems.stream().filter(i -> i.state() == BackupsSetupStatus.State.TODO).toList();
List<QRecord> plans = data.plansNeedingAttention();
if(setupTodo.isEmpty() && plans.isEmpty() && data.failedRuns().isEmpty())
List<QRecord> plans = data.plansNeedingAttention();
if(data.plans().isEmpty())
{
empty(html, data.plans().isEmpty() ? "Nothing to check yet: no plans." : "Nothing needs attention.");
empty(html, "Nothing to check yet: no plans.");
}
else if(plans.isEmpty() && data.failedRuns().isEmpty())
{
empty(html, "Nothing needs attention.");
}
else
{
html.append("<ul style=\"").append(ITEMS).append("\">");
for(BackupsSetupStatus.Item item : setupTodo)
{
html.append("<li>Setup: <a style=\"").append(LINK).append("\" href=\"").append(esc(prefix + "/" + item.tableName())).append("\">").append(esc(item.label())).append("</a>: ")
.append(esc(item.detail())).append("</li>");
}
for(QRecord plan : plans)
{
html.append("<li>").append(recordLink(prefix, BackupsTables.PLAN, plan.getValue("id"), plan.getValueString("name")))
@@ -1,95 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.health;
import com.kingsrook.qbits.backups.model.BackupsTables;
import com.kingsrook.qbits.backups.plan.DestinationProbe;
import com.kingsrook.qbits.backups.run.BackupCatalog;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.health.HealthCheck;
import com.kingsrook.qqq.backend.core.health.HealthCheckInput;
import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome;
import com.kingsrook.qqq.backend.core.health.HealthDetails;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
/*******************************************************************************
** Reuse the destination diagnostic in read-only mode; never publish its text.
*******************************************************************************/
public class BackupsDestinationHealthCheck implements HealthCheck
{
private final Integer planId;
/*******************************************************************************
** Retain only the id; destination and active state are read on each run.
*******************************************************************************/
public BackupsDestinationHealthCheck(Integer planId)
{
this.planId = planId;
}
/*******************************************************************************
** A removed or disabled plan can remain registered until supplier refresh.
*******************************************************************************/
@Override
public HealthCheckOutcome run(HealthCheckInput input)
{
boolean ownsContext = QContext.getQInstance() == null;
if(ownsContext)
{
QContext.init(input.getQInstance(), new QSystemUserSession(input.getQInstance()));
}
try
{
if(BackupsSlaHealthCheck.expired(input))
{
return (HealthCheckOutcome.down("Backup destination check timed out"));
}
QRecord plan = new BackupCatalog().get(BackupsTables.PLAN, planId).orElse(null);
if(plan == null || !Boolean.TRUE.equals(plan.getValueBoolean("isActive")))
{
return (HealthCheckOutcome.unknown("Backup plan is no longer active"));
}
DestinationProbe.Report report = DestinationProbe.probeReadOnly(plan, input.getDeadline());
if(BackupsSlaHealthCheck.expired(input))
{
return (HealthCheckOutcome.down("Backup destination check timed out"));
}
return ((report.ok() ? HealthCheckOutcome.up("Backup destination is readable") : HealthCheckOutcome.down("Backup destination probe failed"))
.withDetails(new HealthDetails().withValue("planId", planId).withValue("readOnly", true)
.withValue("checks", report.checks().size()).withValue("failedChecks", report.checks().stream().filter(check -> !check.ok()).count())));
}
catch(QException | RuntimeException e)
{
return (HealthCheckOutcome.down("Backup destination could not be checked"));
}
finally
{
if(ownsContext)
{
QContext.clear();
}
}
}
}
@@ -1,77 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.health;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import com.kingsrook.qbits.backups.model.BackupsTables;
import com.kingsrook.qbits.backups.run.BackupCatalog;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.health.HealthCategory;
import com.kingsrook.qqq.backend.core.health.HealthCheckKind;
import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData;
import com.kingsrook.qqq.backend.core.health.HealthCheckSupplier;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
/*******************************************************************************
** Discover one daily, read-only destination check per active backup plan.
*******************************************************************************/
public class BackupsDestinationHealthChecks implements HealthCheckSupplier
{
public static final String PREFIX = "backups.destination";
/*******************************************************************************
** Resolve from current catalog data, without opening any storage connections.
*******************************************************************************/
@Override
public List<HealthCheckMetaData> supply(QInstance qInstance) throws QException
{
boolean ownsContext = QContext.getQInstance() == null;
if(ownsContext)
{
QContext.init(qInstance, new QSystemUserSession(qInstance));
}
try
{
List<HealthCheckMetaData> checks = new ArrayList<>();
for(QRecord plan : new BackupCatalog().where(BackupsTables.PLAN, "isActive", true))
{
Integer planId = plan.getValueInteger("id");
checks.add(new HealthCheckMetaData().withName(PREFIX + "." + planId.toString().toLowerCase(Locale.ROOT))
.withLabel("Backup destination " + planId).withComponent("backups")
.withKind(HealthCheckKind.DEPENDENCY).withCategory(HealthCategory.STORAGE)
.withIntervalSeconds(86400).withTimeoutMs(30000).withCheckInstance(new BackupsDestinationHealthCheck(planId)));
}
return (checks);
}
finally
{
if(ownsContext)
{
QContext.clear();
}
}
}
}
@@ -0,0 +1,133 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.health;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.kingsrook.qbits.backups.model.BackupsTables;
import com.kingsrook.qbits.backups.run.BackupCatalog;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
import com.kingsrook.qqq.middleware.health.HealthIndicator;
import com.kingsrook.qqq.middleware.health.model.metadata.HealthCheckResult;
import com.kingsrook.qqq.middleware.health.model.metadata.HealthStatus;
/*******************************************************************************
** The Backups health indicator (design spec section 10.4), for
** qqq-middleware-health (an optional dependency: an app that does not have that
** module never loads this class). It reports DOWN if any active plan marked
** critical is LATE, FAILING or MISSING_IN_STORAGE, and UP with details otherwise.
** It reads the slaStatus the SLA checker maintains; it never touches storage, so
** a health call is cheap.
**
** It is OFF the liveness path by design. Add it to the indicators of the
** instance's health check (the readiness and monitoring view), and do not point a
** Kubernetes liveness probe at a check that includes it: a backup problem must
** never restart pods.
*******************************************************************************/
public class BackupsHealthIndicator implements HealthIndicator
{
public static final String NAME = "backups";
private static final Set<String> CRITICAL_STATUSES = Set.of("LATE", "FAILING", "MISSING_IN_STORAGE");
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public String getName()
{
return (NAME);
}
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public HealthCheckResult check(QInstance qInstance) throws QException
{
long started = System.currentTimeMillis();
boolean ownsContext = QContext.getQInstance() == null;
if(ownsContext)
{
QContext.init(qInstance, systemSession(qInstance));
}
try
{
List<QRecord> plans = new BackupCatalog().query(BackupsTables.PLAN, new QQueryFilter(new QFilterCriteria("isActive", QCriteriaOperator.EQUALS, true)));
List<String> criticalProblems = new ArrayList<>();
Map<String, Integer> byStatus = new LinkedHashMap<>();
for(QRecord plan : plans)
{
String status = plan.getValueString("slaStatus") == null ? "UNKNOWN" : plan.getValueString("slaStatus");
byStatus.merge(status, 1, Integer::sum);
if(Boolean.TRUE.equals(plan.getValueBoolean("critical")) && CRITICAL_STATUSES.contains(status))
{
criticalProblems.add(plan.getValueString("planKey") + ": " + status);
}
}
Map<String, Object> details = new LinkedHashMap<>();
details.put("activePlans", plans.size());
details.put("plansByStatus", byStatus);
if(!criticalProblems.isEmpty())
{
details.put("criticalPlansWithProblems", criticalProblems);
}
return (new HealthCheckResult()
.withStatus(criticalProblems.isEmpty() ? HealthStatus.UP : HealthStatus.DOWN)
.withDurationMs(System.currentTimeMillis() - started)
.withDetails(details));
}
finally
{
if(ownsContext)
{
QContext.clear();
}
}
}
/*******************************************************************************
** The session for a health request that has no QQQ context. It must be the
** core system user: token-based authentication modules (OAuth2, Auth0)
** reject any other tokenless session, which failed the check on such hosts.
*******************************************************************************/
static QSession systemSession(QInstance qInstance)
{
return (new QSystemUserSession(qInstance));
}
}
@@ -1,132 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.health;
import java.time.Instant;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.kingsrook.qbits.backups.model.BackupsTables;
import com.kingsrook.qbits.backups.run.BackupCatalog;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.health.HealthCheck;
import com.kingsrook.qqq.backend.core.health.HealthCheckInput;
import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome;
import com.kingsrook.qqq.backend.core.health.HealthDetails;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
/*******************************************************************************
** Read persisted SLA state using the former indicator's critical-plan rule.
** SLA calculation and storage verification remain the scheduled checker's job.
*******************************************************************************/
public class BackupsSlaHealthCheck implements HealthCheck
{
public static final String NAME = "backups.sla";
private static final Set<String> BAD_STATUSES = Set.of("LATE", "FAILING", "MISSING_IN_STORAGE");
private static final List<String> STATUSES = List.of("OK", "LATE", "FAILING", "MISSING_IN_STORAGE", "UNVERIFIED", "ABANDONED", "OWNER_INVALID", "UNKNOWN");
/*******************************************************************************
** Counts use a fixed vocabulary so stored strings cannot become detail keys.
*******************************************************************************/
@Override
public HealthCheckOutcome run(HealthCheckInput input)
{
boolean ownsContext = QContext.getQInstance() == null;
if(ownsContext)
{
QContext.init(input.getQInstance(), new QSystemUserSession(input.getQInstance()));
}
try
{
if(expired(input))
{
return (HealthCheckOutcome.down("Backup SLA check timed out"));
}
List<QRecord> plans = new BackupCatalog().where(BackupsTables.PLAN, "isActive", true);
Map<String, Integer> counts = new LinkedHashMap<>();
STATUSES.forEach(status -> counts.put(status, 0));
boolean criticalBad = false;
boolean ordinaryBad = false;
for(QRecord plan : plans)
{
String status = plan.getValueString("slaStatus");
status = status != null && STATUSES.contains(status) ? status : "UNKNOWN";
counts.merge(status, 1, Integer::sum);
if(BAD_STATUSES.contains(status))
{
if(Boolean.TRUE.equals(plan.getValueBoolean("critical")))
{
criticalBad = true;
}
else
{
ordinaryBad = true;
}
}
}
if(expired(input))
{
return (HealthCheckOutcome.down("Backup SLA check timed out"));
}
HealthDetails details = new HealthDetails().withValue("activePlans", plans.size());
counts.forEach((status, count) -> details.withValue("slaStatus." + status, count));
HealthCheckOutcome outcome = HealthCheckOutcome.up("Backup SLA has no failures");
if(criticalBad)
{
outcome = HealthCheckOutcome.down("Critical backup plans have SLA failures");
}
else if(ordinaryBad)
{
outcome = HealthCheckOutcome.degraded("Backup plans have SLA failures");
}
else if(!plans.isEmpty() && counts.get("UNVERIFIED") + counts.get("UNKNOWN") == plans.size())
{
outcome = HealthCheckOutcome.unknown("Backup SLA is not verified yet");
}
return (outcome.withDetails(details));
}
catch(QException | RuntimeException e)
{
return (HealthCheckOutcome.down("Backup SLA state could not be read"));
}
finally
{
if(ownsContext)
{
QContext.clear();
}
}
}
/*******************************************************************************
** Stop between catalog operations when the runner's budget is exhausted.
*******************************************************************************/
static boolean expired(HealthCheckInput input)
{
return (Thread.currentThread().isInterrupted() || !Instant.now().isBefore(input.getDeadline()));
}
}
@@ -20,12 +20,10 @@ package com.kingsrook.qbits.backups.metrics;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
import com.kingsrook.qbits.backups.BackupsQBitConfig;
import com.kingsrook.qbits.backups.BackupsRuntime;
import com.kingsrook.qbits.backups.model.BackupsTables;
import com.kingsrook.qbits.backups.run.BackupCatalog;
import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
@@ -58,21 +56,10 @@ public final class BackupsMetrics
{
public static final String ENDPOINT_PATH = "/metrics/backups";
/** Cluster cache name of the rendered text: cleared on every node when a run finishes or a plan changes. */
public static final String CACHE_NAME = "qbit-backups.metrics";
private static final Duration CACHE = Duration.ofSeconds(60);
private static final AtomicReference<Rendered> CACHED = ClusterCaches.shared(CACHE_NAME, new AtomicReference<>(), BackupsMetrics::resetCache, null);
/*******************************************************************************
** The rendered text and when it was rendered.
*******************************************************************************/
private record Rendered(String text, Instant at)
{
}
private static String cachedText;
private static Instant cachedAt;
@@ -97,21 +84,12 @@ public final class BackupsMetrics
/*******************************************************************************
** Forget the cached text on this node.
** Forget the cached text (tests).
*******************************************************************************/
public static void resetCache()
public static synchronized void resetCache()
{
CACHED.set(null);
}
/*******************************************************************************
** Forget the cached text on every node (the backup data changed).
*******************************************************************************/
public static void invalidate()
{
ClusterCaches.invalidate(CACHE_NAME);
cachedText = null;
cachedAt = null;
}
@@ -121,14 +99,13 @@ public final class BackupsMetrics
*******************************************************************************/
public static synchronized String renderCached() throws QException
{
Instant now = BackupsRuntime.getClock().instant();
Rendered cached = CACHED.get();
if(cached == null || cached.at().plus(CACHE).isBefore(now))
Instant now = BackupsRuntime.getClock().instant();
if(cachedText == null || cachedAt == null || cachedAt.plus(CACHE).isBefore(now))
{
cached = new Rendered(render(), now);
CACHED.set(cached);
cachedText = render();
cachedAt = now;
}
return (cached.text());
return (cachedText);
}
@@ -413,22 +413,15 @@ public final class BackupsTables
QCodeReference validation = new QCodeReference(BackupPlanTableCustomizer.class);
table.withCustomizer(TableCustomizers.PRE_INSERT_RECORD, validation);
table.withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, validation);
/////////////////////////////////////////////////////////////////////////////
// after a save or delete, the plan customizer clears the cached metrics on //
// every node //
/////////////////////////////////////////////////////////////////////////////
QCodeReference post = validation;
if(StringUtils.hasContent(config.getSchedulerName()))
{
post = MultiCustomizer.of(
QCodeReferenceWithProperties sync = MultiCustomizer.of(
BaseSyncToScheduledJobTableCustomizer.makeCodeReference(table, new BackupPlanSyncToScheduledJobProcess()),
BaseSyncToScheduledJobTableCustomizer.makeCodeReference(table, new BackupPlanVerifySyncToScheduledJobProcess()),
validation);
BaseSyncToScheduledJobTableCustomizer.makeCodeReference(table, new BackupPlanVerifySyncToScheduledJobProcess()));
table.withCustomizer(TableCustomizers.POST_INSERT_RECORD, sync);
table.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, sync);
table.withCustomizer(TableCustomizers.POST_DELETE_RECORD, sync);
}
table.withCustomizer(TableCustomizers.POST_INSERT_RECORD, post);
table.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, post);
table.withCustomizer(TableCustomizers.POST_DELETE_RECORD, post);
return (table);
}
@@ -22,7 +22,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import com.kingsrook.qbits.backups.metrics.BackupsMetrics;
import com.kingsrook.qbits.backups.model.BackupsTables;
import com.kingsrook.qbits.backups.run.BackupCatalog;
import com.kingsrook.qbits.backups.run.BackupRunFactory;
@@ -32,9 +31,6 @@ import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterfa
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
@@ -47,8 +43,7 @@ import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
** configuration (the processes write the last run and SLA fields) is not
** validated again.
**
** The saving user becomes the plan owner when none is given. After an insert,
** update or delete, the cached Prometheus text is cleared on every node.
** The saving user becomes the plan owner when none is given.
*******************************************************************************/
public class BackupPlanTableCustomizer implements TableCustomizerInterface
{
@@ -127,42 +122,6 @@ public class BackupPlanTableCustomizer implements TableCustomizerInterface
/*******************************************************************************
** A saved plan changes the metrics on every node.
*******************************************************************************/
@Override
public List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
{
BackupsMetrics.invalidate();
return (records);
}
/*******************************************************************************
** A saved plan changes the metrics on every node.
*******************************************************************************/
@Override
public List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
{
BackupsMetrics.invalidate();
return (records);
}
/*******************************************************************************
** A deleted plan changes the metrics on every node.
*******************************************************************************/
@Override
public List<QRecord> postDelete(DeleteInput deleteInput, List<QRecord> records) throws QException
{
BackupsMetrics.invalidate();
return (records);
}
/*******************************************************************************
** The scope rules to resolve: the ones sent with the plan (associated rows), or
** the saved ones.
@@ -20,7 +20,6 @@ package com.kingsrook.qbits.backups.plan;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
@@ -82,28 +81,6 @@ public final class DestinationProbe
** Probe the destination of a plan.
*******************************************************************************/
public static Report probe(QRecord plan)
{
return (probe(plan, false, null));
}
/*******************************************************************************
** Health monitoring only lists and heads; it never resolves prune credentials
** or probes capabilities. The runner bounds in-flight storage calls; check the
** deadline between calls because BackupStorage has no per-call timeout API.
*******************************************************************************/
public static Report probeReadOnly(QRecord plan, Instant deadline)
{
return (probe(plan, true, deadline));
}
/*******************************************************************************
** Share storage resolution and the existing locked-bucket read path.
*******************************************************************************/
private static Report probe(QRecord plan, boolean readOnly, Instant deadline)
{
List<Check> checks = new ArrayList<>();
List<String> warnings = new ArrayList<>();
@@ -112,23 +89,15 @@ public final class DestinationProbe
BackupStorage storage;
try
{
requireTime(deadline);
storage = BackupsRuntime.storageFor(alias);
requireTime(deadline);
}
catch(RuntimeException e)
{
checks.add(new Check("Open the backup bucket " + alias, false, e.getMessage()));
return (new Report(readOnly, checks, List.of(), warnings));
return (new Report(false, checks, List.of(), warnings));
}
checks.add(new Check("Open the backup bucket " + alias, true, "found"));
if(readOnly)
{
headOnly(plan, storage, BackupPruner.planPrefix(plan), checks, deadline);
return (new Report(true, checks, List.of(), warnings));
}
BackupStorage deleteSide = storage;
String pruneAlias = plan.getValueString("pruneBucketAlias");
if(pruneAlias == null || pruneAlias.isBlank())
@@ -164,7 +133,7 @@ public final class DestinationProbe
String prefix = BackupPruner.planPrefix(plan);
if(locked)
{
headOnly(plan, storage, prefix, checks, null);
headOnly(plan, storage, prefix, checks);
}
else
{
@@ -178,11 +147,10 @@ public final class DestinationProbe
/*******************************************************************************
** The probe for a locked bucket: list the prefix, and head the newest manifest.
*******************************************************************************/
private static void headOnly(QRecord plan, BackupStorage storage, String prefix, List<Check> checks, Instant deadline)
private static void headOnly(QRecord plan, BackupStorage storage, String prefix, List<Check> checks)
{
checks.add(new Check("Write and delete a probe object", true,
deadline == null ? "skipped: object lock is on, a probe object could not be deleted again" : "skipped: read-only probe"));
checks.add(attempt("List the plan prefix", deadline, () ->
checks.add(attempt("Write and delete a probe object", () -> "skipped: object lock is on, a probe object could not be deleted again"));
checks.add(attempt("List the plan prefix", () ->
{
List<StoredObjectInfo> objects = storage.list(prefix);
return (objects.size() + " objects under " + prefix);
@@ -194,11 +162,10 @@ public final class DestinationProbe
checks.add(new Check("Head the newest manifest", true, "there is no successful run yet"));
return;
}
checks.add(attempt("Head the newest manifest", deadline, () ->
checks.add(attempt("Head the newest manifest", () ->
{
QRecord run = new com.kingsrook.qbits.backups.run.BackupCatalog().get(com.kingsrook.qbits.backups.model.BackupsTables.RUN, lastSuccess).orElseThrow();
String key = run.getValueString("manifestKey");
requireTime(deadline);
if(key == null || storage.head(key).isEmpty())
{
throw (new IllegalStateException("the manifest of the newest successful run is not in the bucket"));
@@ -271,9 +238,6 @@ public final class DestinationProbe
@FunctionalInterface
private interface Attempt
{
/*******************************************************************************
** Execute one diagnostic operation.
*******************************************************************************/
String run() throws Exception;
}
@@ -283,40 +247,14 @@ public final class DestinationProbe
**
*******************************************************************************/
private static Check attempt(String name, Attempt attempt)
{
return (attempt(name, null, attempt));
}
/*******************************************************************************
** Never begin another storage operation after the health budget expires.
*******************************************************************************/
private static Check attempt(String name, Instant deadline, Attempt attempt)
{
try
{
requireTime(deadline);
String detail = attempt.run();
requireTime(deadline);
return (new Check(name, true, detail));
return (new Check(name, true, attempt.run()));
}
catch(Exception e)
{
return (new Check(name, false, e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage()));
}
}
/*******************************************************************************
** Existing on-demand diagnostics have no deadline; health probes always do.
*******************************************************************************/
private static void requireTime(Instant deadline)
{
if(deadline != null && (Thread.currentThread().isInterrupted() || !Instant.now().isBefore(deadline)))
{
throw (new IllegalStateException("Destination probe timed out"));
}
}
}
@@ -22,8 +22,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import com.kingsrook.qbits.backups.BackupsRuntime;
import com.kingsrook.qbits.backups.model.BackupsTables;
import com.kingsrook.qbits.backups.permissions.BackupsPermissions;
@@ -180,10 +178,6 @@ public final class BackupsProcesses
private final ProcessSummaryLine tables = line(Status.INFO, "table will be backed up", "tables will be backed up", "table was backed up", "tables were backed up");
private final ProcessSummaryLine excluded = line(Status.INFO, "table is left out by its role or scope", "tables are left out by their role or scope",
"table was left out by its role or scope", "tables were left out by their role or scope");
private final ProcessSummaryLine unreadable = line(Status.INFO, "table is left out because its backend cannot be queried (storage only)",
"tables are left out because their backend cannot be queried (storage only)", "table was left out because its backend cannot be queried (storage only)",
"tables were left out because their backend cannot be queried (storage only)");
private final Set<String> unreadableNames = new TreeSet<>();
private final ProcessSummaryLine inactive = line(Status.WARNING, "plan is not active (it can still be run by hand)", "plans are not active (they can still be run by hand)",
"plan was not active", "plans were not active");
private final ProcessSummaryLine invalid = line(Status.ERROR, "plan cannot run", "plans cannot run", "plan could not run", "plans could not run");
@@ -211,14 +205,6 @@ public final class BackupsProcesses
plans.incrementCountAndAddPrimaryKey(plan.getValue("id"));
tables.incrementCount(scope.tables().size());
excluded.incrementCount(scope.excluded().size());
for(ScopeResolver.ExcludedTable out : scope.excluded())
{
if(ScopeResolver.NOT_QUERYABLE.equals(out.reason()) && unreadableNames.add(out.name()))
{
unreadable.incrementCount();
unreadable.setMessageSuffix(": " + String.join(", ", unreadableNames));
}
}
if(!Boolean.TRUE.equals(plan.getValueBoolean("isActive")))
{
inactive.incrementCountAndAddPrimaryKey(plan.getValue("id"));
@@ -249,7 +235,6 @@ public final class BackupsProcesses
plans.addSelfToListIfAnyCount(lines);
tables.addSelfToListIfAnyCount(lines);
excluded.addSelfToListIfAnyCount(lines);
unreadable.addSelfToListIfAnyCount(lines);
inactive.addSelfToListIfAnyCount(lines);
invalid.addSelfToListIfAnyCount(lines);
return (lines);
@@ -118,7 +118,7 @@ public final class ExportTableProcess
continue;
}
TableRole role = TableRoles.roleOf(qInstance, table);
if((role == TableRole.DATA || role == TableRole.AUDIT || role == TableRole.DERIVED) && TableRoles.isQueryable(qInstance, table))
if(role == TableRole.DATA || role == TableRole.AUDIT || role == TableRole.DERIVED)
{
processes.add(processFor(table.getName()));
}
@@ -47,7 +47,6 @@ import com.kingsrook.qbits.backups.files.FileReference;
import com.kingsrook.qbits.backups.files.FileReferenceProvider;
import com.kingsrook.qbits.backups.manifest.BackupManifest;
import com.kingsrook.qbits.backups.manifest.TableFingerprints;
import com.kingsrook.qbits.backups.metrics.BackupsMetrics;
import com.kingsrook.qbits.backups.model.BackupsTables;
import com.kingsrook.qbits.backups.scope.ScopeResolver;
import com.kingsrook.qbits.backups.storage.BackupStorage;
@@ -77,7 +76,7 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
** content-addressed store; checksums.sha256 is written; and the manifest is
** written LAST, then checked with a head. A run with no manifest did not
** complete. The run row is updated every page (heartbeat), and a failure marks
** the run FAILED with a sanitized message and detail (no secrets, no row values).
** the run FAILED with a scrubbed message (never row values).
*******************************************************************************/
public class BackupEngine
{
@@ -142,7 +141,6 @@ public class BackupEngine
runValues.remove("startDate");
runValues.put("endDate", null);
runValues.put("errorMessage", null);
runValues.put("errorDetail", null);
}
if(request.getExistingRunId() != null)
{
@@ -156,12 +154,9 @@ public class BackupEngine
}
catch(Exception e)
{
LOG.warn("Could not create the backup run record", e, logPair("runKey", runKey));
String message = FailureText.summary("Could not create the run record", e);
result.setStatus("FAILED");
result.setErrorMessage(message);
result.setErrorDetail(FailureText.detail(e));
failProgress(progress, message);
result.setErrorMessage("Could not create the run record (" + e.getClass().getSimpleName() + ")");
failProgress(progress, "Could not create the run record (" + e.getClass().getSimpleName() + ")");
return (result);
}
@@ -178,12 +173,12 @@ public class BackupEngine
}
catch(CancelledException e)
{
finish(request, result, "CANCELLED", "Cancelled", null, started);
finish(request, result, "CANCELLED", "Cancelled", started);
progress.info("Backup cancelled");
}
catch(TimedOutException e)
{
finish(request, result, "TIMED_OUT", e.getMessage(), null, started);
finish(request, result, "TIMED_OUT", e.getMessage(), started);
failProgress(progress, e.getMessage());
}
catch(Exception e)
@@ -193,14 +188,14 @@ public class BackupEngine
{
if(interrupted.getReason() != JobInterruptedException.Reason.LEASE_LOST)
{
finish(request, result, interrupted.getReason() == JobInterruptedException.Reason.CANCEL ? "CANCELLED" : "ABANDONED", "Job interrupted", null, started);
finish(request, result, interrupted.getReason() == JobInterruptedException.Reason.CANCEL ? "CANCELLED" : "ABANDONED", "Job interrupted", started);
}
throw (interrupted);
}
LOG.warn("Backup run failed", e, logPair("runKey", runKey));
String message = scrub(e);
failProgress(progress, message);
finish(request, result, "FAILED", message, FailureText.detail(e), started);
// Provider and reader messages can contain credentials or row values.
failProgress(progress, scrub(e));
finish(request, result, "FAILED", scrub(e), started);
}
finally
{
@@ -577,7 +572,7 @@ public class BackupEngine
extra.put("bytesRaw", bytesRaw);
extra.put("bytesStored", bytesStored);
extra.put("warningCount", result.getWarnings().size());
finishWith(request, result, result.getWarnings().isEmpty() ? "SUCCEEDED" : "SUCCEEDED_WITH_WARNINGS", null, null, started, extra);
finishWith(request, result, result.getWarnings().isEmpty() ? "SUCCEEDED" : "SUCCEEDED_WITH_WARNINGS", null, started, extra);
}
@@ -745,11 +740,11 @@ public class BackupEngine
/*******************************************************************************
**
*******************************************************************************/
private void finish(BackupRunRequest request, BackupRunResult result, String status, String message, String detail, Instant started)
private void finish(BackupRunRequest request, BackupRunResult result, String status, String message, Instant started)
{
try
{
finishWith(request, result, status, message, detail, started, new LinkedHashMap<>());
finishWith(request, result, status, message, started, new LinkedHashMap<>());
}
catch(Exception e)
{
@@ -757,7 +752,6 @@ public class BackupEngine
request.getProgress().warn("Could not finish the run record");
result.setStatus(status);
result.setErrorMessage(message);
result.setErrorDetail(detail);
}
}
@@ -766,7 +760,7 @@ public class BackupEngine
/*******************************************************************************
**
*******************************************************************************/
private void finishWith(BackupRunRequest request, BackupRunResult result, String status, String message, String detail, Instant started, Map<String, Serializable> extra) throws Exception
private void finishWith(BackupRunRequest request, BackupRunResult result, String status, String message, Instant started, Map<String, Serializable> extra) throws Exception
{
Instant end = request.getClock().instant();
Map<String, Serializable> v = new LinkedHashMap<>(extra);
@@ -775,14 +769,11 @@ public class BackupEngine
v.put("durationMs", end.toEpochMilli() - started.toEpochMilli());
v.put("heartbeatDate", end);
v.put("errorMessage", message);
v.put("errorDetail", detail);
v.put("warningCount", result.getWarnings().size());
catalog.update(BackupsTables.RUN, result.getRunId(), v);
result.setStatus(status);
result.setErrorMessage(message);
result.setErrorDetail(detail);
signal(request, result, status, message);
BackupsMetrics.invalidate();
}
@@ -835,17 +826,12 @@ public class BackupEngine
/*******************************************************************************
** The error_message of a failed run: "Backup failed: SimpleClass: message".
** Provider, driver and reader messages can contain credentials or row values,
** so the message goes through FailureText.sanitize (URL user-info, presigned
** query parameters, key=value secrets, Authorization values, AWS key ids, PEM
** blocks and Postgres row-value fragments are redacted; object keys and bucket
** names are kept). The class, the cause chain and the root cause frames go to
** error_detail (FailureText.detail); the full trace stays in the server log.
** Provider exceptions may contain credentials or row values. Persist only the
** failure type; detailed diagnostics stay in the server log.
*******************************************************************************/
private static String scrub(Exception e)
{
return (FailureText.summary(e));
return ("Backup failed (" + e.getClass().getSimpleName() + "); see server logs for details.");
}
@@ -43,7 +43,6 @@ public class BackupRunResult
private int filesMissing;
private long fileBytesCopied;
private String errorMessage;
private String errorDetail;
private List<String> warnings = new ArrayList<>();
@@ -398,26 +397,6 @@ public class BackupRunResult
/*******************************************************************************
** Getter for errorDetail (sanitized class, message, cause chain and root frames)
*******************************************************************************/
public String getErrorDetail()
{
return (errorDetail);
}
/*******************************************************************************
** Setter for errorDetail
*******************************************************************************/
public void setErrorDetail(String errorDetail)
{
this.errorDetail = errorDetail;
}
/*******************************************************************************
** Getter for warnings
*******************************************************************************/
@@ -1,184 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.run;
import java.util.Collections;
import java.util.IdentityHashMap;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/*******************************************************************************
** Safe text for a failure: what is persisted on a run row (error_message and
** error_detail) and shown in job output.
**
** Exception messages from storage providers, drivers and readers can carry
** credentials (URL user-info, presigned query strings, key=value secrets,
** Authorization headers, AWS access key ids, PEM blocks) and row values (the
** Postgres "Key (col)=(value)" and "Detail:" fragments). sanitize() redacts
** those and keeps the rest (class names, object keys, bucket names), which is
** what an operator needs to act on a failure without reading the server log.
*******************************************************************************/
public final class FailureText
{
/** Redaction marker. */
public static final String REDACTED = "[REDACTED]";
/** The longest error_detail written. */
public static final int MAX_DETAIL_CHARS = 8192;
/** The longest exception message kept in error_message. */
public static final int MAX_SUMMARY_MESSAGE_CHARS = 500;
/** Stack frames of the root cause kept in error_detail. */
public static final int ROOT_CAUSE_FRAMES = 8;
private static final int MAX_CAUSES = 20;
private static final String SECRET_NAME = "[\\w.-]*(?:password|passwd|pwd|secret|token|credential|signature|api[_-]?key|access[_-]?key|private[_-]?key)[\\w.-]*";
private static final Pattern PEM_BLOCK = Pattern.compile("-----BEGIN [A-Z0-9 ]+-----.*?(?:-----END [A-Z0-9 ]+-----|$)", Pattern.DOTALL);
private static final Pattern URL_USERINFO = Pattern.compile("(?i)\\b([a-z][a-z0-9+.-]*://)[^/\\s@?#]+@");
private static final Pattern QUERY_PARAM = Pattern.compile("(?i)([?&;](?:X-Amz-[\\w-]+|Signature|Credential|AWSAccessKeyId|Policy|sig|key|" + SECRET_NAME + ")=)[^&\\s\"'<>]*");
private static final Pattern PG_KEY_VALUE = Pattern.compile("(?i)\\b(Key \\([^)]*\\)=\\().*?(\\))(?=\\s|$|[.,;:])");
private static final Pattern PG_DETAIL = Pattern.compile("(?i)\\b(Detail:|Failing row contains)[^\\r\\n]*");
private static final Pattern AUTHORIZATION = Pattern.compile("(?i)\\b((?:Proxy-)?Authorization\"?\\s*[:=]\\s*)(\"[^\"]*\"|[^\\r\\n,;]+)");
private static final Pattern BEARER = Pattern.compile("(?i)\\b(Bearer|Basic)\\s+[A-Za-z0-9._~+/=-]{6,}");
private static final Pattern SECRET_JSON = Pattern.compile("(?i)(\"" + SECRET_NAME + "\"\\s*:\\s*)\"[^\"]*\"");
private static final Pattern SECRET_ASSIGN = Pattern.compile("(?i)(?<![\\w.-])((?:" + SECRET_NAME + "|key)\\s*=\\s*)(?!\\[REDACTED)(\"[^\"]*\"|'[^']*'|[^\\s&,;)\\]}]+)");
private static final Pattern AWS_KEY_ID = Pattern.compile("\\b(?:AKIA|ASIA|AGPA|AIDA|AROA|ANPA|ANVA|AIPA)[A-Z0-9]{16}\\b");
private static final String R = Matcher.quoteReplacement(REDACTED);
/*******************************************************************************
** Utility class.
*******************************************************************************/
private FailureText()
{
}
/*******************************************************************************
** Redact secrets and row values from a message. Null stays null.
*******************************************************************************/
public static String sanitize(String text)
{
if(text == null)
{
return (null);
}
String s = PEM_BLOCK.matcher(text).replaceAll(Matcher.quoteReplacement("[REDACTED PEM]"));
s = URL_USERINFO.matcher(s).replaceAll("$1" + R + "@");
s = QUERY_PARAM.matcher(s).replaceAll("$1" + R);
s = PG_KEY_VALUE.matcher(s).replaceAll("$1" + R + "$2");
s = PG_DETAIL.matcher(s).replaceAll("$1 " + R);
s = AUTHORIZATION.matcher(s).replaceAll("$1" + R);
s = BEARER.matcher(s).replaceAll("$1 " + R);
s = SECRET_JSON.matcher(s).replaceAll("$1\"" + R + "\"");
s = SECRET_ASSIGN.matcher(s).replaceAll("$1" + R);
s = AWS_KEY_ID.matcher(s).replaceAll(R);
return (s);
}
/*******************************************************************************
** The short error_message: "Backup failed: SimpleClass: sanitized message".
*******************************************************************************/
public static String summary(Throwable t)
{
return (summary("Backup failed", t));
}
/*******************************************************************************
** A short message: "prefix: SimpleClass: sanitized message" (bounded).
*******************************************************************************/
public static String summary(String prefix, Throwable t)
{
String message = sanitize(t.getMessage());
if(message == null || message.isBlank())
{
return (prefix + ": " + t.getClass().getSimpleName());
}
message = message.strip();
if(message.length() > MAX_SUMMARY_MESSAGE_CHARS)
{
message = message.substring(0, MAX_SUMMARY_MESSAGE_CHARS) + "...";
}
return (prefix + ": " + t.getClass().getSimpleName() + ": " + message);
}
/*******************************************************************************
** The error_detail: the exception class and sanitized message, each cause
** ("Caused by: ..."), then the top stack frames of the root cause, bounded to
** MAX_DETAIL_CHARS. Frames are class, method, file and line only (no values).
*******************************************************************************/
public static String detail(Throwable t)
{
StringBuilder out = new StringBuilder(line(t));
Set<Throwable> seen = Collections.newSetFromMap(new IdentityHashMap<>());
seen.add(t);
Throwable root = t;
Throwable cause = t.getCause();
while(cause != null && seen.size() < MAX_CAUSES && seen.add(cause))
{
out.append("\nCaused by: ").append(line(cause));
root = cause;
cause = cause.getCause();
}
List<StackTraceElement> frames = List.of(root.getStackTrace());
int shown = Math.min(ROOT_CAUSE_FRAMES, frames.size());
if(shown > 0)
{
out.append("\nRoot cause stack (top ").append(shown).append(" of ").append(frames.size()).append(" frames):");
for(StackTraceElement frame : frames.subList(0, shown))
{
out.append("\n\tat ").append(frame);
}
}
if(out.length() > MAX_DETAIL_CHARS)
{
String marker = "\n... (truncated)";
out.setLength(MAX_DETAIL_CHARS - marker.length());
out.append(marker);
}
return (out.toString());
}
/*******************************************************************************
** "fully.qualified.Class: sanitized message".
*******************************************************************************/
private static String line(Throwable t)
{
String message = sanitize(t.getMessage());
return (message == null ? t.getClass().getName() : t.getClass().getName() + ": " + message);
}
}
@@ -48,9 +48,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
*******************************************************************************/
public class ScopeResolver
{
/** Exclusion reason of a table whose backend has no query interface (storage-only): it cannot be read, even when selected. */
public static final String NOT_QUERYABLE = "NOT_QUERYABLE";
private final QInstance qInstance;
@@ -208,10 +205,6 @@ public class ScopeResolver
{
excluded.add(new ExcludedTable(name, "BACKUPS"));
}
else if(!TableRoles.isQueryable(qInstance, table))
{
excluded.add(new ExcludedTable(name, NOT_QUERYABLE));
}
else if((role == TableRole.EPHEMERAL || role == TableRole.DERIVED || role == TableRole.EXTERNAL) && !explicit.contains(name) && !roles.contains(role))
{
excluded.add(new ExcludedTable(name, role.name()));
@@ -35,6 +35,6 @@ public enum TableRole
SECRET,
/** The tables of this qbit: the catalog is rebuilt from manifests. */
BACKUPS,
/** Tables backed by another system (API), or by a backend that cannot be queried (storage-only: never read, even when selected). Excluded by default. */
/** Tables backed by another system (API). Excluded by default; selectable. */
EXTERNAL
}
@@ -17,20 +17,12 @@
package com.kingsrook.qbits.backups.scope;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import com.kingsrook.qbits.backups.BackupsQBitConfig;
import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
@@ -39,14 +31,6 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
*******************************************************************************/
public final class TableRoles
{
private static final QLogger LOG = QLogger.getLogger(TableRoles.class);
////////////////////////////////////////////////////////////////////////////////
// whether a backend type's module has a query interface: the answer depends //
// only on the module class, so it is cached per backend type (roleOf is hot) //
////////////////////////////////////////////////////////////////////////////////
private static final Map<String, Boolean> QUERYABLE_BY_BACKEND_TYPE = new ConcurrentHashMap<>();
private static final Set<String> AUDIT_TABLES = Set.of("audit", "auditTable", "auditUser", "auditDetail");
private static final String[] SECRET_NAME_HINTS = { "refreshToken", "RefreshToken", "oauthToken", "OAuthToken", "apiKey" };
@@ -92,10 +76,6 @@ public final class TableRoles
{
return (TableRole.SECRET);
}
if(qInstance != null && !isQueryable(qInstance, table))
{
return (TableRole.EXTERNAL);
}
if(AUDIT_TABLES.contains(name))
{
return (TableRole.AUDIT);
@@ -128,56 +108,6 @@ public final class TableRoles
/*******************************************************************************
** Whether the rows of a table can be read: its backend module must have a query
** interface. A storage-only module (for example qbit-edi's preview download
** backend) throws from getQueryInterface(), and reading its table would fail the
** whole run, so such a table is never backed up. A backend whose module cannot
** be resolved cannot be read either (not cached: the module may register later).
*******************************************************************************/
public static boolean isQueryable(QInstance qInstance, QTableMetaData table)
{
if(qInstance == null || table == null || table.getBackendName() == null)
{
return (true);
}
QBackendMetaData backend = qInstance.getBackend(table.getBackendName());
if(backend == null || backend.getBackendType() == null)
{
return (true);
}
Boolean cached = QUERYABLE_BY_BACKEND_TYPE.get(backend.getBackendType());
if(cached != null)
{
return (cached);
}
QBackendModuleInterface module;
try
{
module = new QBackendModuleDispatcher().getQBackendModule(backend);
}
catch(QModuleDispatchException e)
{
LOG.debug("No backend module for a table, so it cannot be backed up", logPair("table", table.getName()), logPair("backendType", backend.getBackendType()));
return (false);
}
boolean queryable;
try
{
queryable = module.getQueryInterface() != null;
}
catch(IllegalStateException | UnsupportedOperationException e)
{
queryable = false;
}
QUERYABLE_BY_BACKEND_TYPE.put(backend.getBackendType(), queryable);
return (queryable);
}
/*******************************************************************************
** A table that is hidden and has a password or sensitive field (other than the
** keys and dates) is treated as a table of secrets.
@@ -35,7 +35,6 @@ import com.kingsrook.qbits.backups.model.BackupsTables;
import com.kingsrook.qbits.backups.run.BackupCatalog;
import com.kingsrook.qbits.backups.storage.BackupStorage;
import com.kingsrook.qbits.backups.storage.StoredObjectInfo;
import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
@@ -96,11 +95,8 @@ public class SlaChecker
private static final Set<String> FAILED_STATUSES = Set.of("FAILED", "TIMED_OUT", "ABANDONED");
private static final Set<String> FINISHED_STATUSES = Set.of("SUCCEEDED", "SUCCEEDED_WITH_WARNINGS", "FAILED", "TIMED_OUT", "ABANDONED");
private static final Map<String, Instant> MANIFEST_OK_AT = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
"throttles this node's storage HEAD of a manifest to once per MANIFEST_CACHE; it caches storage state, not editable data, "
+ "and a pruned run is skipped by its prunedDate, so each node is as fresh as a single node");
private static final Map<String, Instant> LAST_AUDIT_AT = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
"throttles this node's bucket audit to once per BUCKET_AUDIT_INTERVAL; another node auditing again is only repeated work");
private static final Map<String, Instant> MANIFEST_OK_AT = new ConcurrentHashMap<>();
private static final Map<String, Instant> LAST_AUDIT_AT = new ConcurrentHashMap<>();
private final BackupCatalog catalog = new BackupCatalog();
+1 -1
View File
@@ -1,5 +1,5 @@
bundle: qbit-backups
version: ${project.version}
version: 0.1.0-SNAPSHOT
layer: QBIT
entries:
- key: table:backups_plan;field:name
@@ -1 +0,0 @@
version=${project.version}
@@ -1,164 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.function.Consumer;
import com.kingsrook.qbits.backups.metrics.BackupsMetrics;
import com.kingsrook.qbits.backups.model.BackupsTables;
import com.kingsrook.qbits.backups.run.BackupCatalog;
import com.kingsrook.qqq.backend.core.cluster.ClusterBus;
import com.kingsrook.qqq.backend.core.cluster.ClusterBusInterface;
import com.kingsrook.qqq.backend.core.cluster.ClusterBusMetaData;
import com.kingsrook.qqq.backend.core.cluster.ClusterCacheAudit;
import com.kingsrook.qqq.backend.core.cluster.ClusterEvent;
import com.kingsrook.qqq.backend.core.cluster.QNodeRoles;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** The Backups caches are correct on several nodes: every static cache is
** registered with ClusterCaches, and the cached metrics are cleared on every
** node when a plan changes.
*******************************************************************************/
class BackupsClusterCachesTest
{
/** the "wire": events every node's transport sees */
static final List<Consumer<ClusterEvent>> WIRE = new CopyOnWriteArrayList<>();
/*******************************************************************************
** A transport over the static WIRE, so a "remote" publish reaches us.
*******************************************************************************/
public static class WireBus implements ClusterBusInterface
{
/*******************************************************************************
**
*******************************************************************************/
@Override
public void publish(ClusterEvent event)
{
WIRE.forEach(listener -> listener.accept(event));
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public AutoCloseable subscribe(Consumer<ClusterEvent> listener)
{
WIRE.add(listener);
return (() -> WIRE.remove(listener));
}
}
/*******************************************************************************
** A backed-up test instance, connected to the wire as node-a.
*******************************************************************************/
@BeforeEach
void setUp() throws Exception
{
VerifyTestSupport.setUp(1);
QInstance instance = QContext.getQInstance();
instance.withSupplementalMetaData(new ClusterBusMetaData().withBus(new QCodeReference(WireBus.class)));
QNodeRoles.set(new QNodeRoles.Settings(EnumSet.allOf(QNodeRoles.Role.class), true, true, true, null, "node-a"));
ClusterBus.connect(instance);
}
/*******************************************************************************
**
*******************************************************************************/
@AfterEach
void afterEach()
{
ClusterBus.close(QContext.getQInstance());
WIRE.clear();
QNodeRoles.set(null);
}
/*******************************************************************************
** Every static Memoization in the QBit is registered (shared or node-local).
*******************************************************************************/
@Test
void testNoUnregisteredStaticCaches() throws Exception
{
assertThat(ClusterCacheAudit.unregisteredStaticCaches("com.kingsrook.qbits.backups")).isEmpty();
}
/*******************************************************************************
** Another node's invalidation clears this node's cached metrics.
*******************************************************************************/
@Test
void testRemoteInvalidationClearsMetricsCache() throws Exception
{
String cached = BackupsMetrics.renderCached();
assertThat(BackupsMetrics.renderCached()).isSameAs(cached);
ClusterEvent remote = new ClusterEvent(ClusterEvent.TYPE_MEMO_INVALIDATE, BackupsMetrics.CACHE_NAME, null);
remote.setOriginNodeId("node-b");
WIRE.forEach(listener -> listener.accept(remote));
assertThat(BackupsMetrics.renderCached()).isNotSameAs(cached);
}
/*******************************************************************************
** A plan edit here clears the metrics here and tells the other nodes.
*******************************************************************************/
@Test
void testPlanEditPublishesMetricsInvalidation() throws Exception
{
QRecord plan = VerifyTestSupport.plan(Map.of());
String cached = BackupsMetrics.renderCached();
List<ClusterEvent> sent = new CopyOnWriteArrayList<>();
WIRE.add(sent::add);
new BackupCatalog().update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("slaHours", 48));
assertThat(BackupsMetrics.renderCached()).isNotSameAs(cached);
assertThat(sent).anySatisfy(event ->
{
assertThat(event.getType()).isEqualTo(ClusterEvent.TYPE_MEMO_INVALIDATE);
assertThat(event.getName()).isEqualTo(BackupsMetrics.CACHE_NAME);
assertThat(event.getOriginNodeId()).isEqualTo("node-a");
});
}
}
@@ -18,7 +18,6 @@ package com.kingsrook.qbits.backups;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundles;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfigValidationException;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
@@ -41,16 +40,12 @@ class BackupsQBitProducerTest
BackupsQBitConfig config = new BackupsQBitConfig().withBackendName("rdbms");
BackupsQBitProducer producer = new BackupsQBitProducer().withQBitConfig(config).withNamespace("backups");
String artifactVersion = System.getProperty("qbit.artifactVersion");
assertThat(artifactVersion).isNotBlank();
assertThat(producer.getQBitMetaData().getVersion()).isEqualTo(artifactVersion).isEqualTo(BackupsQBitProducer.VERSION);
assertThat(producer.isEnabled()).isTrue();
assertThat(producer.getQBitConfig()).isSameAs(config);
assertThat(producer.getNamespace()).isEqualTo("backups");
assertThat(config.getDefaultBackendNameForTables()).isEqualTo("rdbms");
producer.produce(qInstance);
assertThat(HelpBundles.of(qInstance).getBundles().get(0).getVersion()).isEqualTo(artifactVersion).isEqualTo(BackupsQBitProducer.VERSION);
assertThat(qInstance.getQBits()).hasSize(1);
assertThat(qInstance.getQBits().values().iterator().next().getArtifactId()).isEqualTo("qbit-backups");
}
@@ -88,7 +88,7 @@ class BackupsHomeTest
void testEmptyStates()
{
String html = BackupsHomeView.render(new BackupsHomeData(List.of(), List.of(), List.of(), List.of(), List.of(), Instant.now()), "/app");
assertThat(html).contains("No backup plans yet").contains("Setup: <a").contains("No plan: nothing is backed up.").doesNotContain("Nothing needs attention").contains("No test restore yet")
assertThat(html).contains("No backup plans yet").contains("Nothing to check yet").contains("No test restore yet")
.contains("Nothing has run yet").contains("Trends appear after the first successful run").contains("Retention starts once runs exist");
assertThat(BackupsHomeView.renderHealthCard(new BackupsHomeData(List.of(), List.of(), List.of(), List.of(), List.of(), Instant.now()), "/app")).contains("No backup plans yet");
}
@@ -121,45 +121,6 @@ class BackupsHomeTest
/*******************************************************************************
** The Setup card and the Needs attention card never contradict each other: a plan
** whose storage aliases do not resolve is a Setup fault, and it is listed under
** Needs attention too, with the same count.
*******************************************************************************/
@Test
void testUnresolvedAliasAppearsInSetupAndNeedsAttention() throws Exception
{
BackupsRuntime.getConfig().withStorageResolver(alias -> BackupsTestUtils.BACKUPS);
new BackupCatalog().insert(BackupsTables.PLAN, new LinkedHashMap<>(Map.of("name", "Offsite", "planKey", "offsite", "isActive", true, "scopeType", "INSTANCE",
"storageBucketAlias", "backupsAws", "pruneBucketAlias", "backupsAwsPrune")));
//////////////////////////////////////////////////////////////////////////////
// the aliases resolved when the plan was saved; now the bucket is gone //
//////////////////////////////////////////////////////////////////////////////
BackupsRuntime.getConfig().withStorageResolver(alias -> "backups".equals(alias) ? BackupsTestUtils.BACKUPS : null);
BackupsHomeData data = BackupsHomeData.load();
List<BackupsSetupStatus.Item> setupTodo = BackupsSetupStatus.load(data.plans()).stream().filter(i -> i.state() == BackupsSetupStatus.State.TODO).toList();
assertThat(setupTodo).extracting(BackupsSetupStatus.Item::label).contains("Storage");
String html = BackupsHomeView.render(data, "/app");
String attention = html.substring(html.indexOf(">Needs attention<"));
attention = attention.substring(0, attention.indexOf("All runs"));
assertThat(attention).doesNotContain("Nothing needs attention").contains("backupsAws").contains("backupsAwsPrune");
///////////////////////////////////////////////////////////////////////////////
// every Setup TODO is listed under Needs attention (counts agree: 2 and 2) //
///////////////////////////////////////////////////////////////////////////////
assertThat(html).contains(setupTodo.size() + " items need attention.");
for(BackupsSetupStatus.Item item : setupTodo)
{
assertThat(attention).contains("Setup: <a").contains(item.label());
}
assertThat(attention.split("<li>", -1).length - 1).isEqualTo(setupTodo.size() + data.plansNeedingAttention().size() + data.failedRuns().size());
}
/*******************************************************************************
** The links follow the configured table path prefix.
*******************************************************************************/
@@ -1,495 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.health;
import java.io.InputStream;
import java.io.OutputStream;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import com.kingsrook.qbits.backups.BackupsRuntime;
import com.kingsrook.qbits.backups.BackupsTestUtils;
import com.kingsrook.qbits.backups.VerifyTestSupport;
import com.kingsrook.qbits.backups.model.BackupsTables;
import com.kingsrook.qbits.backups.plan.DestinationProbe;
import com.kingsrook.qbits.backups.run.BackupCatalog;
import com.kingsrook.qbits.backups.storage.InMemoryBackupStorage;
import com.kingsrook.qbits.backups.storage.StoredObjectInfo;
import com.kingsrook.qbits.backups.storage.WriteOptions;
import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.health.HealthCategory;
import com.kingsrook.qqq.backend.core.health.HealthCheck;
import com.kingsrook.qqq.backend.core.health.HealthCheckInput;
import com.kingsrook.qqq.backend.core.health.HealthCheckKind;
import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData;
import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome;
import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData;
import com.kingsrook.qqq.backend.core.health.HealthStatus;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** Central registration, migrated SLA rules, read-only probes and secret safety.
*******************************************************************************/
class BackupsHealthChecksTest
{
private static final String SECRET = "S3NT1NEL-secret";
private final BackupCatalog catalog = new BackupCatalog();
private ReadOnlyStorage storage;
/*******************************************************************************
** All probes use memory storage; the resolver represents credential injection.
*******************************************************************************/
@BeforeEach
void setUp() throws Exception
{
VerifyTestSupport.setUp(3);
storage = new ReadOnlyStorage(SECRET);
BackupsRuntime.getConfig().withStorageResolver(alias ->
{
assertThat(alias).as("never resolve prune credentials").isNotEqualTo("prune");
return (storage);
});
}
/*******************************************************************************
** Use the actual producer registry, including its reflective supplier loading.
*******************************************************************************/
@Test
void testRegistrationAndDynamicPlans() throws Exception
{
HealthChecksMetaData registry = HealthChecksMetaData.ofOrWithNew(QContext.getQInstance());
assertThat(registry.getChecks()).extracting(HealthCheckMetaData::getName).containsExactly("backups.sla");
assertThat(registry.getSuppliers()).singleElement().satisfies(supplier ->
{
assertThat(supplier.getNamePrefix()).isEqualTo("backups.destination");
assertThat(supplier.getComponent()).isEqualTo("backups");
});
QRecord active = VerifyTestSupport.plan(Map.of());
VerifyTestSupport.plan(Map.of("name", "Off", "planKey", "off", "isActive", false));
List<HealthCheckMetaData> checks = registry.resolveAll(QContext.getQInstance());
assertThat(checks).hasSize(2).allSatisfy(check ->
{
assertThat(check.getComponent()).isEqualTo("backups");
assertThat(check.getKind()).isEqualTo(HealthCheckKind.DEPENDENCY);
assertThat(check.getCategory()).isEqualTo(HealthCategory.STORAGE);
assertThat(check.getIncludeInReadiness()).isFalse();
assertThat(check.getName()).matches("[a-z0-9]+([._-][a-z0-9]+)*");
});
assertThat(checks.get(0).getIntervalSeconds()).isEqualTo(900);
assertThat(checks.get(0).getTimeoutMs()).isEqualTo(5000);
assertThat(QCodeLoader.getAdHoc(HealthCheck.class, checks.get(0).getCheck()).run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
HealthCheckMetaData destination = checks.get(1);
assertThat(destination.getName()).isEqualTo("backups.destination." + active.getValueInteger("id"));
assertThat(destination.getIntervalSeconds()).isEqualTo(86400);
assertThat(destination.getTimeoutMs()).isEqualTo(30000);
assertThat(destination.getCheckInstance().run(input()).getStatus()).isEqualTo(HealthStatus.UP);
catalog.update(BackupsTables.PLAN, active.getValueInteger("id"), Map.of("isActive", false));
assertThat(destination.getCheckInstance().run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
assertThat(registry.resolveAll(QContext.getQInstance())).hasSize(1);
VerifyTestSupport.plan(Map.of("name", "New", "planKey", "new"));
assertThat(registry.resolveAll(QContext.getQInstance())).hasSize(2);
}
/*******************************************************************************
** Carry over the indicator's empty, critical, ordinary and inactive cases.
*******************************************************************************/
@Test
void testSlaUpDownAndDegraded() throws Exception
{
BackupsSlaHealthCheck check = new BackupsSlaHealthCheck();
HealthCheckOutcome empty = check.run(input());
assertThat(empty.getStatus()).isEqualTo(HealthStatus.UP);
assertThat(empty.getDetails().getValues()).containsEntry("activePlans", 0);
QRecord critical = VerifyTestSupport.plan(Map.of("critical", true));
QRecord ordinary = VerifyTestSupport.plan(Map.of("name", "Other", "planKey", "other"));
QRecord inactive = VerifyTestSupport.plan(Map.of("name", "Off", "planKey", "off", "isActive", false, "critical", true));
setStatus(inactive, "LATE");
setStatus(critical, "OK");
setStatus(ordinary, "OK");
assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UP);
for(String status : List.of("LATE", "FAILING", "MISSING_IN_STORAGE"))
{
setStatus(ordinary, status);
assertThat(check.run(input()).getStatus()).as(status).isEqualTo(HealthStatus.DEGRADED);
setStatus(critical, status);
HealthCheckOutcome down = check.run(input());
assertThat(down.getStatus()).as(status).isEqualTo(HealthStatus.DOWN);
assertThat(down.getDetails().getValues()).containsEntry("activePlans", 2).containsEntry("slaStatus." + status, 2);
assertClean(down);
setStatus(critical, "OK");
}
}
/*******************************************************************************
** Unknown or unverified plans do not become a false UP or a critical DOWN.
*******************************************************************************/
@Test
void testSlaUnknownAndMixedStates() throws Exception
{
QRecord plan = VerifyTestSupport.plan(Map.of("critical", true));
BackupsSlaHealthCheck check = new BackupsSlaHealthCheck();
assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
for(String status : List.of("UNVERIFIED", "UNKNOWN"))
{
setStatus(plan, status);
assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
}
QRecord other = VerifyTestSupport.plan(Map.of("name", "Other", "planKey", "other"));
setStatus(other, "UNVERIFIED");
assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
setStatus(other, "OK");
assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UP);
}
/*******************************************************************************
** Preserve the old indicator's no-context behavior and existing sessions.
*******************************************************************************/
@Test
void testWithoutContext() throws Exception
{
QRecord plan = VerifyTestSupport.plan(Map.of());
QInstance instance = QContext.getQInstance();
QSession session = QContext.getQSession();
new BackupsSlaHealthCheck().run(input());
assertThat(QContext.getQSession()).isSameAs(session);
QContext.clear();
try
{
HealthCheckInput input = new HealthCheckInput().withQInstance(instance).withDeadline(Instant.now().plusSeconds(5));
assertThat(new BackupsSlaHealthCheck().run(input).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
assertThat(QContext.getQInstance()).isNull();
assertThat(new BackupsDestinationHealthChecks().supply(instance)).hasSize(1);
assertThat(QContext.getQInstance()).isNull();
BackupsRuntime.getConfig().withStorageResolver(alias ->
{
assertThat(QContext.getQSession()).isInstanceOf(com.kingsrook.qqq.backend.core.model.session.QSystemUserSession.class);
return (storage);
});
assertThat(destination(plan).run(input).getStatus()).isEqualTo(HealthStatus.UP);
assertThat(QContext.getQInstance()).isNull();
}
finally
{
QContext.init(instance, session);
}
}
/*******************************************************************************
** Locked and unlocked plans only list/head; neither reads object contents.
*******************************************************************************/
@Test
void testDestinationReadOnlyAndMissingManifest() throws Exception
{
for(String lockMode : List.of("NONE", "GOVERNANCE"))
{
BackupsRuntime.getConfig().withStorageResolver(alias -> BackupsTestUtils.BACKUPS);
QRecord plan = VerifyTestSupport.plan(Map.of("name", lockMode, "planKey", lockMode.toLowerCase(), "objectLockMode", lockMode, "objectLockDays", 3, "pruneBucketAlias", "prune"));
BackupsRuntime.getConfig().withStorageResolver(alias ->
{
assertThat(alias).as("never resolve prune credentials").isNotEqualTo("prune");
return (storage);
});
Integer runId = catalog.insert(BackupsTables.RUN, Map.of("manifestKey", "manifest.json"));
catalog.update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("lastSuccessRunId", runId));
storage.putBytes("manifest.json", new byte[] { 1 });
assertThat(destination(plan).run(input()).getStatus()).isEqualTo(HealthStatus.UP);
storage.clear();
assertThat(destination(plan).run(input()).getStatus()).isEqualTo(HealthStatus.DOWN);
}
assertThat(storage.lists).isEqualTo(4);
assertThat(storage.heads).isEqualTo(4);
}
/*******************************************************************************
** A deleted plan is UNKNOWN and does not touch storage.
*******************************************************************************/
@Test
void testDeletedPlan() throws Exception
{
QRecord plan = VerifyTestSupport.plan(Map.of());
catalog.deleteByIds(BackupsTables.PLAN, List.of(plan.getValueInteger("id")));
assertThat(destination(plan).run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
assertThat(storage.lists).isZero();
}
/*******************************************************************************
** No work starts with an expired deadline, including direct probe calls.
*******************************************************************************/
@Test
void testExpiredDeadline() throws Exception
{
QRecord plan = VerifyTestSupport.plan(Map.of());
HealthCheckInput expired = input().withDeadline(Instant.now().minusSeconds(1));
assertThat(new BackupsSlaHealthCheck().run(expired).getStatus()).isEqualTo(HealthStatus.DOWN);
assertThat(destination(plan).run(expired).getStatus()).isEqualTo(HealthStatus.DOWN);
assertThat(DestinationProbe.probeReadOnly(plan, expired.getDeadline()).ok()).isFalse();
assertThat(storage.lists).isZero();
assertThat(storage.heads).isZero();
}
/*******************************************************************************
** Runner cancellation after list prevents the following manifest head.
*******************************************************************************/
@Test
void testCancellationBetweenStorageCalls() throws Exception
{
QRecord plan = VerifyTestSupport.plan(Map.of());
Integer runId = catalog.insert(BackupsTables.RUN, Map.of("manifestKey", "manifest.json"));
catalog.update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("lastSuccessRunId", runId));
storage.interruptAfterList = true;
try
{
assertThat(destination(plan).run(input()).getStatus()).isEqualTo(HealthStatus.DOWN);
assertThat(storage.lists).isEqualTo(1);
assertThat(storage.heads).isZero();
}
finally
{
Thread.interrupted();
}
}
/*******************************************************************************
** Poison configuration, persisted state and provider errors. sanitize is
** package-private in core; assert raw outcomes already contain no secrets.
*******************************************************************************/
@Test
void testNoSecretsInSuccessAndFailureOutcomes() throws Exception
{
QRecord plan = VerifyTestSupport.plan(Map.of("name", SECRET, "storageBucketAlias", SECRET, "critical", true));
Integer runId = catalog.insert(BackupsTables.RUN, Map.of("manifestKey", SECRET));
catalog.update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("lastSuccessRunId", runId));
storage.putBytes(SECRET, new byte[] { 1 });
for(String status : List.of("OK", "LATE", "UNVERIFIED", SECRET))
{
setStatus(plan, status);
assertClean(new BackupsSlaHealthCheck().run(input()));
assertClean(destination(plan).run(input()));
}
storage.failList = true;
HealthCheckOutcome failure = destination(plan).run(input());
assertThat(failure.getStatus()).isEqualTo(HealthStatus.DOWN);
assertClean(failure);
storage.failList = false;
storage.failHead = true;
failure = destination(plan).run(input());
assertThat(failure.getStatus()).isEqualTo(HealthStatus.DOWN);
assertClean(failure);
BackupsRuntime.getConfig().withStorageResolver(alias ->
{
throw (new IllegalStateException(SECRET));
});
failure = destination(plan).run(input());
assertThat(failure.getStatus()).isEqualTo(HealthStatus.DOWN);
assertClean(failure);
assertClean(new BackupsSlaHealthCheck().run(input()));
}
/*******************************************************************************
** Catalog failures are DOWN with fixed messages, even if metadata is poisoned.
*******************************************************************************/
@Test
void testCatalogFailureDoesNotExposeSecrets()
{
QContext.getQInstance().getTable(BackupsTables.PLAN).setBackendName(SECRET);
HealthCheckOutcome sla = new BackupsSlaHealthCheck().run(input());
HealthCheckOutcome destination = new BackupsDestinationHealthCheck(1).run(input());
assertThat(sla.getStatus()).isEqualTo(HealthStatus.DOWN);
assertThat(destination.getStatus()).isEqualTo(HealthStatus.DOWN);
assertClean(sla);
assertClean(destination);
}
/*******************************************************************************
** Update persisted SLA state without running the scheduled calculator.
*******************************************************************************/
private void setStatus(QRecord plan, String status) throws Exception
{
catalog.update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("slaStatus", status));
}
/*******************************************************************************
** Fresh bounded input for each call.
*******************************************************************************/
private HealthCheckInput input()
{
return (new HealthCheckInput().withQInstance(QContext.getQInstance()).withDeadline(Instant.now().plusSeconds(5)));
}
/*******************************************************************************
** Construct the same per-plan check as the supplier.
*******************************************************************************/
private BackupsDestinationHealthCheck destination(QRecord plan)
{
return (new BackupsDestinationHealthCheck(plan.getValueInteger("id")));
}
/*******************************************************************************
** Validate the raw public message and flattened detail representation.
*******************************************************************************/
private void assertClean(HealthCheckOutcome outcome)
{
assertThat(outcome.getMessage()).doesNotContain(SECRET, "\n", "\r");
assertThat(outcome.getDetails().toJson()).doesNotContain(SECRET);
}
/*******************************************************************************
** Existing memory fake with read failures and assertions against side effects.
*******************************************************************************/
private static class ReadOnlyStorage extends InMemoryBackupStorage
{
private final String credential;
private boolean failList;
private boolean failHead;
private boolean interruptAfterList;
private int lists;
private int heads;
/*******************************************************************************
** Mimic the credential held by a host-provided storage adapter.
*******************************************************************************/
ReadOnlyStorage(String credential)
{
this.credential = credential;
}
/*******************************************************************************
** Record list calls and inject credential-bearing failures or cancellation.
*******************************************************************************/
@Override
public List<StoredObjectInfo> list(String prefix)
{
lists++;
if(failList)
{
throw (new IllegalStateException(credential));
}
if(interruptAfterList)
{
Thread.currentThread().interrupt();
}
return (super.list(prefix));
}
/*******************************************************************************
** Record head calls and inject credential-bearing failures.
*******************************************************************************/
@Override
public Optional<StoredObjectInfo> head(String key)
{
heads++;
if(failHead)
{
throw (new IllegalStateException(credential));
}
return (super.head(key));
}
/*******************************************************************************
** Health probes must not write, even to an unlocked bucket.
*******************************************************************************/
@Override
public OutputStream openWrite(String key, WriteOptions options)
{
throw (new AssertionError("Health probe wrote an object"));
}
/*******************************************************************************
** Health probes must not fetch backup contents.
*******************************************************************************/
@Override
public InputStream openRead(String key)
{
throw (new AssertionError("Health probe read object contents"));
}
/*******************************************************************************
** Health probes must not delete.
*******************************************************************************/
@Override
public DeleteResult deletePermanent(String key)
{
throw (new AssertionError("Health probe deleted an object"));
}
/*******************************************************************************
** Capability diagnostics can be active probes; health must not invoke them.
*******************************************************************************/
@Override
public Set<StorageCapability> capabilities()
{
throw (new AssertionError("Health probe requested capabilities"));
}
}
}
@@ -0,0 +1,132 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.health;
import java.util.Map;
import com.kingsrook.qbits.backups.VerifyTestSupport;
import com.kingsrook.qbits.backups.model.BackupsTables;
import com.kingsrook.qbits.backups.run.BackupCatalog;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.middleware.health.model.metadata.HealthCheckResult;
import com.kingsrook.qqq.middleware.health.model.metadata.HealthStatus;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** The health indicator: DOWN only when a critical, active plan is LATE, FAILING or
** MISSING_IN_STORAGE.
*******************************************************************************/
class BackupsHealthIndicatorTest
{
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
void setUp() throws Exception
{
VerifyTestSupport.setUp(3);
}
/*******************************************************************************
**
*******************************************************************************/
private void setStatus(QRecord plan, String status) throws Exception
{
new BackupCatalog().update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("slaStatus", status));
}
/*******************************************************************************
** UP with details when nothing critical is wrong, including a non-critical plan
** that is LATE; DOWN when a critical plan is LATE, FAILING or MISSING_IN_STORAGE.
*******************************************************************************/
@Test
void testUpAndDown() throws Exception
{
BackupsHealthIndicator indicator = new BackupsHealthIndicator();
assertThat(indicator.getName()).isEqualTo("backups");
HealthCheckResult empty = indicator.check(QContext.getQInstance());
assertThat(empty.getStatus()).isEqualTo(HealthStatus.UP);
assertThat(empty.getDetails()).containsEntry("activePlans", 0);
QRecord critical = VerifyTestSupport.plan(Map.of("critical", true));
QRecord ordinary = VerifyTestSupport.plan(Map.of("name", "Other", "planKey", "other"));
VerifyTestSupport.plan(Map.of("name", "Off", "planKey", "off", "isActive", false, "critical", true));
setStatus(critical, "OK");
setStatus(ordinary, "LATE");
HealthCheckResult up = indicator.check(QContext.getQInstance());
assertThat(up.getStatus()).isEqualTo(HealthStatus.UP);
assertThat(up.getDetails()).containsEntry("activePlans", 2).doesNotContainKey("criticalPlansWithProblems");
assertThat(up.getDurationMs()).isNotNull();
for(String status : new String[] { "LATE", "FAILING", "MISSING_IN_STORAGE" })
{
setStatus(critical, status);
HealthCheckResult down = indicator.check(QContext.getQInstance());
assertThat(down.getStatus()).as(status).isEqualTo(HealthStatus.DOWN);
assertThat(down.getDetails().get("criticalPlansWithProblems").toString()).contains("nightly: " + status);
}
setStatus(critical, "UNVERIFIED");
assertThat(indicator.check(QContext.getQInstance()).getStatus()).as("not verified is not down").isEqualTo(HealthStatus.UP);
}
/*******************************************************************************
** On a thread with no QQQ context (a health request) the indicator makes its own
** and cleans up.
*******************************************************************************/
@Test
void testWorksWithoutAContext() throws Exception
{
QInstance qInstance = QContext.getQInstance();
QContext.clear();
try
{
assertThat(new BackupsHealthIndicator().check(qInstance).getStatus()).isEqualTo(HealthStatus.UP);
assertThat(QContext.getQInstance()).isNull();
}
finally
{
QContext.init(qInstance, new com.kingsrook.qqq.backend.core.model.session.QSession());
}
}
/*******************************************************************************
** The indicator's own context runs as the core system user, which token-based
** authentication modules accept without a token.
*******************************************************************************/
@Test
void testOwnContextUsesSystemUserSession()
{
assertThat(BackupsHealthIndicator.systemSession(QContext.getQInstance()))
.isInstanceOf(com.kingsrook.qqq.backend.core.model.session.QSystemUserSession.class);
}
}
@@ -87,15 +87,10 @@ class BackupsMetricsTest
String empty = BackupsMetrics.renderCached();
assertThat(empty).doesNotContain("{instance_name");
VerifyTestSupport.clock.advance(Duration.ofSeconds(30));
assertThat(BackupsMetrics.renderCached()).as("cached").isSameAs(empty);
VerifyTestSupport.clock.advance(Duration.ofSeconds(31));
assertThat(BackupsMetrics.renderCached()).as("expired").isNotSameAs(empty);
/////////////////////////////////////////////////////////////
// a saved plan and a finished run clear the cache at once //
/////////////////////////////////////////////////////////////
VerifyTestSupport.backup(VerifyTestSupport.plan(Map.of()));
VerifyTestSupport.clock.advance(Duration.ofSeconds(30));
assertThat(BackupsMetrics.renderCached()).as("cached").isEqualTo(empty);
VerifyTestSupport.clock.advance(Duration.ofSeconds(31));
assertThat(BackupsMetrics.renderCached()).contains("plan=\"nightly\"");
assertThat(BackupsMetrics.escape("a\"b\\c\nd")).isEqualTo("a\\\"b\\\\c\\nd");
@@ -29,8 +29,6 @@ import com.kingsrook.qbits.backups.manifest.BackupManifest;
import com.kingsrook.qbits.backups.model.BackupsTables;
import com.kingsrook.qbits.backups.scope.PlanSettings;
import com.kingsrook.qbits.backups.scope.ScopeResolver;
import com.kingsrook.qbits.backups.scope.TableRole;
import com.kingsrook.qbits.backups.scope.TableRoles;
import com.kingsrook.qqq.backend.core.actions.async.AsyncJobCallback;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.jobs.plans.JobChunkScope;
@@ -39,11 +37,6 @@ import com.kingsrook.qqq.backend.core.jobs.progress.JobLogLevel;
import com.kingsrook.qqq.backend.core.jobs.progress.JobPhase;
import com.kingsrook.qqq.backend.core.jobs.progress.JobProgress;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
@@ -237,38 +230,6 @@ class BackupEngineTest
/*******************************************************************************
** A table on a backend with no query interface (a storage-only module, like
** qbit-edi's edi_preview_download) is left out of the scope, even when it is
** selected by name, and the run succeeds. Before the fix the reader queried it
** and the run FAILED with "Query is not implemented in this module".
*******************************************************************************/
@Test
void testUnqueryableTableIsExcluded() throws Exception
{
QInstance qInstance = QContext.getQInstance();
qInstance.addBackend(new QBackendMetaData().withName("storageOnly").withBackendType(StorageOnlyBackendModule.class));
qInstance.addTable(new QTableMetaData().withName("previewDownload").withBackendName("storageOnly").withPrimaryKeyField("id")
.withField(new QFieldMetaData("id", QFieldType.INTEGER)));
BackupRunResult result = run(plan(Map.of()));
assertThat(result.getErrorMessage()).isNull();
assertThat(result.getStatus()).isEqualTo("SUCCEEDED_WITH_WARNINGS");
BackupManifest manifest = BackupManifest.read(BackupsTestUtils.BACKUPS.getBytes(result.getManifestKey()));
assertThat(manifest.table("previewDownload")).isNull();
assertThat(manifest.table("blogPost")).isNotNull();
assertThat(TableRoles.roleOf(qInstance, qInstance.getTable("previewDownload"))).isEqualTo(TableRole.EXTERNAL);
assertThat(TableRoles.isQueryable(qInstance, qInstance.getTable("blogPost"))).isTrue();
ScopeResolver.ResolvedScope scope = new ScopeResolver(qInstance).resolve(PlanSettings.adHoc().with("scopeType", "TABLES").with("includeRoles", "DATA,EXTERNAL"),
List.of(new ScopeResolver.ScopeRule(true, "TABLE", "previewDownload", null, false, null), new ScopeResolver.ScopeRule(true, "ROLE", "EXTERNAL", null, false, null)));
assertThat(scope.names()).doesNotContain("previewDownload");
assertThat(scope.excluded()).contains(new ScopeResolver.ExcludedTable("previewDownload", ScopeResolver.NOT_QUERYABLE));
}
/*******************************************************************************
** Required snapshots fail before anything is written (no backend can open one yet).
*******************************************************************************/
@@ -391,14 +352,13 @@ class BackupEngineTest
BackupRunRequest request = BackupRunFactory.forPlan(plan(Map.of()), "MANUAL").withProgress(progress)
.withReader((table, filter, start, size, transaction, sink) ->
{
throw (new com.kingsrook.qqq.backend.core.exceptions.QException("https://storage-user:storage-password@host/?secret=storage-secret accessKey=storage-access Detail: Key (email)=(private-row) already exists."));
throw (new com.kingsrook.qqq.backend.core.exceptions.QException("https://storage-user:storage-password@host/?secret=storage-secret accessKey=storage-access row=private-row"));
});
BackupRunResult result = new BackupEngine().execute(request);
assertThat(result.getStatus()).isEqualTo("FAILED");
assertThat(progress.logs).contains("ERROR:" + result.getErrorMessage());
assertThat(progress.getSnapshot().getPhases()).anyMatch(p -> p.getStatus() == JobPhase.Status.FAILED);
assertThat(result.getErrorMessage()).startsWith("Backup failed: QException: https://[REDACTED]@host/");
assertThat(String.join("\n", progress.logs) + result.getErrorMessage() + result.getErrorDetail()).doesNotContain("storage-user", "storage-password", "storage-secret", "storage-access", "private-row");
assertThat(String.join("\n", progress.logs) + result.getErrorMessage()).doesNotContain("storage-user", "storage-password", "storage-secret", "storage-access", "private-row", "https://");
}
@@ -448,52 +408,7 @@ class BackupEngineTest
BackupRunResult result = new BackupEngine().execute(BackupRunFactory.forPlan(plan(Map.of()), "MANUAL").withProgress(progress).withStorage(storage));
assertThat(result.getStatus()).isEqualTo("FAILED");
assertThat(progress.logs).contains("ERROR:" + result.getErrorMessage());
assertThat(String.join("\n", progress.logs) + result.getErrorMessage() + result.getErrorDetail()).doesNotContain("endpoint-user", "endpoint-password", "access-secret", "secret-secret");
}
/*******************************************************************************
** A failed put records what an operator needs on the run row, with no secrets:
** error_message is "Backup failed: Class: message" and error_detail has the
** class, the cause chain and the root cause frames; the presigned signature,
** the access key id and the URL user-info are redacted, the object key is kept.
*******************************************************************************/
@Test
void testFailedPutRecordsSanitizedMessageAndDetail() throws Exception
{
var storage = new com.kingsrook.qbits.backups.storage.InMemoryBackupStorage()
{
/*******************************************************************************
** A MinIO-style failure with a presigned URL in the message.
*******************************************************************************/
@Override
public java.io.OutputStream openWrite(String key, com.kingsrook.qbits.backups.storage.WriteOptions options)
{
throw (new com.kingsrook.qbits.backups.storage.BackupStorageException(com.kingsrook.qbits.backups.storage.BackupStorageException.Reason.ACCESS_DENIED,
"AccessDenied on PUT https://minio-user:minio-pass@minio.galaxy.lan/kof22-backups/" + key
+ "?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20261016%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Signature=deadbeefcafe0123",
new IOException("XMinioStorageFull: Storage backend has reached its minimum free drive threshold. Authorization: AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/x, Signature=feedface")));
}
};
BackupRunResult result = new BackupEngine().execute(BackupRunFactory.forPlan(plan(Map.of()), "MANUAL").withStorage(storage));
assertThat(result.getStatus()).isEqualTo("FAILED");
QRecord run = new BackupCatalog().get(BackupsTables.RUN, result.getRunId()).orElseThrow();
String message = run.getValueString("errorMessage");
String detail = run.getValueString("errorDetail");
assertThat(message).isEqualTo(result.getErrorMessage());
assertThat(detail).isEqualTo(result.getErrorDetail());
assertThat(message).startsWith("Backup failed: BackupStorageException: AccessDenied on PUT https://[REDACTED]@minio.galaxy.lan/kof22-backups/qqq-backups/testapp/nightly/")
.doesNotContain("see server logs");
assertThat(detail).startsWith("com.kingsrook.qbits.backups.storage.BackupStorageException: AccessDenied on PUT")
.contains("\nCaused by: java.io.IOException: XMinioStorageFull: Storage backend has reached its minimum free drive threshold.")
.contains("X-Amz-Signature=[REDACTED]")
.contains("Root cause stack (top ")
.contains("\tat ");
assertThat(detail.length()).isLessThanOrEqualTo(FailureText.MAX_DETAIL_CHARS);
assertThat(message + detail).doesNotContain("deadbeefcafe0123", "AKIAIOSFODNN7EXAMPLE", "minio-user", "minio-pass", "feedface");
assertThat(String.join("\n", progress.logs) + result.getErrorMessage()).doesNotContain("endpoint-user", "endpoint-password", "access-secret", "secret-secret", "https://");
}
@@ -1,186 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.run;
import java.io.IOException;
import java.sql.SQLException;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** Unit tests of the failure text sanitizer, summary and detail.
*******************************************************************************/
class FailureTextTest
{
/*******************************************************************************
** Null and plain text pass through; object keys and bucket names are kept.
*******************************************************************************/
@Test
void testPlainTextIsKept()
{
assertThat(FailureText.sanitize(null)).isNull();
String plain = "Query is not implemented in this module: PreviewDownloadBackend (bucket kof22-backups, key qqq-backups/kof22/nightly/2026/10/16/run/tables/a/part-00001.jsonl.gz)";
assertThat(FailureText.sanitize(plain)).isEqualTo(plain);
assertThat(FailureText.sanitize("objectKey=qqq-backups/x/y.gz bucket=kof22-backups")).isEqualTo("objectKey=qqq-backups/x/y.gz bucket=kof22-backups");
}
/*******************************************************************************
** URL user-info and presigned query parameters are redacted; host and path kept.
*******************************************************************************/
@Test
void testUrls()
{
assertThat(FailureText.sanitize("PUT https://user:pass@minio.lan/bucket/a/b.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2Fx&X-Amz-Signature=abc123&partNumber=2 failed"))
.isEqualTo("PUT https://[REDACTED]@minio.lan/bucket/a/b.gz?X-Amz-Algorithm=[REDACTED]&X-Amz-Credential=[REDACTED]&X-Amz-Signature=[REDACTED]&partNumber=2 failed");
assertThat(FailureText.sanitize("jdbc:postgresql://app:hunter2@db:5432/app")).isEqualTo("jdbc:postgresql://[REDACTED]@db:5432/app");
assertThat(FailureText.sanitize("https://h/p?Signature=s1&Expires=1&AWSAccessKeyId=k&access_token=t&key=v"))
.isEqualTo("https://h/p?Signature=[REDACTED]&Expires=1&AWSAccessKeyId=[REDACTED]&access_token=[REDACTED]&key=[REDACTED]");
}
/*******************************************************************************
** key=value and JSON secrets, Authorization and Bearer values.
*******************************************************************************/
@Test
void testAssignmentsAndHeaders()
{
assertThat(FailureText.sanitize("password=p1 secret=\"s 2\" token='t3' key=k4 accessKey=a5 secretKey=s6 apiKey=x7 db.password = p8, other=ok"))
.isEqualTo("password=[REDACTED] secret=[REDACTED] token=[REDACTED] key=[REDACTED] accessKey=[REDACTED] secretKey=[REDACTED] apiKey=[REDACTED] db.password = [REDACTED], other=ok");
assertThat(FailureText.sanitize("{\"clientSecret\": \"abc\", \"name\": \"keep\"}")).isEqualTo("{\"clientSecret\": \"[REDACTED]\", \"name\": \"keep\"}");
assertThat(FailureText.sanitize("Authorization: Bearer eyJhbGciOi.payload.sig\nnext line")).isEqualTo("Authorization: [REDACTED]\nnext line");
assertThat(FailureText.sanitize("sent Bearer abcdefghijk to the API")).isEqualTo("sent Bearer [REDACTED] to the API");
assertThat(FailureText.sanitize("id AKIAIOSFODNN7EXAMPLE and ASIAIOSFODNN7EXAMPLE")).isEqualTo("id [REDACTED] and [REDACTED]");
}
/*******************************************************************************
** PEM blocks (terminated or not) and Postgres row values.
*******************************************************************************/
@Test
void testPemAndRowValues()
{
assertThat(FailureText.sanitize("bad key -----BEGIN PRIVATE KEY-----\nMIIabc\n-----END PRIVATE KEY----- here")).isEqualTo("bad key [REDACTED PEM] here");
assertThat(FailureText.sanitize("cut -----BEGIN RSA PRIVATE KEY-----\nMIIabc")).isEqualTo("cut [REDACTED PEM]");
assertThat(FailureText.sanitize("duplicate key value violates unique constraint \"u_email\" Key (email)=(ada@example.com) already exists."))
.isEqualTo("duplicate key value violates unique constraint \"u_email\" Key (email)=([REDACTED]) already exists.");
assertThat(FailureText.sanitize("ERROR: null value in column \"x\"\n Detail: Failing row contains (1, ada, secret-row).\n Where: x"))
.isEqualTo("ERROR: null value in column \"x\"\n Detail: [REDACTED]\n Where: x");
}
/*******************************************************************************
** The summary is short, sanitized and bounded; no message falls back to the class.
*******************************************************************************/
@Test
void testSummary()
{
assertThat(FailureText.summary(new IllegalStateException("Query is not implemented in this module: PreviewDownloadBackend")))
.isEqualTo("Backup failed: IllegalStateException: Query is not implemented in this module: PreviewDownloadBackend");
assertThat(FailureText.summary(new NullPointerException())).isEqualTo("Backup failed: NullPointerException");
assertThat(FailureText.summary("Could not create the run record", new IOException(" password=x "))).isEqualTo("Could not create the run record: IOException: password=[REDACTED]");
assertThat(FailureText.summary(new IOException("x".repeat(2000)))).hasSize("Backup failed: IOException: ".length() + FailureText.MAX_SUMMARY_MESSAGE_CHARS + 3);
}
/*******************************************************************************
** The detail has the class, the cause chain and the root cause frames, bounded.
*******************************************************************************/
@Test
void testDetail()
{
SQLException root = new SQLException("Key (id)=(4242xyz) already exists.");
IOException middle = new IOException("write failed password=pwsecret", root);
RuntimeException top = new RuntimeException("Backup of t failed", middle);
String detail = FailureText.detail(top);
assertThat(detail).startsWith("java.lang.RuntimeException: Backup of t failed\nCaused by: java.io.IOException: write failed password=[REDACTED]\nCaused by: java.sql.SQLException: Key (id)=([REDACTED]) already exists.\nRoot cause stack (top ");
assertThat(detail).contains("\tat com.kingsrook.qbits.backups.run.FailureTextTest.testDetail(FailureTextTest.java:");
assertThat(detail.split("\n\tat ")).hasSize(FailureText.ROOT_CAUSE_FRAMES + 1);
assertThat(detail).doesNotContain("4242xyz", "pwsecret");
IOException noFrames = new IOException((String) null);
noFrames.setStackTrace(new StackTraceElement[0]);
assertThat(FailureText.detail(noFrames)).isEqualTo("java.io.IOException");
String huge = FailureText.detail(new IOException("y".repeat(20_000)));
assertThat(huge).hasSize(FailureText.MAX_DETAIL_CHARS).endsWith("... (truncated)");
}
/*******************************************************************************
** A cause cycle does not loop.
*******************************************************************************/
@Test
void testCauseCycle()
{
CyclicException a = new CyclicException("a");
CyclicException b = new CyclicException("b");
a.setCause(b);
b.setCause(a);
assertThat(FailureText.detail(a)).startsWith("com.kingsrook.qbits.backups.run.FailureTextTest$CyclicException: a\nCaused by: com.kingsrook.qbits.backups.run.FailureTextTest$CyclicException: b\nRoot cause stack");
}
/***************************************************************************
** An exception whose cause can be set to form a cycle.
***************************************************************************/
private static class CyclicException extends Exception
{
private Throwable cause;
/*******************************************************************************
** Constructor.
*******************************************************************************/
CyclicException(String message)
{
super(message);
}
/*******************************************************************************
** Set the cause.
*******************************************************************************/
void setCause(Throwable cause)
{
this.cause = cause;
}
/*******************************************************************************
** The cause.
*******************************************************************************/
@Override
public synchronized Throwable getCause()
{
return (cause);
}
}
}
@@ -1,86 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.run;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
/*******************************************************************************
** A storage-only backend module (like qbit-edi's PreviewDownloadBackend): it
** streams objects but has no query interface, so a table on it cannot be read.
*******************************************************************************/
public class StorageOnlyBackendModule implements QBackendModuleInterface, QStorageInterface
{
public static final String TYPE = "test-storage-only";
static
{
QBackendModuleDispatcher.registerBackendModule(new StorageOnlyBackendModule());
}
/*******************************************************************************
** The backend type.
*******************************************************************************/
@Override
public String getBackendType()
{
return (TYPE);
}
/*******************************************************************************
** Storage only.
*******************************************************************************/
@Override
public QStorageInterface getStorageInterface()
{
return (this);
}
/*******************************************************************************
** Nothing is stored.
*******************************************************************************/
@Override
public InputStream getInputStream(StorageInput input)
{
return (new ByteArrayInputStream(new byte[0]));
}
/*******************************************************************************
** Writes are discarded.
*******************************************************************************/
@Override
public OutputStream createOutputStream(StorageInput input)
{
return (new ByteArrayOutputStream());
}
}