Compare commits
13
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fa9c4c1774 | ||
|
|
c8ba79f55b
|
||
|
|
7830226982 | ||
|
|
b1f258a3d5
|
||
|
|
3df24cda5a | ||
|
|
987c20899b
|
||
|
|
14bb6fa1b4 | ||
|
|
b1ff70d7d3
|
||
|
|
cda334f2fe | ||
|
|
590b132fb6
|
||
|
|
f0a8c5000e | ||
|
|
8f3f9ec60a
|
||
|
|
cdaf218201 |
No files matched your search
+57
-11
@@ -1,14 +1,60 @@
|
||||
version: 2.1
|
||||
setup: true
|
||||
|
||||
orbs:
|
||||
qqq-orb: kingsrook/qqq-orb@0.8
|
||||
qqq-orb: kingsrook/qqq-orb@0.6.9
|
||||
|
||||
commands:
|
||||
configure-package-install:
|
||||
steps:
|
||||
- run:
|
||||
name: Keep package installation unattended
|
||||
command: |
|
||||
# These build machines are discarded after the job.
|
||||
sudo install -d /etc/needrestart/conf.d
|
||||
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
|
||||
|
||||
workflows:
|
||||
setup:
|
||||
jobs:
|
||||
- qqq-orb/generate_pipeline:
|
||||
orb_version: '0.8'
|
||||
filters:
|
||||
branches:
|
||||
ignore: automation/qbit-rc-pending
|
||||
tags:
|
||||
only: /^v\d+\.\d+\.\d+$/
|
||||
test_only:
|
||||
jobs:
|
||||
- qqq-orb/mvn_test_only:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
filters:
|
||||
branches:
|
||||
ignore: /(develop|release\/.*)/
|
||||
|
||||
publish_snapshot:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
branch_type: snapshot
|
||||
filters:
|
||||
branches:
|
||||
only: [develop]
|
||||
|
||||
publish_release_candidate:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
branch_type: release_candidate
|
||||
filters:
|
||||
branches:
|
||||
only: [/release\/.*/]
|
||||
|
||||
publish_release:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
branch_type: release
|
||||
filters:
|
||||
branches:
|
||||
ignore: /.*/
|
||||
tags:
|
||||
only: [/^v\d+\.\d+\.\d+$/]
|
||||
@@ -1,7 +0,0 @@
|
||||
name: build
|
||||
on:
|
||||
push:
|
||||
jobs:
|
||||
maven:
|
||||
uses: self:KofTwentyTwo/ci-workflows/.gitea/workflows/maven.yaml@v1
|
||||
secrets: inherit
|
||||
@@ -1,23 +0,0 @@
|
||||
name: CodeQL
|
||||
'on':
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- develop
|
||||
- release/**
|
||||
- hotfix/**
|
||||
pull_request: {}
|
||||
schedule:
|
||||
- cron: 41 16 * * 0
|
||||
workflow_dispatch: {}
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
security-events: write
|
||||
packages: read
|
||||
concurrency:
|
||||
group: codeql-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
jobs:
|
||||
analyze:
|
||||
uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22
|
||||
@@ -1,74 +1,25 @@
|
||||
# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
|
||||
# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
|
||||
# the release line and the release/* push runs CircleCI publish_release_candidate.
|
||||
name: Release candidate
|
||||
'on':
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: Plan a cut without changing release state
|
||||
description: Print the plan without merging or creating branches
|
||||
type: boolean
|
||||
default: false
|
||||
finalize:
|
||||
description: Retry finalization of an existing cut; do not cut another RC
|
||||
type: boolean
|
||||
default: false
|
||||
recover:
|
||||
description: Recover the same published RC after a deployment timeout; never
|
||||
reupload
|
||||
type: boolean
|
||||
default: false
|
||||
circle_build:
|
||||
description: Original Circle publisher build number (recovery only)
|
||||
type: string
|
||||
default: ''
|
||||
merge_sha:
|
||||
description: Existing cut merge_sha (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
bump_sha:
|
||||
description: Existing cut bump_sha (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
branch:
|
||||
description: Existing cut branch (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
version:
|
||||
description: Existing cut version (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
pull_request:
|
||||
types:
|
||||
- closed
|
||||
branches:
|
||||
- develop
|
||||
repository_dispatch:
|
||||
types:
|
||||
- qbit-rc-published
|
||||
types: [closed]
|
||||
branches: [develop]
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
actions: read
|
||||
checks: read
|
||||
|
||||
jobs:
|
||||
release-rc:
|
||||
if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
|
||||
|| (github.event_name == 'pull_request' && github.event.pull_request.merged
|
||||
== true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
|
||||
'release:rc'))
|
||||
uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
|
||||
if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
|
||||
uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
|
||||
with:
|
||||
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
|
||||
}}
|
||||
finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
|
||||
== 'workflow_dispatch' && inputs.finalize) }}
|
||||
recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
|
||||
}}
|
||||
circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
|
||||
|| '' }}
|
||||
merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
|
||||
|| inputs.merge_sha || '' }}
|
||||
bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
|
||||
|| inputs.bump_sha || '' }}
|
||||
branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
|
||||
|| inputs.branch || '' }}
|
||||
version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
|
||||
|| inputs.version || '' }}
|
||||
secrets: inherit
|
||||
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
|
||||
-16
@@ -1,16 +0,0 @@
|
||||
profile: qbit
|
||||
orb_version: '0.8'
|
||||
default_branch: develop
|
||||
java_version: '21'
|
||||
browsers: true
|
||||
api_checks: false
|
||||
docs_only: false
|
||||
release_on_main: false
|
||||
release_tag_policy: stable
|
||||
test_branch_policy: non-release
|
||||
publish:
|
||||
snapshot: true
|
||||
release_candidate: true
|
||||
release: true
|
||||
asciidoc: false
|
||||
release_completion: true
|
||||
@@ -5,7 +5,6 @@ All notable changes to this project are documented here. The format follows Keep
|
||||
## [0.1.0] - Unreleased
|
||||
|
||||
### Added
|
||||
- Central `backups.sla` and per-active-plan `backups.destination.<planId>` health checks, with bounded read-only destination diagnostics and secret-free status details.
|
||||
- Always-durable backup, resume, scheduled backup, restore and verify processes with conservative `NONE` idempotency and stable backup identities on explicit job retry.
|
||||
- Weighted backup phases, table/file counts, stored bytes, safe per-table/artifact logs, warnings, failures and resume notices through QQQ `JobProgress`.
|
||||
- Cooperative job cancellation during backup pages, media copying, restore pages and verification scans; local media-reference spooling keeps phase separation from growing heap usage across tables.
|
||||
@@ -16,11 +15,6 @@ All notable changes to this project are documented here. The format follows Keep
|
||||
- Repository scaffold: `BackupsQBitConfig` and `BackupsQBitProducer` (table prefix `backups_`, permissions `backups.*`), build, CI and repository standard files.
|
||||
|
||||
### Changed
|
||||
- Tables whose backend module has no query interface (storage-only modules such as qbit-edi `edi_preview_download`) are classified EXTERNAL and always left out of a backup with the reason `NOT_QUERYABLE`, even when selected by name or role; the Run backup review lists them. Before, such a table was read as DATA and the whole run FAILED with "Query is not implemented in this module".
|
||||
- Failed runs persist an actionable `error_message` ("Backup failed: Class: message") and fill `error_detail` with the exception class, the cause chain and the top root-cause frames (bounded to 8 KB). Both go through `FailureText.sanitize`, which redacts URL user-info, presigned query parameters, key=value secrets, Authorization/Bearer values, AWS access key ids, PEM blocks and Postgres row values while keeping object keys and bucket names.
|
||||
- Multi-node deployments: the cached Prometheus text (`BackupsMetrics`) is registered with QQQ `ClusterCaches` as `qbit-backups.metrics` and cleared on every node when a run finishes or a plan is inserted, updated or deleted; the 60 s TTL remains the safety net. `SlaChecker`'s manifest-head and bucket-audit throttles and `BackupsRuntime`'s process-permission map are registered as node-local, with reasons. `BackupsClusterCachesTest` audits the package for unregistered static caches and covers a remote invalidation.
|
||||
- Builds against QQQ `4.1.0-SNAPSHOT` (`qqq.snapshot.version`) for `ClusterCaches` until the next QQQ RC ships; a release must pin that RC.
|
||||
- Read the QBit metadata version from the Maven artifact version so release builds report their released version.
|
||||
- Target QQQ `4.1.0-RC.8` (job progress).
|
||||
- Persist controlled backup failure messages instead of untrusted provider exception text.
|
||||
- Pin QQQ to released 4.1.0-RC.7 and replace the temporary help loader with core HelpBundles registration.
|
||||
@@ -32,6 +26,3 @@ All notable changes to this project are documented here. The format follows Keep
|
||||
- New manifests identify `qqq-table-schema-v1` and carry core schema descriptors; restore and content verification still accept pre-RC.6 manifests with the original fingerprint algorithm.
|
||||
- Preserve primary-key ordering, exclusive resume cursors, bounded pages and caller-owned transactions; reject non-orderable primary keys and scope filters with skip/limit.
|
||||
- Require the core `backups.restore` permission during Review when using a core restore writer, including anonymous-mode hosts; preserve supplied timestamps through `RestoreWriteFlag`.
|
||||
|
||||
### Removed
|
||||
- The legacy `BackupsHealthIndicator` and its optional `qqq-middleware-health` dependency; central health registration replaces manual indicator configuration.
|
||||
@@ -43,11 +43,6 @@ Follow the QQQ [adoption checklist](https://github.com/QRun-IO/qqq/blob/develop/
|
||||
See [the RC.6 migration notes](docs/RC6-MIGRATION.md) for fingerprint compatibility,
|
||||
paging guarantees and restore permissions.
|
||||
|
||||
Health checks use the core health API released in QQQ `4.1.0-RC.8`.
|
||||
The producer registers `backups.sla` and a read-only destination check for each
|
||||
active plan (`backups.destination.<planId>`). With dependencies cached locally,
|
||||
run `./mvnw -o verify`; no enforcer skip flags are required.
|
||||
|
||||
## In-app help
|
||||
|
||||
The producer registers [help/qbit-backups.yaml](src/main/resources/help/qbit-backups.yaml)
|
||||
@@ -100,23 +95,3 @@ The backup SLA checker still owns expected intervals and grace periods.
|
||||
RC.5 requires hyphenated alert names: `backups.run.missing-in-storage` and
|
||||
`backups.plan.owner-invalid` replace the former underscore spellings, including
|
||||
their deduplication keys. Update host routing rules that used those spellings.
|
||||
|
||||
## Health checks
|
||||
|
||||
The producer registers these central checks under component `backups`, category `STORAGE`:
|
||||
|
||||
| Name | Kind | Interval | Timeout |
|
||||
| --- | --- | --- | --- |
|
||||
| `backups.sla` | DEPENDENCY | 900s (15 minutes) | 5000ms |
|
||||
| `backups.destination.<planId>` | DEPENDENCY | 86400s (24 hours) | 30000ms |
|
||||
|
||||
The SLA check reads persisted `slaStatus`: critical active plans that are LATE,
|
||||
FAILING or MISSING_IN_STORAGE yield DOWN; non-critical failures yield DEGRADED.
|
||||
Only UNVERIFIED/UNKNOWN plans yield UNKNOWN; no active plans yield UP. Details
|
||||
contain counts per SLA status. The scheduled `backupCheckSla` still computes SLA state.
|
||||
|
||||
Destination checks are supplied for active plans and reuse `DestinationProbe` in
|
||||
read-only list/head mode, including on unlocked buckets. They never write, delete,
|
||||
or resolve prune credentials. The Test destination action retains its write tests.
|
||||
Storage calls are bounded by the central runner, with deadline checks between calls.
|
||||
Override intervals in `health_check` and disable checks through `health_check.enabled`.
|
||||
@@ -18,10 +18,9 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
|
||||
<properties>
|
||||
<qqq.version>4.1.0-RC.26</qqq.version> <!-- feeds qbit-build-parent's qqq-bom-pom import -->
|
||||
<qbit.guardrails.enforce>true</qbit.guardrails.enforce>
|
||||
<qbit.guardrails.strictUiApiRoutes>true</qbit.guardrails.strictUiApiRoutes>
|
||||
<revision>0.1.0-SNAPSHOT</revision>
|
||||
<revision>0.1.0-RC.7</revision>
|
||||
<maven.compiler.source>21</maven.compiler.source>
|
||||
<maven.compiler.target>21</maven.compiler.target>
|
||||
<jacoco.version>0.8.15</jacoco.version>
|
||||
@@ -30,7 +29,7 @@
|
||||
<parent>
|
||||
<groupId>com.kingsrook</groupId>
|
||||
<artifactId>qbit-build-parent</artifactId>
|
||||
<version>2.2.1</version>
|
||||
<version>2.1.0</version>
|
||||
</parent>
|
||||
|
||||
<groupId>com.kingsrook.qbits</groupId>
|
||||
@@ -75,6 +74,13 @@
|
||||
<artifactId>qqq-backend-core</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- optional: the BackupsHealthIndicator (apps without qqq-middleware-health never load it) -->
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
<artifactId>qqq-middleware-health</artifactId>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
|
||||
<!-- optional: ESB events (apps without qqq-esb never load it) -->
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
@@ -87,7 +93,7 @@
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qbits</groupId>
|
||||
<artifactId>qbit-object-storage</artifactId>
|
||||
<version>0.1.0-RC.10</version>
|
||||
<version>0.1.0-RC.2</version>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
|
||||
@@ -107,7 +113,7 @@
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
<artifactId>qqq-frontend-next</artifactId>
|
||||
<version>1.0.0-RC.38</version>
|
||||
<version>1.0.0-RC.14</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
@@ -125,33 +131,11 @@
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<resources>
|
||||
<!-- help bundles carry the artifact version (QQQ help bundle provenance) -->
|
||||
<resource>
|
||||
<directory>src/main/resources</directory>
|
||||
<filtering>true</filtering>
|
||||
<includes>
|
||||
<include>help/*.yaml</include>
|
||||
<include>qbit-backups.properties</include>
|
||||
</includes>
|
||||
</resource>
|
||||
<resource>
|
||||
<directory>src/main/resources</directory>
|
||||
<filtering>false</filtering>
|
||||
<excludes>
|
||||
<exclude>help/*.yaml</exclude>
|
||||
<exclude>qbit-backups.properties</exclude>
|
||||
</excludes>
|
||||
</resource>
|
||||
</resources>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<configuration>
|
||||
<systemPropertyVariables combine.children="append">
|
||||
<qbit.artifactVersion>${project.version}</qbit.artifactVersion>
|
||||
</systemPropertyVariables>
|
||||
<!-- the MinIO test container default credentials, resolved through ${env.X} refs like production -->
|
||||
<environmentVariables>
|
||||
<STORAGE_TEST_ACCESS_KEY>minioadmin</STORAGE_TEST_ACCESS_KEY>
|
||||
@@ -217,7 +201,7 @@
|
||||
<activeByDefault>true</activeByDefault>
|
||||
</activation>
|
||||
<properties>
|
||||
<qqq.snapshot.version>4.1.0-RC.26</qqq.snapshot.version>
|
||||
<qqq.snapshot.version>4.1.0-RC.8</qqq.snapshot.version>
|
||||
</properties>
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
|
||||
@@ -17,17 +17,6 @@
|
||||
package com.kingsrook.qbits.backups;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.Properties;
|
||||
import com.kingsrook.qbits.backups.health.BackupsDestinationHealthChecks;
|
||||
import com.kingsrook.qbits.backups.health.BackupsSlaHealthCheck;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCategory;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCheckKind;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCheckSupplierMetaData;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundles;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaDataProducer;
|
||||
@@ -41,41 +30,13 @@ public class BackupsQBitProducer implements QBitMetaDataProducer<BackupsQBitConf
|
||||
{
|
||||
public static final String GROUP_ID = "com.kingsrook.qbits";
|
||||
public static final String ARTIFACT_ID = "qbit-backups";
|
||||
public static final String VERSION = loadVersion();
|
||||
public static final String VERSION = "0.1.0";
|
||||
|
||||
private BackupsQBitConfig qBitConfig;
|
||||
private String namespace;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the artifact version from the Maven-filtered resource.
|
||||
*******************************************************************************/
|
||||
private static String loadVersion()
|
||||
{
|
||||
try(InputStream inputStream = BackupsQBitProducer.class.getClassLoader().getResourceAsStream("qbit-backups.properties"))
|
||||
{
|
||||
if(inputStream == null)
|
||||
{
|
||||
throw (new IllegalStateException("Missing qbit-backups.properties; build with Maven to generate the artifact version"));
|
||||
}
|
||||
Properties properties = new Properties();
|
||||
properties.load(inputStream);
|
||||
String version = properties.getProperty("version");
|
||||
if(version == null || version.isBlank() || version.contains("${"))
|
||||
{
|
||||
throw (new IllegalStateException("Missing or unfiltered version in qbit-backups.properties; build with Maven"));
|
||||
}
|
||||
return (version);
|
||||
}
|
||||
catch(IOException e)
|
||||
{
|
||||
throw (new IllegalStateException("Could not read artifact version from qbit-backups.properties", e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for the config.
|
||||
*******************************************************************************/
|
||||
@@ -203,13 +164,6 @@ public class BackupsQBitProducer implements QBitMetaDataProducer<BackupsQBitConf
|
||||
// Registration is metadata-only; core guards startup sync when helpContent is absent.
|
||||
HelpBundles.register(qInstance, "help/qbit-backups.yaml");
|
||||
|
||||
HealthChecksMetaData.ofOrWithNew(qInstance)
|
||||
.withCheck(new HealthCheckMetaData().withName(BackupsSlaHealthCheck.NAME).withLabel("Backup SLA")
|
||||
.withComponent("backups").withKind(HealthCheckKind.DEPENDENCY).withCategory(HealthCategory.STORAGE)
|
||||
.withIntervalSeconds(900).withCheck(new QCodeReference(BackupsSlaHealthCheck.class)))
|
||||
.withSupplier(new HealthCheckSupplierMetaData().withNamePrefix(BackupsDestinationHealthChecks.PREFIX)
|
||||
.withComponent("backups").withSupplier(new QCodeReference(BackupsDestinationHealthChecks.class)));
|
||||
|
||||
// A parent app only lists the apps that are its children, so attach ourselves (for example Admin, then Backups).
|
||||
com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData parent = qBitConfig.getParentAppName() == null ? null : qInstance.getApp(qBitConfig.getParentAppName());
|
||||
if(parent != null && (parent.getChildren() == null || parent.getChildren().stream().noneMatch(child -> app.getName().equals(child.getName()))))
|
||||
|
||||
@@ -25,7 +25,6 @@ import com.kingsrook.qbits.backups.alerts.BackupAlertSink;
|
||||
import com.kingsrook.qbits.backups.alerts.NoOpBackupAlertSink;
|
||||
import com.kingsrook.qbits.backups.storage.BackupStorage;
|
||||
import com.kingsrook.qbits.backups.storage.BackupStorageException;
|
||||
import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@@ -35,8 +34,7 @@ import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
|
||||
*******************************************************************************/
|
||||
public final class BackupsRuntime
|
||||
{
|
||||
private static final Map<String, String> PROCESS_PERMISSIONS = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
|
||||
"built from the qbit config at startup, the same on every node");
|
||||
private static final Map<String, String> PROCESS_PERMISSIONS = new ConcurrentHashMap<>();
|
||||
|
||||
private static volatile BackupsQBitConfig config;
|
||||
private static volatile Clock clock = Clock.systemUTC();
|
||||
|
||||
@@ -69,11 +69,10 @@ public final class BackupsHomeView
|
||||
public static String render(BackupsHomeData data, String pathPrefix)
|
||||
{
|
||||
StringBuilder html = new StringBuilder("<div class=\"backups-home\" style=\"" + ROOT + "\">");
|
||||
List<BackupsSetupStatus.Item> setupItems = BackupsSetupStatus.load(data.plans());
|
||||
planStatus(html, data, pathPrefix);
|
||||
setup(html, setupItems, pathPrefix);
|
||||
setup(html, data, pathPrefix);
|
||||
html.append("<div style=\"").append(GRID).append("\">");
|
||||
needsAttention(html, data, setupItems, pathPrefix);
|
||||
needsAttention(html, data, pathPrefix);
|
||||
latestTestRestore(html, data, pathPrefix);
|
||||
recentActivity(html, data, pathPrefix);
|
||||
trend(html, data);
|
||||
@@ -145,8 +144,9 @@ public final class BackupsHomeView
|
||||
** The Setup card: what a working backup needs, whether each part is in place,
|
||||
** and the next step, linked to its table.
|
||||
*******************************************************************************/
|
||||
private static void setup(StringBuilder html, List<BackupsSetupStatus.Item> items, String prefix)
|
||||
private static void setup(StringBuilder html, BackupsHomeData data, String prefix)
|
||||
{
|
||||
List<BackupsSetupStatus.Item> items = BackupsSetupStatus.load(data.plans());
|
||||
open(html, "Setup");
|
||||
long todo = items.stream().filter(i -> i.state() == BackupsSetupStatus.State.TODO).count();
|
||||
html.append("<p style=\"margin:0 0 .6rem;font-size:.9rem;\">")
|
||||
@@ -185,27 +185,23 @@ public final class BackupsHomeView
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Needs attention: every Setup item that needs setup (the same list the Setup
|
||||
** card counts), plus plans in a problem state and recent failed runs, so the two
|
||||
** cards never disagree.
|
||||
** Needs attention.
|
||||
*******************************************************************************/
|
||||
private static void needsAttention(StringBuilder html, BackupsHomeData data, List<BackupsSetupStatus.Item> setupItems, String prefix)
|
||||
private static void needsAttention(StringBuilder html, BackupsHomeData data, String prefix)
|
||||
{
|
||||
open(html, "Needs attention");
|
||||
List<BackupsSetupStatus.Item> setupTodo = setupItems.stream().filter(i -> i.state() == BackupsSetupStatus.State.TODO).toList();
|
||||
List<QRecord> plans = data.plansNeedingAttention();
|
||||
if(setupTodo.isEmpty() && plans.isEmpty() && data.failedRuns().isEmpty())
|
||||
List<QRecord> plans = data.plansNeedingAttention();
|
||||
if(data.plans().isEmpty())
|
||||
{
|
||||
empty(html, data.plans().isEmpty() ? "Nothing to check yet: no plans." : "Nothing needs attention.");
|
||||
empty(html, "Nothing to check yet: no plans.");
|
||||
}
|
||||
else if(plans.isEmpty() && data.failedRuns().isEmpty())
|
||||
{
|
||||
empty(html, "Nothing needs attention.");
|
||||
}
|
||||
else
|
||||
{
|
||||
html.append("<ul style=\"").append(ITEMS).append("\">");
|
||||
for(BackupsSetupStatus.Item item : setupTodo)
|
||||
{
|
||||
html.append("<li>Setup: <a style=\"").append(LINK).append("\" href=\"").append(esc(prefix + "/" + item.tableName())).append("\">").append(esc(item.label())).append("</a>: ")
|
||||
.append(esc(item.detail())).append("</li>");
|
||||
}
|
||||
for(QRecord plan : plans)
|
||||
{
|
||||
html.append("<li>").append(recordLink(prefix, BackupsTables.PLAN, plan.getValue("id"), plan.getValueString("name")))
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.health;
|
||||
|
||||
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.plan.DestinationProbe;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCheck;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCheckInput;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthDetails;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reuse the destination diagnostic in read-only mode; never publish its text.
|
||||
*******************************************************************************/
|
||||
public class BackupsDestinationHealthCheck implements HealthCheck
|
||||
{
|
||||
private final Integer planId;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Retain only the id; destination and active state are read on each run.
|
||||
*******************************************************************************/
|
||||
public BackupsDestinationHealthCheck(Integer planId)
|
||||
{
|
||||
this.planId = planId;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A removed or disabled plan can remain registered until supplier refresh.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public HealthCheckOutcome run(HealthCheckInput input)
|
||||
{
|
||||
boolean ownsContext = QContext.getQInstance() == null;
|
||||
if(ownsContext)
|
||||
{
|
||||
QContext.init(input.getQInstance(), new QSystemUserSession(input.getQInstance()));
|
||||
}
|
||||
try
|
||||
{
|
||||
if(BackupsSlaHealthCheck.expired(input))
|
||||
{
|
||||
return (HealthCheckOutcome.down("Backup destination check timed out"));
|
||||
}
|
||||
QRecord plan = new BackupCatalog().get(BackupsTables.PLAN, planId).orElse(null);
|
||||
if(plan == null || !Boolean.TRUE.equals(plan.getValueBoolean("isActive")))
|
||||
{
|
||||
return (HealthCheckOutcome.unknown("Backup plan is no longer active"));
|
||||
}
|
||||
DestinationProbe.Report report = DestinationProbe.probeReadOnly(plan, input.getDeadline());
|
||||
if(BackupsSlaHealthCheck.expired(input))
|
||||
{
|
||||
return (HealthCheckOutcome.down("Backup destination check timed out"));
|
||||
}
|
||||
return ((report.ok() ? HealthCheckOutcome.up("Backup destination is readable") : HealthCheckOutcome.down("Backup destination probe failed"))
|
||||
.withDetails(new HealthDetails().withValue("planId", planId).withValue("readOnly", true)
|
||||
.withValue("checks", report.checks().size()).withValue("failedChecks", report.checks().stream().filter(check -> !check.ok()).count())));
|
||||
}
|
||||
catch(QException | RuntimeException e)
|
||||
{
|
||||
return (HealthCheckOutcome.down("Backup destination could not be checked"));
|
||||
}
|
||||
finally
|
||||
{
|
||||
if(ownsContext)
|
||||
{
|
||||
QContext.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,77 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.health;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCategory;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCheckKind;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCheckSupplier;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Discover one daily, read-only destination check per active backup plan.
|
||||
*******************************************************************************/
|
||||
public class BackupsDestinationHealthChecks implements HealthCheckSupplier
|
||||
{
|
||||
public static final String PREFIX = "backups.destination";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve from current catalog data, without opening any storage connections.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<HealthCheckMetaData> supply(QInstance qInstance) throws QException
|
||||
{
|
||||
boolean ownsContext = QContext.getQInstance() == null;
|
||||
if(ownsContext)
|
||||
{
|
||||
QContext.init(qInstance, new QSystemUserSession(qInstance));
|
||||
}
|
||||
try
|
||||
{
|
||||
List<HealthCheckMetaData> checks = new ArrayList<>();
|
||||
for(QRecord plan : new BackupCatalog().where(BackupsTables.PLAN, "isActive", true))
|
||||
{
|
||||
Integer planId = plan.getValueInteger("id");
|
||||
checks.add(new HealthCheckMetaData().withName(PREFIX + "." + planId.toString().toLowerCase(Locale.ROOT))
|
||||
.withLabel("Backup destination " + planId).withComponent("backups")
|
||||
.withKind(HealthCheckKind.DEPENDENCY).withCategory(HealthCategory.STORAGE)
|
||||
.withIntervalSeconds(86400).withTimeoutMs(30000).withCheckInstance(new BackupsDestinationHealthCheck(planId)));
|
||||
}
|
||||
return (checks);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if(ownsContext)
|
||||
{
|
||||
QContext.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.health;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
|
||||
import com.kingsrook.qqq.middleware.health.HealthIndicator;
|
||||
import com.kingsrook.qqq.middleware.health.model.metadata.HealthCheckResult;
|
||||
import com.kingsrook.qqq.middleware.health.model.metadata.HealthStatus;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The Backups health indicator (design spec section 10.4), for
|
||||
** qqq-middleware-health (an optional dependency: an app that does not have that
|
||||
** module never loads this class). It reports DOWN if any active plan marked
|
||||
** critical is LATE, FAILING or MISSING_IN_STORAGE, and UP with details otherwise.
|
||||
** It reads the slaStatus the SLA checker maintains; it never touches storage, so
|
||||
** a health call is cheap.
|
||||
**
|
||||
** It is OFF the liveness path by design. Add it to the indicators of the
|
||||
** instance's health check (the readiness and monitoring view), and do not point a
|
||||
** Kubernetes liveness probe at a check that includes it: a backup problem must
|
||||
** never restart pods.
|
||||
*******************************************************************************/
|
||||
public class BackupsHealthIndicator implements HealthIndicator
|
||||
{
|
||||
public static final String NAME = "backups";
|
||||
|
||||
private static final Set<String> CRITICAL_STATUSES = Set.of("LATE", "FAILING", "MISSING_IN_STORAGE");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getName()
|
||||
{
|
||||
return (NAME);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public HealthCheckResult check(QInstance qInstance) throws QException
|
||||
{
|
||||
long started = System.currentTimeMillis();
|
||||
boolean ownsContext = QContext.getQInstance() == null;
|
||||
if(ownsContext)
|
||||
{
|
||||
QContext.init(qInstance, systemSession(qInstance));
|
||||
}
|
||||
try
|
||||
{
|
||||
List<QRecord> plans = new BackupCatalog().query(BackupsTables.PLAN, new QQueryFilter(new QFilterCriteria("isActive", QCriteriaOperator.EQUALS, true)));
|
||||
|
||||
List<String> criticalProblems = new ArrayList<>();
|
||||
Map<String, Integer> byStatus = new LinkedHashMap<>();
|
||||
for(QRecord plan : plans)
|
||||
{
|
||||
String status = plan.getValueString("slaStatus") == null ? "UNKNOWN" : plan.getValueString("slaStatus");
|
||||
byStatus.merge(status, 1, Integer::sum);
|
||||
if(Boolean.TRUE.equals(plan.getValueBoolean("critical")) && CRITICAL_STATUSES.contains(status))
|
||||
{
|
||||
criticalProblems.add(plan.getValueString("planKey") + ": " + status);
|
||||
}
|
||||
}
|
||||
|
||||
Map<String, Object> details = new LinkedHashMap<>();
|
||||
details.put("activePlans", plans.size());
|
||||
details.put("plansByStatus", byStatus);
|
||||
if(!criticalProblems.isEmpty())
|
||||
{
|
||||
details.put("criticalPlansWithProblems", criticalProblems);
|
||||
}
|
||||
return (new HealthCheckResult()
|
||||
.withStatus(criticalProblems.isEmpty() ? HealthStatus.UP : HealthStatus.DOWN)
|
||||
.withDurationMs(System.currentTimeMillis() - started)
|
||||
.withDetails(details));
|
||||
}
|
||||
finally
|
||||
{
|
||||
if(ownsContext)
|
||||
{
|
||||
QContext.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The session for a health request that has no QQQ context. It must be the
|
||||
** core system user: token-based authentication modules (OAuth2, Auth0)
|
||||
** reject any other tokenless session, which failed the check on such hosts.
|
||||
*******************************************************************************/
|
||||
static QSession systemSession(QInstance qInstance)
|
||||
{
|
||||
return (new QSystemUserSession(qInstance));
|
||||
}
|
||||
}
|
||||
@@ -1,132 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.health;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCheck;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCheckInput;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthDetails;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read persisted SLA state using the former indicator's critical-plan rule.
|
||||
** SLA calculation and storage verification remain the scheduled checker's job.
|
||||
*******************************************************************************/
|
||||
public class BackupsSlaHealthCheck implements HealthCheck
|
||||
{
|
||||
public static final String NAME = "backups.sla";
|
||||
|
||||
private static final Set<String> BAD_STATUSES = Set.of("LATE", "FAILING", "MISSING_IN_STORAGE");
|
||||
private static final List<String> STATUSES = List.of("OK", "LATE", "FAILING", "MISSING_IN_STORAGE", "UNVERIFIED", "ABANDONED", "OWNER_INVALID", "UNKNOWN");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Counts use a fixed vocabulary so stored strings cannot become detail keys.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public HealthCheckOutcome run(HealthCheckInput input)
|
||||
{
|
||||
boolean ownsContext = QContext.getQInstance() == null;
|
||||
if(ownsContext)
|
||||
{
|
||||
QContext.init(input.getQInstance(), new QSystemUserSession(input.getQInstance()));
|
||||
}
|
||||
try
|
||||
{
|
||||
if(expired(input))
|
||||
{
|
||||
return (HealthCheckOutcome.down("Backup SLA check timed out"));
|
||||
}
|
||||
List<QRecord> plans = new BackupCatalog().where(BackupsTables.PLAN, "isActive", true);
|
||||
Map<String, Integer> counts = new LinkedHashMap<>();
|
||||
STATUSES.forEach(status -> counts.put(status, 0));
|
||||
boolean criticalBad = false;
|
||||
boolean ordinaryBad = false;
|
||||
for(QRecord plan : plans)
|
||||
{
|
||||
String status = plan.getValueString("slaStatus");
|
||||
status = status != null && STATUSES.contains(status) ? status : "UNKNOWN";
|
||||
counts.merge(status, 1, Integer::sum);
|
||||
if(BAD_STATUSES.contains(status))
|
||||
{
|
||||
if(Boolean.TRUE.equals(plan.getValueBoolean("critical")))
|
||||
{
|
||||
criticalBad = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
ordinaryBad = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if(expired(input))
|
||||
{
|
||||
return (HealthCheckOutcome.down("Backup SLA check timed out"));
|
||||
}
|
||||
HealthDetails details = new HealthDetails().withValue("activePlans", plans.size());
|
||||
counts.forEach((status, count) -> details.withValue("slaStatus." + status, count));
|
||||
HealthCheckOutcome outcome = HealthCheckOutcome.up("Backup SLA has no failures");
|
||||
if(criticalBad)
|
||||
{
|
||||
outcome = HealthCheckOutcome.down("Critical backup plans have SLA failures");
|
||||
}
|
||||
else if(ordinaryBad)
|
||||
{
|
||||
outcome = HealthCheckOutcome.degraded("Backup plans have SLA failures");
|
||||
}
|
||||
else if(!plans.isEmpty() && counts.get("UNVERIFIED") + counts.get("UNKNOWN") == plans.size())
|
||||
{
|
||||
outcome = HealthCheckOutcome.unknown("Backup SLA is not verified yet");
|
||||
}
|
||||
return (outcome.withDetails(details));
|
||||
}
|
||||
catch(QException | RuntimeException e)
|
||||
{
|
||||
return (HealthCheckOutcome.down("Backup SLA state could not be read"));
|
||||
}
|
||||
finally
|
||||
{
|
||||
if(ownsContext)
|
||||
{
|
||||
QContext.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stop between catalog operations when the runner's budget is exhausted.
|
||||
*******************************************************************************/
|
||||
static boolean expired(HealthCheckInput input)
|
||||
{
|
||||
return (Thread.currentThread().isInterrupted() || !Instant.now().isBefore(input.getDeadline()));
|
||||
}
|
||||
}
|
||||
@@ -20,12 +20,10 @@ package com.kingsrook.qbits.backups.metrics;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import com.kingsrook.qbits.backups.BackupsQBitConfig;
|
||||
import com.kingsrook.qbits.backups.BackupsRuntime;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
@@ -58,21 +56,10 @@ public final class BackupsMetrics
|
||||
{
|
||||
public static final String ENDPOINT_PATH = "/metrics/backups";
|
||||
|
||||
/** Cluster cache name of the rendered text: cleared on every node when a run finishes or a plan changes. */
|
||||
public static final String CACHE_NAME = "qbit-backups.metrics";
|
||||
|
||||
private static final Duration CACHE = Duration.ofSeconds(60);
|
||||
|
||||
private static final AtomicReference<Rendered> CACHED = ClusterCaches.shared(CACHE_NAME, new AtomicReference<>(), BackupsMetrics::resetCache, null);
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The rendered text and when it was rendered.
|
||||
*******************************************************************************/
|
||||
private record Rendered(String text, Instant at)
|
||||
{
|
||||
}
|
||||
private static String cachedText;
|
||||
private static Instant cachedAt;
|
||||
|
||||
|
||||
|
||||
@@ -97,21 +84,12 @@ public final class BackupsMetrics
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Forget the cached text on this node.
|
||||
** Forget the cached text (tests).
|
||||
*******************************************************************************/
|
||||
public static void resetCache()
|
||||
public static synchronized void resetCache()
|
||||
{
|
||||
CACHED.set(null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Forget the cached text on every node (the backup data changed).
|
||||
*******************************************************************************/
|
||||
public static void invalidate()
|
||||
{
|
||||
ClusterCaches.invalidate(CACHE_NAME);
|
||||
cachedText = null;
|
||||
cachedAt = null;
|
||||
}
|
||||
|
||||
|
||||
@@ -121,14 +99,13 @@ public final class BackupsMetrics
|
||||
*******************************************************************************/
|
||||
public static synchronized String renderCached() throws QException
|
||||
{
|
||||
Instant now = BackupsRuntime.getClock().instant();
|
||||
Rendered cached = CACHED.get();
|
||||
if(cached == null || cached.at().plus(CACHE).isBefore(now))
|
||||
Instant now = BackupsRuntime.getClock().instant();
|
||||
if(cachedText == null || cachedAt == null || cachedAt.plus(CACHE).isBefore(now))
|
||||
{
|
||||
cached = new Rendered(render(), now);
|
||||
CACHED.set(cached);
|
||||
cachedText = render();
|
||||
cachedAt = now;
|
||||
}
|
||||
return (cached.text());
|
||||
return (cachedText);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -413,22 +413,15 @@ public final class BackupsTables
|
||||
QCodeReference validation = new QCodeReference(BackupPlanTableCustomizer.class);
|
||||
table.withCustomizer(TableCustomizers.PRE_INSERT_RECORD, validation);
|
||||
table.withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, validation);
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////
|
||||
// after a save or delete, the plan customizer clears the cached metrics on //
|
||||
// every node //
|
||||
/////////////////////////////////////////////////////////////////////////////
|
||||
QCodeReference post = validation;
|
||||
if(StringUtils.hasContent(config.getSchedulerName()))
|
||||
{
|
||||
post = MultiCustomizer.of(
|
||||
QCodeReferenceWithProperties sync = MultiCustomizer.of(
|
||||
BaseSyncToScheduledJobTableCustomizer.makeCodeReference(table, new BackupPlanSyncToScheduledJobProcess()),
|
||||
BaseSyncToScheduledJobTableCustomizer.makeCodeReference(table, new BackupPlanVerifySyncToScheduledJobProcess()),
|
||||
validation);
|
||||
BaseSyncToScheduledJobTableCustomizer.makeCodeReference(table, new BackupPlanVerifySyncToScheduledJobProcess()));
|
||||
table.withCustomizer(TableCustomizers.POST_INSERT_RECORD, sync);
|
||||
table.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, sync);
|
||||
table.withCustomizer(TableCustomizers.POST_DELETE_RECORD, sync);
|
||||
}
|
||||
table.withCustomizer(TableCustomizers.POST_INSERT_RECORD, post);
|
||||
table.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, post);
|
||||
table.withCustomizer(TableCustomizers.POST_DELETE_RECORD, post);
|
||||
return (table);
|
||||
}
|
||||
|
||||
|
||||
@@ -22,7 +22,6 @@ import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import com.kingsrook.qbits.backups.metrics.BackupsMetrics;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qbits.backups.run.BackupRunFactory;
|
||||
@@ -32,9 +31,6 @@ import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterfa
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
|
||||
@@ -47,8 +43,7 @@ import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
|
||||
** configuration (the processes write the last run and SLA fields) is not
|
||||
** validated again.
|
||||
**
|
||||
** The saving user becomes the plan owner when none is given. After an insert,
|
||||
** update or delete, the cached Prometheus text is cleared on every node.
|
||||
** The saving user becomes the plan owner when none is given.
|
||||
*******************************************************************************/
|
||||
public class BackupPlanTableCustomizer implements TableCustomizerInterface
|
||||
{
|
||||
@@ -127,42 +122,6 @@ public class BackupPlanTableCustomizer implements TableCustomizerInterface
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A saved plan changes the metrics on every node.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
|
||||
{
|
||||
BackupsMetrics.invalidate();
|
||||
return (records);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A saved plan changes the metrics on every node.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
|
||||
{
|
||||
BackupsMetrics.invalidate();
|
||||
return (records);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A deleted plan changes the metrics on every node.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> postDelete(DeleteInput deleteInput, List<QRecord> records) throws QException
|
||||
{
|
||||
BackupsMetrics.invalidate();
|
||||
return (records);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The scope rules to resolve: the ones sent with the plan (associated rows), or
|
||||
** the saved ones.
|
||||
|
||||
@@ -20,7 +20,6 @@ package com.kingsrook.qbits.backups.plan;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
@@ -82,28 +81,6 @@ public final class DestinationProbe
|
||||
** Probe the destination of a plan.
|
||||
*******************************************************************************/
|
||||
public static Report probe(QRecord plan)
|
||||
{
|
||||
return (probe(plan, false, null));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Health monitoring only lists and heads; it never resolves prune credentials
|
||||
** or probes capabilities. The runner bounds in-flight storage calls; check the
|
||||
** deadline between calls because BackupStorage has no per-call timeout API.
|
||||
*******************************************************************************/
|
||||
public static Report probeReadOnly(QRecord plan, Instant deadline)
|
||||
{
|
||||
return (probe(plan, true, deadline));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Share storage resolution and the existing locked-bucket read path.
|
||||
*******************************************************************************/
|
||||
private static Report probe(QRecord plan, boolean readOnly, Instant deadline)
|
||||
{
|
||||
List<Check> checks = new ArrayList<>();
|
||||
List<String> warnings = new ArrayList<>();
|
||||
@@ -112,23 +89,15 @@ public final class DestinationProbe
|
||||
BackupStorage storage;
|
||||
try
|
||||
{
|
||||
requireTime(deadline);
|
||||
storage = BackupsRuntime.storageFor(alias);
|
||||
requireTime(deadline);
|
||||
}
|
||||
catch(RuntimeException e)
|
||||
{
|
||||
checks.add(new Check("Open the backup bucket " + alias, false, e.getMessage()));
|
||||
return (new Report(readOnly, checks, List.of(), warnings));
|
||||
return (new Report(false, checks, List.of(), warnings));
|
||||
}
|
||||
checks.add(new Check("Open the backup bucket " + alias, true, "found"));
|
||||
|
||||
if(readOnly)
|
||||
{
|
||||
headOnly(plan, storage, BackupPruner.planPrefix(plan), checks, deadline);
|
||||
return (new Report(true, checks, List.of(), warnings));
|
||||
}
|
||||
|
||||
BackupStorage deleteSide = storage;
|
||||
String pruneAlias = plan.getValueString("pruneBucketAlias");
|
||||
if(pruneAlias == null || pruneAlias.isBlank())
|
||||
@@ -164,7 +133,7 @@ public final class DestinationProbe
|
||||
String prefix = BackupPruner.planPrefix(plan);
|
||||
if(locked)
|
||||
{
|
||||
headOnly(plan, storage, prefix, checks, null);
|
||||
headOnly(plan, storage, prefix, checks);
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -178,11 +147,10 @@ public final class DestinationProbe
|
||||
/*******************************************************************************
|
||||
** The probe for a locked bucket: list the prefix, and head the newest manifest.
|
||||
*******************************************************************************/
|
||||
private static void headOnly(QRecord plan, BackupStorage storage, String prefix, List<Check> checks, Instant deadline)
|
||||
private static void headOnly(QRecord plan, BackupStorage storage, String prefix, List<Check> checks)
|
||||
{
|
||||
checks.add(new Check("Write and delete a probe object", true,
|
||||
deadline == null ? "skipped: object lock is on, a probe object could not be deleted again" : "skipped: read-only probe"));
|
||||
checks.add(attempt("List the plan prefix", deadline, () ->
|
||||
checks.add(attempt("Write and delete a probe object", () -> "skipped: object lock is on, a probe object could not be deleted again"));
|
||||
checks.add(attempt("List the plan prefix", () ->
|
||||
{
|
||||
List<StoredObjectInfo> objects = storage.list(prefix);
|
||||
return (objects.size() + " objects under " + prefix);
|
||||
@@ -194,11 +162,10 @@ public final class DestinationProbe
|
||||
checks.add(new Check("Head the newest manifest", true, "there is no successful run yet"));
|
||||
return;
|
||||
}
|
||||
checks.add(attempt("Head the newest manifest", deadline, () ->
|
||||
checks.add(attempt("Head the newest manifest", () ->
|
||||
{
|
||||
QRecord run = new com.kingsrook.qbits.backups.run.BackupCatalog().get(com.kingsrook.qbits.backups.model.BackupsTables.RUN, lastSuccess).orElseThrow();
|
||||
String key = run.getValueString("manifestKey");
|
||||
requireTime(deadline);
|
||||
if(key == null || storage.head(key).isEmpty())
|
||||
{
|
||||
throw (new IllegalStateException("the manifest of the newest successful run is not in the bucket"));
|
||||
@@ -271,9 +238,6 @@ public final class DestinationProbe
|
||||
@FunctionalInterface
|
||||
private interface Attempt
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Execute one diagnostic operation.
|
||||
*******************************************************************************/
|
||||
String run() throws Exception;
|
||||
}
|
||||
|
||||
@@ -283,40 +247,14 @@ public final class DestinationProbe
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static Check attempt(String name, Attempt attempt)
|
||||
{
|
||||
return (attempt(name, null, attempt));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Never begin another storage operation after the health budget expires.
|
||||
*******************************************************************************/
|
||||
private static Check attempt(String name, Instant deadline, Attempt attempt)
|
||||
{
|
||||
try
|
||||
{
|
||||
requireTime(deadline);
|
||||
String detail = attempt.run();
|
||||
requireTime(deadline);
|
||||
return (new Check(name, true, detail));
|
||||
return (new Check(name, true, attempt.run()));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
return (new Check(name, false, e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage()));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Existing on-demand diagnostics have no deadline; health probes always do.
|
||||
*******************************************************************************/
|
||||
private static void requireTime(Instant deadline)
|
||||
{
|
||||
if(deadline != null && (Thread.currentThread().isInterrupted() || !Instant.now().isBefore(deadline)))
|
||||
{
|
||||
throw (new IllegalStateException("Destination probe timed out"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -22,8 +22,6 @@ import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
import com.kingsrook.qbits.backups.BackupsRuntime;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.permissions.BackupsPermissions;
|
||||
@@ -180,10 +178,6 @@ public final class BackupsProcesses
|
||||
private final ProcessSummaryLine tables = line(Status.INFO, "table will be backed up", "tables will be backed up", "table was backed up", "tables were backed up");
|
||||
private final ProcessSummaryLine excluded = line(Status.INFO, "table is left out by its role or scope", "tables are left out by their role or scope",
|
||||
"table was left out by its role or scope", "tables were left out by their role or scope");
|
||||
private final ProcessSummaryLine unreadable = line(Status.INFO, "table is left out because its backend cannot be queried (storage only)",
|
||||
"tables are left out because their backend cannot be queried (storage only)", "table was left out because its backend cannot be queried (storage only)",
|
||||
"tables were left out because their backend cannot be queried (storage only)");
|
||||
private final Set<String> unreadableNames = new TreeSet<>();
|
||||
private final ProcessSummaryLine inactive = line(Status.WARNING, "plan is not active (it can still be run by hand)", "plans are not active (they can still be run by hand)",
|
||||
"plan was not active", "plans were not active");
|
||||
private final ProcessSummaryLine invalid = line(Status.ERROR, "plan cannot run", "plans cannot run", "plan could not run", "plans could not run");
|
||||
@@ -211,14 +205,6 @@ public final class BackupsProcesses
|
||||
plans.incrementCountAndAddPrimaryKey(plan.getValue("id"));
|
||||
tables.incrementCount(scope.tables().size());
|
||||
excluded.incrementCount(scope.excluded().size());
|
||||
for(ScopeResolver.ExcludedTable out : scope.excluded())
|
||||
{
|
||||
if(ScopeResolver.NOT_QUERYABLE.equals(out.reason()) && unreadableNames.add(out.name()))
|
||||
{
|
||||
unreadable.incrementCount();
|
||||
unreadable.setMessageSuffix(": " + String.join(", ", unreadableNames));
|
||||
}
|
||||
}
|
||||
if(!Boolean.TRUE.equals(plan.getValueBoolean("isActive")))
|
||||
{
|
||||
inactive.incrementCountAndAddPrimaryKey(plan.getValue("id"));
|
||||
@@ -249,7 +235,6 @@ public final class BackupsProcesses
|
||||
plans.addSelfToListIfAnyCount(lines);
|
||||
tables.addSelfToListIfAnyCount(lines);
|
||||
excluded.addSelfToListIfAnyCount(lines);
|
||||
unreadable.addSelfToListIfAnyCount(lines);
|
||||
inactive.addSelfToListIfAnyCount(lines);
|
||||
invalid.addSelfToListIfAnyCount(lines);
|
||||
return (lines);
|
||||
|
||||
@@ -118,7 +118,7 @@ public final class ExportTableProcess
|
||||
continue;
|
||||
}
|
||||
TableRole role = TableRoles.roleOf(qInstance, table);
|
||||
if((role == TableRole.DATA || role == TableRole.AUDIT || role == TableRole.DERIVED) && TableRoles.isQueryable(qInstance, table))
|
||||
if(role == TableRole.DATA || role == TableRole.AUDIT || role == TableRole.DERIVED)
|
||||
{
|
||||
processes.add(processFor(table.getName()));
|
||||
}
|
||||
|
||||
@@ -47,7 +47,6 @@ import com.kingsrook.qbits.backups.files.FileReference;
|
||||
import com.kingsrook.qbits.backups.files.FileReferenceProvider;
|
||||
import com.kingsrook.qbits.backups.manifest.BackupManifest;
|
||||
import com.kingsrook.qbits.backups.manifest.TableFingerprints;
|
||||
import com.kingsrook.qbits.backups.metrics.BackupsMetrics;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.scope.ScopeResolver;
|
||||
import com.kingsrook.qbits.backups.storage.BackupStorage;
|
||||
@@ -77,7 +76,7 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
** content-addressed store; checksums.sha256 is written; and the manifest is
|
||||
** written LAST, then checked with a head. A run with no manifest did not
|
||||
** complete. The run row is updated every page (heartbeat), and a failure marks
|
||||
** the run FAILED with a sanitized message and detail (no secrets, no row values).
|
||||
** the run FAILED with a scrubbed message (never row values).
|
||||
*******************************************************************************/
|
||||
public class BackupEngine
|
||||
{
|
||||
@@ -142,7 +141,6 @@ public class BackupEngine
|
||||
runValues.remove("startDate");
|
||||
runValues.put("endDate", null);
|
||||
runValues.put("errorMessage", null);
|
||||
runValues.put("errorDetail", null);
|
||||
}
|
||||
if(request.getExistingRunId() != null)
|
||||
{
|
||||
@@ -156,12 +154,9 @@ public class BackupEngine
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("Could not create the backup run record", e, logPair("runKey", runKey));
|
||||
String message = FailureText.summary("Could not create the run record", e);
|
||||
result.setStatus("FAILED");
|
||||
result.setErrorMessage(message);
|
||||
result.setErrorDetail(FailureText.detail(e));
|
||||
failProgress(progress, message);
|
||||
result.setErrorMessage("Could not create the run record (" + e.getClass().getSimpleName() + ")");
|
||||
failProgress(progress, "Could not create the run record (" + e.getClass().getSimpleName() + ")");
|
||||
return (result);
|
||||
}
|
||||
|
||||
@@ -178,12 +173,12 @@ public class BackupEngine
|
||||
}
|
||||
catch(CancelledException e)
|
||||
{
|
||||
finish(request, result, "CANCELLED", "Cancelled", null, started);
|
||||
finish(request, result, "CANCELLED", "Cancelled", started);
|
||||
progress.info("Backup cancelled");
|
||||
}
|
||||
catch(TimedOutException e)
|
||||
{
|
||||
finish(request, result, "TIMED_OUT", e.getMessage(), null, started);
|
||||
finish(request, result, "TIMED_OUT", e.getMessage(), started);
|
||||
failProgress(progress, e.getMessage());
|
||||
}
|
||||
catch(Exception e)
|
||||
@@ -193,14 +188,14 @@ public class BackupEngine
|
||||
{
|
||||
if(interrupted.getReason() != JobInterruptedException.Reason.LEASE_LOST)
|
||||
{
|
||||
finish(request, result, interrupted.getReason() == JobInterruptedException.Reason.CANCEL ? "CANCELLED" : "ABANDONED", "Job interrupted", null, started);
|
||||
finish(request, result, interrupted.getReason() == JobInterruptedException.Reason.CANCEL ? "CANCELLED" : "ABANDONED", "Job interrupted", started);
|
||||
}
|
||||
throw (interrupted);
|
||||
}
|
||||
LOG.warn("Backup run failed", e, logPair("runKey", runKey));
|
||||
String message = scrub(e);
|
||||
failProgress(progress, message);
|
||||
finish(request, result, "FAILED", message, FailureText.detail(e), started);
|
||||
// Provider and reader messages can contain credentials or row values.
|
||||
failProgress(progress, scrub(e));
|
||||
finish(request, result, "FAILED", scrub(e), started);
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -577,7 +572,7 @@ public class BackupEngine
|
||||
extra.put("bytesRaw", bytesRaw);
|
||||
extra.put("bytesStored", bytesStored);
|
||||
extra.put("warningCount", result.getWarnings().size());
|
||||
finishWith(request, result, result.getWarnings().isEmpty() ? "SUCCEEDED" : "SUCCEEDED_WITH_WARNINGS", null, null, started, extra);
|
||||
finishWith(request, result, result.getWarnings().isEmpty() ? "SUCCEEDED" : "SUCCEEDED_WITH_WARNINGS", null, started, extra);
|
||||
}
|
||||
|
||||
|
||||
@@ -745,11 +740,11 @@ public class BackupEngine
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private void finish(BackupRunRequest request, BackupRunResult result, String status, String message, String detail, Instant started)
|
||||
private void finish(BackupRunRequest request, BackupRunResult result, String status, String message, Instant started)
|
||||
{
|
||||
try
|
||||
{
|
||||
finishWith(request, result, status, message, detail, started, new LinkedHashMap<>());
|
||||
finishWith(request, result, status, message, started, new LinkedHashMap<>());
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
@@ -757,7 +752,6 @@ public class BackupEngine
|
||||
request.getProgress().warn("Could not finish the run record");
|
||||
result.setStatus(status);
|
||||
result.setErrorMessage(message);
|
||||
result.setErrorDetail(detail);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -766,7 +760,7 @@ public class BackupEngine
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private void finishWith(BackupRunRequest request, BackupRunResult result, String status, String message, String detail, Instant started, Map<String, Serializable> extra) throws Exception
|
||||
private void finishWith(BackupRunRequest request, BackupRunResult result, String status, String message, Instant started, Map<String, Serializable> extra) throws Exception
|
||||
{
|
||||
Instant end = request.getClock().instant();
|
||||
Map<String, Serializable> v = new LinkedHashMap<>(extra);
|
||||
@@ -775,14 +769,11 @@ public class BackupEngine
|
||||
v.put("durationMs", end.toEpochMilli() - started.toEpochMilli());
|
||||
v.put("heartbeatDate", end);
|
||||
v.put("errorMessage", message);
|
||||
v.put("errorDetail", detail);
|
||||
v.put("warningCount", result.getWarnings().size());
|
||||
catalog.update(BackupsTables.RUN, result.getRunId(), v);
|
||||
result.setStatus(status);
|
||||
result.setErrorMessage(message);
|
||||
result.setErrorDetail(detail);
|
||||
signal(request, result, status, message);
|
||||
BackupsMetrics.invalidate();
|
||||
}
|
||||
|
||||
|
||||
@@ -835,17 +826,12 @@ public class BackupEngine
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The error_message of a failed run: "Backup failed: SimpleClass: message".
|
||||
** Provider, driver and reader messages can contain credentials or row values,
|
||||
** so the message goes through FailureText.sanitize (URL user-info, presigned
|
||||
** query parameters, key=value secrets, Authorization values, AWS key ids, PEM
|
||||
** blocks and Postgres row-value fragments are redacted; object keys and bucket
|
||||
** names are kept). The class, the cause chain and the root cause frames go to
|
||||
** error_detail (FailureText.detail); the full trace stays in the server log.
|
||||
** Provider exceptions may contain credentials or row values. Persist only the
|
||||
** failure type; detailed diagnostics stay in the server log.
|
||||
*******************************************************************************/
|
||||
private static String scrub(Exception e)
|
||||
{
|
||||
return (FailureText.summary(e));
|
||||
return ("Backup failed (" + e.getClass().getSimpleName() + "); see server logs for details.");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -43,7 +43,6 @@ public class BackupRunResult
|
||||
private int filesMissing;
|
||||
private long fileBytesCopied;
|
||||
private String errorMessage;
|
||||
private String errorDetail;
|
||||
private List<String> warnings = new ArrayList<>();
|
||||
|
||||
|
||||
@@ -398,26 +397,6 @@ public class BackupRunResult
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for errorDetail (sanitized class, message, cause chain and root frames)
|
||||
*******************************************************************************/
|
||||
public String getErrorDetail()
|
||||
{
|
||||
return (errorDetail);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for errorDetail
|
||||
*******************************************************************************/
|
||||
public void setErrorDetail(String errorDetail)
|
||||
{
|
||||
this.errorDetail = errorDetail;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for warnings
|
||||
*******************************************************************************/
|
||||
|
||||
@@ -1,184 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.run;
|
||||
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.IdentityHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Safe text for a failure: what is persisted on a run row (error_message and
|
||||
** error_detail) and shown in job output.
|
||||
**
|
||||
** Exception messages from storage providers, drivers and readers can carry
|
||||
** credentials (URL user-info, presigned query strings, key=value secrets,
|
||||
** Authorization headers, AWS access key ids, PEM blocks) and row values (the
|
||||
** Postgres "Key (col)=(value)" and "Detail:" fragments). sanitize() redacts
|
||||
** those and keeps the rest (class names, object keys, bucket names), which is
|
||||
** what an operator needs to act on a failure without reading the server log.
|
||||
*******************************************************************************/
|
||||
public final class FailureText
|
||||
{
|
||||
/** Redaction marker. */
|
||||
public static final String REDACTED = "[REDACTED]";
|
||||
|
||||
/** The longest error_detail written. */
|
||||
public static final int MAX_DETAIL_CHARS = 8192;
|
||||
|
||||
/** The longest exception message kept in error_message. */
|
||||
public static final int MAX_SUMMARY_MESSAGE_CHARS = 500;
|
||||
|
||||
/** Stack frames of the root cause kept in error_detail. */
|
||||
public static final int ROOT_CAUSE_FRAMES = 8;
|
||||
|
||||
private static final int MAX_CAUSES = 20;
|
||||
|
||||
private static final String SECRET_NAME = "[\\w.-]*(?:password|passwd|pwd|secret|token|credential|signature|api[_-]?key|access[_-]?key|private[_-]?key)[\\w.-]*";
|
||||
|
||||
private static final Pattern PEM_BLOCK = Pattern.compile("-----BEGIN [A-Z0-9 ]+-----.*?(?:-----END [A-Z0-9 ]+-----|$)", Pattern.DOTALL);
|
||||
private static final Pattern URL_USERINFO = Pattern.compile("(?i)\\b([a-z][a-z0-9+.-]*://)[^/\\s@?#]+@");
|
||||
private static final Pattern QUERY_PARAM = Pattern.compile("(?i)([?&;](?:X-Amz-[\\w-]+|Signature|Credential|AWSAccessKeyId|Policy|sig|key|" + SECRET_NAME + ")=)[^&\\s\"'<>]*");
|
||||
private static final Pattern PG_KEY_VALUE = Pattern.compile("(?i)\\b(Key \\([^)]*\\)=\\().*?(\\))(?=\\s|$|[.,;:])");
|
||||
private static final Pattern PG_DETAIL = Pattern.compile("(?i)\\b(Detail:|Failing row contains)[^\\r\\n]*");
|
||||
private static final Pattern AUTHORIZATION = Pattern.compile("(?i)\\b((?:Proxy-)?Authorization\"?\\s*[:=]\\s*)(\"[^\"]*\"|[^\\r\\n,;]+)");
|
||||
private static final Pattern BEARER = Pattern.compile("(?i)\\b(Bearer|Basic)\\s+[A-Za-z0-9._~+/=-]{6,}");
|
||||
private static final Pattern SECRET_JSON = Pattern.compile("(?i)(\"" + SECRET_NAME + "\"\\s*:\\s*)\"[^\"]*\"");
|
||||
private static final Pattern SECRET_ASSIGN = Pattern.compile("(?i)(?<![\\w.-])((?:" + SECRET_NAME + "|key)\\s*=\\s*)(?!\\[REDACTED)(\"[^\"]*\"|'[^']*'|[^\\s&,;)\\]}]+)");
|
||||
private static final Pattern AWS_KEY_ID = Pattern.compile("\\b(?:AKIA|ASIA|AGPA|AIDA|AROA|ANPA|ANVA|AIPA)[A-Z0-9]{16}\\b");
|
||||
|
||||
private static final String R = Matcher.quoteReplacement(REDACTED);
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private FailureText()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Redact secrets and row values from a message. Null stays null.
|
||||
*******************************************************************************/
|
||||
public static String sanitize(String text)
|
||||
{
|
||||
if(text == null)
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
String s = PEM_BLOCK.matcher(text).replaceAll(Matcher.quoteReplacement("[REDACTED PEM]"));
|
||||
s = URL_USERINFO.matcher(s).replaceAll("$1" + R + "@");
|
||||
s = QUERY_PARAM.matcher(s).replaceAll("$1" + R);
|
||||
s = PG_KEY_VALUE.matcher(s).replaceAll("$1" + R + "$2");
|
||||
s = PG_DETAIL.matcher(s).replaceAll("$1 " + R);
|
||||
s = AUTHORIZATION.matcher(s).replaceAll("$1" + R);
|
||||
s = BEARER.matcher(s).replaceAll("$1 " + R);
|
||||
s = SECRET_JSON.matcher(s).replaceAll("$1\"" + R + "\"");
|
||||
s = SECRET_ASSIGN.matcher(s).replaceAll("$1" + R);
|
||||
s = AWS_KEY_ID.matcher(s).replaceAll(R);
|
||||
return (s);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The short error_message: "Backup failed: SimpleClass: sanitized message".
|
||||
*******************************************************************************/
|
||||
public static String summary(Throwable t)
|
||||
{
|
||||
return (summary("Backup failed", t));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A short message: "prefix: SimpleClass: sanitized message" (bounded).
|
||||
*******************************************************************************/
|
||||
public static String summary(String prefix, Throwable t)
|
||||
{
|
||||
String message = sanitize(t.getMessage());
|
||||
if(message == null || message.isBlank())
|
||||
{
|
||||
return (prefix + ": " + t.getClass().getSimpleName());
|
||||
}
|
||||
message = message.strip();
|
||||
if(message.length() > MAX_SUMMARY_MESSAGE_CHARS)
|
||||
{
|
||||
message = message.substring(0, MAX_SUMMARY_MESSAGE_CHARS) + "...";
|
||||
}
|
||||
return (prefix + ": " + t.getClass().getSimpleName() + ": " + message);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The error_detail: the exception class and sanitized message, each cause
|
||||
** ("Caused by: ..."), then the top stack frames of the root cause, bounded to
|
||||
** MAX_DETAIL_CHARS. Frames are class, method, file and line only (no values).
|
||||
*******************************************************************************/
|
||||
public static String detail(Throwable t)
|
||||
{
|
||||
StringBuilder out = new StringBuilder(line(t));
|
||||
Set<Throwable> seen = Collections.newSetFromMap(new IdentityHashMap<>());
|
||||
seen.add(t);
|
||||
Throwable root = t;
|
||||
Throwable cause = t.getCause();
|
||||
while(cause != null && seen.size() < MAX_CAUSES && seen.add(cause))
|
||||
{
|
||||
out.append("\nCaused by: ").append(line(cause));
|
||||
root = cause;
|
||||
cause = cause.getCause();
|
||||
}
|
||||
|
||||
List<StackTraceElement> frames = List.of(root.getStackTrace());
|
||||
int shown = Math.min(ROOT_CAUSE_FRAMES, frames.size());
|
||||
if(shown > 0)
|
||||
{
|
||||
out.append("\nRoot cause stack (top ").append(shown).append(" of ").append(frames.size()).append(" frames):");
|
||||
for(StackTraceElement frame : frames.subList(0, shown))
|
||||
{
|
||||
out.append("\n\tat ").append(frame);
|
||||
}
|
||||
}
|
||||
|
||||
if(out.length() > MAX_DETAIL_CHARS)
|
||||
{
|
||||
String marker = "\n... (truncated)";
|
||||
out.setLength(MAX_DETAIL_CHARS - marker.length());
|
||||
out.append(marker);
|
||||
}
|
||||
return (out.toString());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** "fully.qualified.Class: sanitized message".
|
||||
*******************************************************************************/
|
||||
private static String line(Throwable t)
|
||||
{
|
||||
String message = sanitize(t.getMessage());
|
||||
return (message == null ? t.getClass().getName() : t.getClass().getName() + ": " + message);
|
||||
}
|
||||
}
|
||||
@@ -48,9 +48,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
*******************************************************************************/
|
||||
public class ScopeResolver
|
||||
{
|
||||
/** Exclusion reason of a table whose backend has no query interface (storage-only): it cannot be read, even when selected. */
|
||||
public static final String NOT_QUERYABLE = "NOT_QUERYABLE";
|
||||
|
||||
private final QInstance qInstance;
|
||||
|
||||
|
||||
@@ -208,10 +205,6 @@ public class ScopeResolver
|
||||
{
|
||||
excluded.add(new ExcludedTable(name, "BACKUPS"));
|
||||
}
|
||||
else if(!TableRoles.isQueryable(qInstance, table))
|
||||
{
|
||||
excluded.add(new ExcludedTable(name, NOT_QUERYABLE));
|
||||
}
|
||||
else if((role == TableRole.EPHEMERAL || role == TableRole.DERIVED || role == TableRole.EXTERNAL) && !explicit.contains(name) && !roles.contains(role))
|
||||
{
|
||||
excluded.add(new ExcludedTable(name, role.name()));
|
||||
|
||||
@@ -35,6 +35,6 @@ public enum TableRole
|
||||
SECRET,
|
||||
/** The tables of this qbit: the catalog is rebuilt from manifests. */
|
||||
BACKUPS,
|
||||
/** Tables backed by another system (API), or by a backend that cannot be queried (storage-only: never read, even when selected). Excluded by default. */
|
||||
/** Tables backed by another system (API). Excluded by default; selectable. */
|
||||
EXTERNAL
|
||||
}
|
||||
@@ -17,20 +17,12 @@
|
||||
package com.kingsrook.qbits.backups.scope;
|
||||
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import com.kingsrook.qbits.backups.BackupsQBitConfig;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@@ -39,14 +31,6 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
*******************************************************************************/
|
||||
public final class TableRoles
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(TableRoles.class);
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// whether a backend type's module has a query interface: the answer depends //
|
||||
// only on the module class, so it is cached per backend type (roleOf is hot) //
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
private static final Map<String, Boolean> QUERYABLE_BY_BACKEND_TYPE = new ConcurrentHashMap<>();
|
||||
|
||||
private static final Set<String> AUDIT_TABLES = Set.of("audit", "auditTable", "auditUser", "auditDetail");
|
||||
|
||||
private static final String[] SECRET_NAME_HINTS = { "refreshToken", "RefreshToken", "oauthToken", "OAuthToken", "apiKey" };
|
||||
@@ -92,10 +76,6 @@ public final class TableRoles
|
||||
{
|
||||
return (TableRole.SECRET);
|
||||
}
|
||||
if(qInstance != null && !isQueryable(qInstance, table))
|
||||
{
|
||||
return (TableRole.EXTERNAL);
|
||||
}
|
||||
if(AUDIT_TABLES.contains(name))
|
||||
{
|
||||
return (TableRole.AUDIT);
|
||||
@@ -128,56 +108,6 @@ public final class TableRoles
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether the rows of a table can be read: its backend module must have a query
|
||||
** interface. A storage-only module (for example qbit-edi's preview download
|
||||
** backend) throws from getQueryInterface(), and reading its table would fail the
|
||||
** whole run, so such a table is never backed up. A backend whose module cannot
|
||||
** be resolved cannot be read either (not cached: the module may register later).
|
||||
*******************************************************************************/
|
||||
public static boolean isQueryable(QInstance qInstance, QTableMetaData table)
|
||||
{
|
||||
if(qInstance == null || table == null || table.getBackendName() == null)
|
||||
{
|
||||
return (true);
|
||||
}
|
||||
QBackendMetaData backend = qInstance.getBackend(table.getBackendName());
|
||||
if(backend == null || backend.getBackendType() == null)
|
||||
{
|
||||
return (true);
|
||||
}
|
||||
Boolean cached = QUERYABLE_BY_BACKEND_TYPE.get(backend.getBackendType());
|
||||
if(cached != null)
|
||||
{
|
||||
return (cached);
|
||||
}
|
||||
|
||||
QBackendModuleInterface module;
|
||||
try
|
||||
{
|
||||
module = new QBackendModuleDispatcher().getQBackendModule(backend);
|
||||
}
|
||||
catch(QModuleDispatchException e)
|
||||
{
|
||||
LOG.debug("No backend module for a table, so it cannot be backed up", logPair("table", table.getName()), logPair("backendType", backend.getBackendType()));
|
||||
return (false);
|
||||
}
|
||||
|
||||
boolean queryable;
|
||||
try
|
||||
{
|
||||
queryable = module.getQueryInterface() != null;
|
||||
}
|
||||
catch(IllegalStateException | UnsupportedOperationException e)
|
||||
{
|
||||
queryable = false;
|
||||
}
|
||||
QUERYABLE_BY_BACKEND_TYPE.put(backend.getBackendType(), queryable);
|
||||
return (queryable);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A table that is hidden and has a password or sensitive field (other than the
|
||||
** keys and dates) is treated as a table of secrets.
|
||||
|
||||
@@ -35,7 +35,6 @@ import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qbits.backups.storage.BackupStorage;
|
||||
import com.kingsrook.qbits.backups.storage.StoredObjectInfo;
|
||||
import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
@@ -96,11 +95,8 @@ public class SlaChecker
|
||||
private static final Set<String> FAILED_STATUSES = Set.of("FAILED", "TIMED_OUT", "ABANDONED");
|
||||
private static final Set<String> FINISHED_STATUSES = Set.of("SUCCEEDED", "SUCCEEDED_WITH_WARNINGS", "FAILED", "TIMED_OUT", "ABANDONED");
|
||||
|
||||
private static final Map<String, Instant> MANIFEST_OK_AT = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
|
||||
"throttles this node's storage HEAD of a manifest to once per MANIFEST_CACHE; it caches storage state, not editable data, "
|
||||
+ "and a pruned run is skipped by its prunedDate, so each node is as fresh as a single node");
|
||||
private static final Map<String, Instant> LAST_AUDIT_AT = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
|
||||
"throttles this node's bucket audit to once per BUCKET_AUDIT_INTERVAL; another node auditing again is only repeated work");
|
||||
private static final Map<String, Instant> MANIFEST_OK_AT = new ConcurrentHashMap<>();
|
||||
private static final Map<String, Instant> LAST_AUDIT_AT = new ConcurrentHashMap<>();
|
||||
|
||||
private final BackupCatalog catalog = new BackupCatalog();
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
bundle: qbit-backups
|
||||
version: ${project.version}
|
||||
version: 0.1.0-SNAPSHOT
|
||||
layer: QBIT
|
||||
entries:
|
||||
- key: table:backups_plan;field:name
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
version=${project.version}
|
||||
@@ -1,164 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups;
|
||||
|
||||
|
||||
import java.util.EnumSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
import java.util.function.Consumer;
|
||||
import com.kingsrook.qbits.backups.metrics.BackupsMetrics;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qqq.backend.core.cluster.ClusterBus;
|
||||
import com.kingsrook.qqq.backend.core.cluster.ClusterBusInterface;
|
||||
import com.kingsrook.qqq.backend.core.cluster.ClusterBusMetaData;
|
||||
import com.kingsrook.qqq.backend.core.cluster.ClusterCacheAudit;
|
||||
import com.kingsrook.qqq.backend.core.cluster.ClusterEvent;
|
||||
import com.kingsrook.qqq.backend.core.cluster.QNodeRoles;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The Backups caches are correct on several nodes: every static cache is
|
||||
** registered with ClusterCaches, and the cached metrics are cleared on every
|
||||
** node when a plan changes.
|
||||
*******************************************************************************/
|
||||
class BackupsClusterCachesTest
|
||||
{
|
||||
/** the "wire": events every node's transport sees */
|
||||
static final List<Consumer<ClusterEvent>> WIRE = new CopyOnWriteArrayList<>();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A transport over the static WIRE, so a "remote" publish reaches us.
|
||||
*******************************************************************************/
|
||||
public static class WireBus implements ClusterBusInterface
|
||||
{
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void publish(ClusterEvent event)
|
||||
{
|
||||
WIRE.forEach(listener -> listener.accept(event));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public AutoCloseable subscribe(Consumer<ClusterEvent> listener)
|
||||
{
|
||||
WIRE.add(listener);
|
||||
return (() -> WIRE.remove(listener));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A backed-up test instance, connected to the wire as node-a.
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp() throws Exception
|
||||
{
|
||||
VerifyTestSupport.setUp(1);
|
||||
QInstance instance = QContext.getQInstance();
|
||||
instance.withSupplementalMetaData(new ClusterBusMetaData().withBus(new QCodeReference(WireBus.class)));
|
||||
QNodeRoles.set(new QNodeRoles.Settings(EnumSet.allOf(QNodeRoles.Role.class), true, true, true, null, "node-a"));
|
||||
ClusterBus.connect(instance);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@AfterEach
|
||||
void afterEach()
|
||||
{
|
||||
ClusterBus.close(QContext.getQInstance());
|
||||
WIRE.clear();
|
||||
QNodeRoles.set(null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every static Memoization in the QBit is registered (shared or node-local).
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testNoUnregisteredStaticCaches() throws Exception
|
||||
{
|
||||
assertThat(ClusterCacheAudit.unregisteredStaticCaches("com.kingsrook.qbits.backups")).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Another node's invalidation clears this node's cached metrics.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRemoteInvalidationClearsMetricsCache() throws Exception
|
||||
{
|
||||
String cached = BackupsMetrics.renderCached();
|
||||
assertThat(BackupsMetrics.renderCached()).isSameAs(cached);
|
||||
|
||||
ClusterEvent remote = new ClusterEvent(ClusterEvent.TYPE_MEMO_INVALIDATE, BackupsMetrics.CACHE_NAME, null);
|
||||
remote.setOriginNodeId("node-b");
|
||||
WIRE.forEach(listener -> listener.accept(remote));
|
||||
|
||||
assertThat(BackupsMetrics.renderCached()).isNotSameAs(cached);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A plan edit here clears the metrics here and tells the other nodes.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testPlanEditPublishesMetricsInvalidation() throws Exception
|
||||
{
|
||||
QRecord plan = VerifyTestSupport.plan(Map.of());
|
||||
String cached = BackupsMetrics.renderCached();
|
||||
|
||||
List<ClusterEvent> sent = new CopyOnWriteArrayList<>();
|
||||
WIRE.add(sent::add);
|
||||
|
||||
new BackupCatalog().update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("slaHours", 48));
|
||||
|
||||
assertThat(BackupsMetrics.renderCached()).isNotSameAs(cached);
|
||||
assertThat(sent).anySatisfy(event ->
|
||||
{
|
||||
assertThat(event.getType()).isEqualTo(ClusterEvent.TYPE_MEMO_INVALIDATE);
|
||||
assertThat(event.getName()).isEqualTo(BackupsMetrics.CACHE_NAME);
|
||||
assertThat(event.getOriginNodeId()).isEqualTo("node-a");
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -18,7 +18,6 @@ package com.kingsrook.qbits.backups;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundles;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfigValidationException;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
@@ -41,16 +40,12 @@ class BackupsQBitProducerTest
|
||||
BackupsQBitConfig config = new BackupsQBitConfig().withBackendName("rdbms");
|
||||
BackupsQBitProducer producer = new BackupsQBitProducer().withQBitConfig(config).withNamespace("backups");
|
||||
|
||||
String artifactVersion = System.getProperty("qbit.artifactVersion");
|
||||
assertThat(artifactVersion).isNotBlank();
|
||||
assertThat(producer.getQBitMetaData().getVersion()).isEqualTo(artifactVersion).isEqualTo(BackupsQBitProducer.VERSION);
|
||||
assertThat(producer.isEnabled()).isTrue();
|
||||
assertThat(producer.getQBitConfig()).isSameAs(config);
|
||||
assertThat(producer.getNamespace()).isEqualTo("backups");
|
||||
assertThat(config.getDefaultBackendNameForTables()).isEqualTo("rdbms");
|
||||
|
||||
producer.produce(qInstance);
|
||||
assertThat(HelpBundles.of(qInstance).getBundles().get(0).getVersion()).isEqualTo(artifactVersion).isEqualTo(BackupsQBitProducer.VERSION);
|
||||
assertThat(qInstance.getQBits()).hasSize(1);
|
||||
assertThat(qInstance.getQBits().values().iterator().next().getArtifactId()).isEqualTo("qbit-backups");
|
||||
}
|
||||
|
||||
@@ -88,7 +88,7 @@ class BackupsHomeTest
|
||||
void testEmptyStates()
|
||||
{
|
||||
String html = BackupsHomeView.render(new BackupsHomeData(List.of(), List.of(), List.of(), List.of(), List.of(), Instant.now()), "/app");
|
||||
assertThat(html).contains("No backup plans yet").contains("Setup: <a").contains("No plan: nothing is backed up.").doesNotContain("Nothing needs attention").contains("No test restore yet")
|
||||
assertThat(html).contains("No backup plans yet").contains("Nothing to check yet").contains("No test restore yet")
|
||||
.contains("Nothing has run yet").contains("Trends appear after the first successful run").contains("Retention starts once runs exist");
|
||||
assertThat(BackupsHomeView.renderHealthCard(new BackupsHomeData(List.of(), List.of(), List.of(), List.of(), List.of(), Instant.now()), "/app")).contains("No backup plans yet");
|
||||
}
|
||||
@@ -121,45 +121,6 @@ class BackupsHomeTest
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The Setup card and the Needs attention card never contradict each other: a plan
|
||||
** whose storage aliases do not resolve is a Setup fault, and it is listed under
|
||||
** Needs attention too, with the same count.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testUnresolvedAliasAppearsInSetupAndNeedsAttention() throws Exception
|
||||
{
|
||||
BackupsRuntime.getConfig().withStorageResolver(alias -> BackupsTestUtils.BACKUPS);
|
||||
new BackupCatalog().insert(BackupsTables.PLAN, new LinkedHashMap<>(Map.of("name", "Offsite", "planKey", "offsite", "isActive", true, "scopeType", "INSTANCE",
|
||||
"storageBucketAlias", "backupsAws", "pruneBucketAlias", "backupsAwsPrune")));
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
// the aliases resolved when the plan was saved; now the bucket is gone //
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
BackupsRuntime.getConfig().withStorageResolver(alias -> "backups".equals(alias) ? BackupsTestUtils.BACKUPS : null);
|
||||
|
||||
BackupsHomeData data = BackupsHomeData.load();
|
||||
List<BackupsSetupStatus.Item> setupTodo = BackupsSetupStatus.load(data.plans()).stream().filter(i -> i.state() == BackupsSetupStatus.State.TODO).toList();
|
||||
assertThat(setupTodo).extracting(BackupsSetupStatus.Item::label).contains("Storage");
|
||||
|
||||
String html = BackupsHomeView.render(data, "/app");
|
||||
String attention = html.substring(html.indexOf(">Needs attention<"));
|
||||
attention = attention.substring(0, attention.indexOf("All runs"));
|
||||
assertThat(attention).doesNotContain("Nothing needs attention").contains("backupsAws").contains("backupsAwsPrune");
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// every Setup TODO is listed under Needs attention (counts agree: 2 and 2) //
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
assertThat(html).contains(setupTodo.size() + " items need attention.");
|
||||
for(BackupsSetupStatus.Item item : setupTodo)
|
||||
{
|
||||
assertThat(attention).contains("Setup: <a").contains(item.label());
|
||||
}
|
||||
assertThat(attention.split("<li>", -1).length - 1).isEqualTo(setupTodo.size() + data.plansNeedingAttention().size() + data.failedRuns().size());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The links follow the configured table path prefix.
|
||||
*******************************************************************************/
|
||||
|
||||
@@ -1,495 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.health;
|
||||
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.backups.BackupsRuntime;
|
||||
import com.kingsrook.qbits.backups.BackupsTestUtils;
|
||||
import com.kingsrook.qbits.backups.VerifyTestSupport;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.plan.DestinationProbe;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qbits.backups.storage.InMemoryBackupStorage;
|
||||
import com.kingsrook.qbits.backups.storage.StoredObjectInfo;
|
||||
import com.kingsrook.qbits.backups.storage.WriteOptions;
|
||||
import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCategory;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCheck;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCheckInput;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCheckKind;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthStatus;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Central registration, migrated SLA rules, read-only probes and secret safety.
|
||||
*******************************************************************************/
|
||||
class BackupsHealthChecksTest
|
||||
{
|
||||
private static final String SECRET = "S3NT1NEL-secret";
|
||||
|
||||
private final BackupCatalog catalog = new BackupCatalog();
|
||||
private ReadOnlyStorage storage;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** All probes use memory storage; the resolver represents credential injection.
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp() throws Exception
|
||||
{
|
||||
VerifyTestSupport.setUp(3);
|
||||
storage = new ReadOnlyStorage(SECRET);
|
||||
BackupsRuntime.getConfig().withStorageResolver(alias ->
|
||||
{
|
||||
assertThat(alias).as("never resolve prune credentials").isNotEqualTo("prune");
|
||||
return (storage);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Use the actual producer registry, including its reflective supplier loading.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRegistrationAndDynamicPlans() throws Exception
|
||||
{
|
||||
HealthChecksMetaData registry = HealthChecksMetaData.ofOrWithNew(QContext.getQInstance());
|
||||
assertThat(registry.getChecks()).extracting(HealthCheckMetaData::getName).containsExactly("backups.sla");
|
||||
assertThat(registry.getSuppliers()).singleElement().satisfies(supplier ->
|
||||
{
|
||||
assertThat(supplier.getNamePrefix()).isEqualTo("backups.destination");
|
||||
assertThat(supplier.getComponent()).isEqualTo("backups");
|
||||
});
|
||||
QRecord active = VerifyTestSupport.plan(Map.of());
|
||||
VerifyTestSupport.plan(Map.of("name", "Off", "planKey", "off", "isActive", false));
|
||||
List<HealthCheckMetaData> checks = registry.resolveAll(QContext.getQInstance());
|
||||
assertThat(checks).hasSize(2).allSatisfy(check ->
|
||||
{
|
||||
assertThat(check.getComponent()).isEqualTo("backups");
|
||||
assertThat(check.getKind()).isEqualTo(HealthCheckKind.DEPENDENCY);
|
||||
assertThat(check.getCategory()).isEqualTo(HealthCategory.STORAGE);
|
||||
assertThat(check.getIncludeInReadiness()).isFalse();
|
||||
assertThat(check.getName()).matches("[a-z0-9]+([._-][a-z0-9]+)*");
|
||||
});
|
||||
assertThat(checks.get(0).getIntervalSeconds()).isEqualTo(900);
|
||||
assertThat(checks.get(0).getTimeoutMs()).isEqualTo(5000);
|
||||
assertThat(QCodeLoader.getAdHoc(HealthCheck.class, checks.get(0).getCheck()).run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
|
||||
HealthCheckMetaData destination = checks.get(1);
|
||||
assertThat(destination.getName()).isEqualTo("backups.destination." + active.getValueInteger("id"));
|
||||
assertThat(destination.getIntervalSeconds()).isEqualTo(86400);
|
||||
assertThat(destination.getTimeoutMs()).isEqualTo(30000);
|
||||
assertThat(destination.getCheckInstance().run(input()).getStatus()).isEqualTo(HealthStatus.UP);
|
||||
catalog.update(BackupsTables.PLAN, active.getValueInteger("id"), Map.of("isActive", false));
|
||||
assertThat(destination.getCheckInstance().run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
|
||||
assertThat(registry.resolveAll(QContext.getQInstance())).hasSize(1);
|
||||
VerifyTestSupport.plan(Map.of("name", "New", "planKey", "new"));
|
||||
assertThat(registry.resolveAll(QContext.getQInstance())).hasSize(2);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Carry over the indicator's empty, critical, ordinary and inactive cases.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testSlaUpDownAndDegraded() throws Exception
|
||||
{
|
||||
BackupsSlaHealthCheck check = new BackupsSlaHealthCheck();
|
||||
HealthCheckOutcome empty = check.run(input());
|
||||
assertThat(empty.getStatus()).isEqualTo(HealthStatus.UP);
|
||||
assertThat(empty.getDetails().getValues()).containsEntry("activePlans", 0);
|
||||
QRecord critical = VerifyTestSupport.plan(Map.of("critical", true));
|
||||
QRecord ordinary = VerifyTestSupport.plan(Map.of("name", "Other", "planKey", "other"));
|
||||
QRecord inactive = VerifyTestSupport.plan(Map.of("name", "Off", "planKey", "off", "isActive", false, "critical", true));
|
||||
setStatus(inactive, "LATE");
|
||||
setStatus(critical, "OK");
|
||||
setStatus(ordinary, "OK");
|
||||
assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UP);
|
||||
for(String status : List.of("LATE", "FAILING", "MISSING_IN_STORAGE"))
|
||||
{
|
||||
setStatus(ordinary, status);
|
||||
assertThat(check.run(input()).getStatus()).as(status).isEqualTo(HealthStatus.DEGRADED);
|
||||
setStatus(critical, status);
|
||||
HealthCheckOutcome down = check.run(input());
|
||||
assertThat(down.getStatus()).as(status).isEqualTo(HealthStatus.DOWN);
|
||||
assertThat(down.getDetails().getValues()).containsEntry("activePlans", 2).containsEntry("slaStatus." + status, 2);
|
||||
assertClean(down);
|
||||
setStatus(critical, "OK");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unknown or unverified plans do not become a false UP or a critical DOWN.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testSlaUnknownAndMixedStates() throws Exception
|
||||
{
|
||||
QRecord plan = VerifyTestSupport.plan(Map.of("critical", true));
|
||||
BackupsSlaHealthCheck check = new BackupsSlaHealthCheck();
|
||||
assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
|
||||
for(String status : List.of("UNVERIFIED", "UNKNOWN"))
|
||||
{
|
||||
setStatus(plan, status);
|
||||
assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
|
||||
}
|
||||
QRecord other = VerifyTestSupport.plan(Map.of("name", "Other", "planKey", "other"));
|
||||
setStatus(other, "UNVERIFIED");
|
||||
assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
|
||||
setStatus(other, "OK");
|
||||
assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UP);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Preserve the old indicator's no-context behavior and existing sessions.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testWithoutContext() throws Exception
|
||||
{
|
||||
QRecord plan = VerifyTestSupport.plan(Map.of());
|
||||
QInstance instance = QContext.getQInstance();
|
||||
QSession session = QContext.getQSession();
|
||||
new BackupsSlaHealthCheck().run(input());
|
||||
assertThat(QContext.getQSession()).isSameAs(session);
|
||||
QContext.clear();
|
||||
try
|
||||
{
|
||||
HealthCheckInput input = new HealthCheckInput().withQInstance(instance).withDeadline(Instant.now().plusSeconds(5));
|
||||
assertThat(new BackupsSlaHealthCheck().run(input).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
|
||||
assertThat(QContext.getQInstance()).isNull();
|
||||
assertThat(new BackupsDestinationHealthChecks().supply(instance)).hasSize(1);
|
||||
assertThat(QContext.getQInstance()).isNull();
|
||||
BackupsRuntime.getConfig().withStorageResolver(alias ->
|
||||
{
|
||||
assertThat(QContext.getQSession()).isInstanceOf(com.kingsrook.qqq.backend.core.model.session.QSystemUserSession.class);
|
||||
return (storage);
|
||||
});
|
||||
assertThat(destination(plan).run(input).getStatus()).isEqualTo(HealthStatus.UP);
|
||||
assertThat(QContext.getQInstance()).isNull();
|
||||
}
|
||||
finally
|
||||
{
|
||||
QContext.init(instance, session);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Locked and unlocked plans only list/head; neither reads object contents.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testDestinationReadOnlyAndMissingManifest() throws Exception
|
||||
{
|
||||
for(String lockMode : List.of("NONE", "GOVERNANCE"))
|
||||
{
|
||||
BackupsRuntime.getConfig().withStorageResolver(alias -> BackupsTestUtils.BACKUPS);
|
||||
QRecord plan = VerifyTestSupport.plan(Map.of("name", lockMode, "planKey", lockMode.toLowerCase(), "objectLockMode", lockMode, "objectLockDays", 3, "pruneBucketAlias", "prune"));
|
||||
BackupsRuntime.getConfig().withStorageResolver(alias ->
|
||||
{
|
||||
assertThat(alias).as("never resolve prune credentials").isNotEqualTo("prune");
|
||||
return (storage);
|
||||
});
|
||||
Integer runId = catalog.insert(BackupsTables.RUN, Map.of("manifestKey", "manifest.json"));
|
||||
catalog.update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("lastSuccessRunId", runId));
|
||||
storage.putBytes("manifest.json", new byte[] { 1 });
|
||||
assertThat(destination(plan).run(input()).getStatus()).isEqualTo(HealthStatus.UP);
|
||||
storage.clear();
|
||||
assertThat(destination(plan).run(input()).getStatus()).isEqualTo(HealthStatus.DOWN);
|
||||
}
|
||||
assertThat(storage.lists).isEqualTo(4);
|
||||
assertThat(storage.heads).isEqualTo(4);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A deleted plan is UNKNOWN and does not touch storage.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testDeletedPlan() throws Exception
|
||||
{
|
||||
QRecord plan = VerifyTestSupport.plan(Map.of());
|
||||
catalog.deleteByIds(BackupsTables.PLAN, List.of(plan.getValueInteger("id")));
|
||||
assertThat(destination(plan).run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
|
||||
assertThat(storage.lists).isZero();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** No work starts with an expired deadline, including direct probe calls.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testExpiredDeadline() throws Exception
|
||||
{
|
||||
QRecord plan = VerifyTestSupport.plan(Map.of());
|
||||
HealthCheckInput expired = input().withDeadline(Instant.now().minusSeconds(1));
|
||||
assertThat(new BackupsSlaHealthCheck().run(expired).getStatus()).isEqualTo(HealthStatus.DOWN);
|
||||
assertThat(destination(plan).run(expired).getStatus()).isEqualTo(HealthStatus.DOWN);
|
||||
assertThat(DestinationProbe.probeReadOnly(plan, expired.getDeadline()).ok()).isFalse();
|
||||
assertThat(storage.lists).isZero();
|
||||
assertThat(storage.heads).isZero();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Runner cancellation after list prevents the following manifest head.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testCancellationBetweenStorageCalls() throws Exception
|
||||
{
|
||||
QRecord plan = VerifyTestSupport.plan(Map.of());
|
||||
Integer runId = catalog.insert(BackupsTables.RUN, Map.of("manifestKey", "manifest.json"));
|
||||
catalog.update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("lastSuccessRunId", runId));
|
||||
storage.interruptAfterList = true;
|
||||
try
|
||||
{
|
||||
assertThat(destination(plan).run(input()).getStatus()).isEqualTo(HealthStatus.DOWN);
|
||||
assertThat(storage.lists).isEqualTo(1);
|
||||
assertThat(storage.heads).isZero();
|
||||
}
|
||||
finally
|
||||
{
|
||||
Thread.interrupted();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Poison configuration, persisted state and provider errors. sanitize is
|
||||
** package-private in core; assert raw outcomes already contain no secrets.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testNoSecretsInSuccessAndFailureOutcomes() throws Exception
|
||||
{
|
||||
QRecord plan = VerifyTestSupport.plan(Map.of("name", SECRET, "storageBucketAlias", SECRET, "critical", true));
|
||||
Integer runId = catalog.insert(BackupsTables.RUN, Map.of("manifestKey", SECRET));
|
||||
catalog.update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("lastSuccessRunId", runId));
|
||||
storage.putBytes(SECRET, new byte[] { 1 });
|
||||
for(String status : List.of("OK", "LATE", "UNVERIFIED", SECRET))
|
||||
{
|
||||
setStatus(plan, status);
|
||||
assertClean(new BackupsSlaHealthCheck().run(input()));
|
||||
assertClean(destination(plan).run(input()));
|
||||
}
|
||||
storage.failList = true;
|
||||
HealthCheckOutcome failure = destination(plan).run(input());
|
||||
assertThat(failure.getStatus()).isEqualTo(HealthStatus.DOWN);
|
||||
assertClean(failure);
|
||||
storage.failList = false;
|
||||
storage.failHead = true;
|
||||
failure = destination(plan).run(input());
|
||||
assertThat(failure.getStatus()).isEqualTo(HealthStatus.DOWN);
|
||||
assertClean(failure);
|
||||
BackupsRuntime.getConfig().withStorageResolver(alias ->
|
||||
{
|
||||
throw (new IllegalStateException(SECRET));
|
||||
});
|
||||
failure = destination(plan).run(input());
|
||||
assertThat(failure.getStatus()).isEqualTo(HealthStatus.DOWN);
|
||||
assertClean(failure);
|
||||
assertClean(new BackupsSlaHealthCheck().run(input()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Catalog failures are DOWN with fixed messages, even if metadata is poisoned.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testCatalogFailureDoesNotExposeSecrets()
|
||||
{
|
||||
QContext.getQInstance().getTable(BackupsTables.PLAN).setBackendName(SECRET);
|
||||
HealthCheckOutcome sla = new BackupsSlaHealthCheck().run(input());
|
||||
HealthCheckOutcome destination = new BackupsDestinationHealthCheck(1).run(input());
|
||||
assertThat(sla.getStatus()).isEqualTo(HealthStatus.DOWN);
|
||||
assertThat(destination.getStatus()).isEqualTo(HealthStatus.DOWN);
|
||||
assertClean(sla);
|
||||
assertClean(destination);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Update persisted SLA state without running the scheduled calculator.
|
||||
*******************************************************************************/
|
||||
private void setStatus(QRecord plan, String status) throws Exception
|
||||
{
|
||||
catalog.update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("slaStatus", status));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fresh bounded input for each call.
|
||||
*******************************************************************************/
|
||||
private HealthCheckInput input()
|
||||
{
|
||||
return (new HealthCheckInput().withQInstance(QContext.getQInstance()).withDeadline(Instant.now().plusSeconds(5)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Construct the same per-plan check as the supplier.
|
||||
*******************************************************************************/
|
||||
private BackupsDestinationHealthCheck destination(QRecord plan)
|
||||
{
|
||||
return (new BackupsDestinationHealthCheck(plan.getValueInteger("id")));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Validate the raw public message and flattened detail representation.
|
||||
*******************************************************************************/
|
||||
private void assertClean(HealthCheckOutcome outcome)
|
||||
{
|
||||
assertThat(outcome.getMessage()).doesNotContain(SECRET, "\n", "\r");
|
||||
assertThat(outcome.getDetails().toJson()).doesNotContain(SECRET);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Existing memory fake with read failures and assertions against side effects.
|
||||
*******************************************************************************/
|
||||
private static class ReadOnlyStorage extends InMemoryBackupStorage
|
||||
{
|
||||
private final String credential;
|
||||
private boolean failList;
|
||||
private boolean failHead;
|
||||
private boolean interruptAfterList;
|
||||
private int lists;
|
||||
private int heads;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Mimic the credential held by a host-provided storage adapter.
|
||||
*******************************************************************************/
|
||||
ReadOnlyStorage(String credential)
|
||||
{
|
||||
this.credential = credential;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Record list calls and inject credential-bearing failures or cancellation.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<StoredObjectInfo> list(String prefix)
|
||||
{
|
||||
lists++;
|
||||
if(failList)
|
||||
{
|
||||
throw (new IllegalStateException(credential));
|
||||
}
|
||||
if(interruptAfterList)
|
||||
{
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
return (super.list(prefix));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Record head calls and inject credential-bearing failures.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public Optional<StoredObjectInfo> head(String key)
|
||||
{
|
||||
heads++;
|
||||
if(failHead)
|
||||
{
|
||||
throw (new IllegalStateException(credential));
|
||||
}
|
||||
return (super.head(key));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Health probes must not write, even to an unlocked bucket.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public OutputStream openWrite(String key, WriteOptions options)
|
||||
{
|
||||
throw (new AssertionError("Health probe wrote an object"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Health probes must not fetch backup contents.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public InputStream openRead(String key)
|
||||
{
|
||||
throw (new AssertionError("Health probe read object contents"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Health probes must not delete.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public DeleteResult deletePermanent(String key)
|
||||
{
|
||||
throw (new AssertionError("Health probe deleted an object"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Capability diagnostics can be active probes; health must not invoke them.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public Set<StorageCapability> capabilities()
|
||||
{
|
||||
throw (new AssertionError("Health probe requested capabilities"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.health;
|
||||
|
||||
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.backups.VerifyTestSupport;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.middleware.health.model.metadata.HealthCheckResult;
|
||||
import com.kingsrook.qqq.middleware.health.model.metadata.HealthStatus;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The health indicator: DOWN only when a critical, active plan is LATE, FAILING or
|
||||
** MISSING_IN_STORAGE.
|
||||
*******************************************************************************/
|
||||
class BackupsHealthIndicatorTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp() throws Exception
|
||||
{
|
||||
VerifyTestSupport.setUp(3);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private void setStatus(QRecord plan, String status) throws Exception
|
||||
{
|
||||
new BackupCatalog().update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("slaStatus", status));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** UP with details when nothing critical is wrong, including a non-critical plan
|
||||
** that is LATE; DOWN when a critical plan is LATE, FAILING or MISSING_IN_STORAGE.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testUpAndDown() throws Exception
|
||||
{
|
||||
BackupsHealthIndicator indicator = new BackupsHealthIndicator();
|
||||
assertThat(indicator.getName()).isEqualTo("backups");
|
||||
|
||||
HealthCheckResult empty = indicator.check(QContext.getQInstance());
|
||||
assertThat(empty.getStatus()).isEqualTo(HealthStatus.UP);
|
||||
assertThat(empty.getDetails()).containsEntry("activePlans", 0);
|
||||
|
||||
QRecord critical = VerifyTestSupport.plan(Map.of("critical", true));
|
||||
QRecord ordinary = VerifyTestSupport.plan(Map.of("name", "Other", "planKey", "other"));
|
||||
VerifyTestSupport.plan(Map.of("name", "Off", "planKey", "off", "isActive", false, "critical", true));
|
||||
|
||||
setStatus(critical, "OK");
|
||||
setStatus(ordinary, "LATE");
|
||||
HealthCheckResult up = indicator.check(QContext.getQInstance());
|
||||
assertThat(up.getStatus()).isEqualTo(HealthStatus.UP);
|
||||
assertThat(up.getDetails()).containsEntry("activePlans", 2).doesNotContainKey("criticalPlansWithProblems");
|
||||
assertThat(up.getDurationMs()).isNotNull();
|
||||
|
||||
for(String status : new String[] { "LATE", "FAILING", "MISSING_IN_STORAGE" })
|
||||
{
|
||||
setStatus(critical, status);
|
||||
HealthCheckResult down = indicator.check(QContext.getQInstance());
|
||||
assertThat(down.getStatus()).as(status).isEqualTo(HealthStatus.DOWN);
|
||||
assertThat(down.getDetails().get("criticalPlansWithProblems").toString()).contains("nightly: " + status);
|
||||
}
|
||||
|
||||
setStatus(critical, "UNVERIFIED");
|
||||
assertThat(indicator.check(QContext.getQInstance()).getStatus()).as("not verified is not down").isEqualTo(HealthStatus.UP);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** On a thread with no QQQ context (a health request) the indicator makes its own
|
||||
** and cleans up.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testWorksWithoutAContext() throws Exception
|
||||
{
|
||||
QInstance qInstance = QContext.getQInstance();
|
||||
QContext.clear();
|
||||
try
|
||||
{
|
||||
assertThat(new BackupsHealthIndicator().check(qInstance).getStatus()).isEqualTo(HealthStatus.UP);
|
||||
assertThat(QContext.getQInstance()).isNull();
|
||||
}
|
||||
finally
|
||||
{
|
||||
QContext.init(qInstance, new com.kingsrook.qqq.backend.core.model.session.QSession());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The indicator's own context runs as the core system user, which token-based
|
||||
** authentication modules accept without a token.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testOwnContextUsesSystemUserSession()
|
||||
{
|
||||
assertThat(BackupsHealthIndicator.systemSession(QContext.getQInstance()))
|
||||
.isInstanceOf(com.kingsrook.qqq.backend.core.model.session.QSystemUserSession.class);
|
||||
}
|
||||
}
|
||||
@@ -87,15 +87,10 @@ class BackupsMetricsTest
|
||||
String empty = BackupsMetrics.renderCached();
|
||||
assertThat(empty).doesNotContain("{instance_name");
|
||||
|
||||
VerifyTestSupport.clock.advance(Duration.ofSeconds(30));
|
||||
assertThat(BackupsMetrics.renderCached()).as("cached").isSameAs(empty);
|
||||
VerifyTestSupport.clock.advance(Duration.ofSeconds(31));
|
||||
assertThat(BackupsMetrics.renderCached()).as("expired").isNotSameAs(empty);
|
||||
|
||||
/////////////////////////////////////////////////////////////
|
||||
// a saved plan and a finished run clear the cache at once //
|
||||
/////////////////////////////////////////////////////////////
|
||||
VerifyTestSupport.backup(VerifyTestSupport.plan(Map.of()));
|
||||
VerifyTestSupport.clock.advance(Duration.ofSeconds(30));
|
||||
assertThat(BackupsMetrics.renderCached()).as("cached").isEqualTo(empty);
|
||||
VerifyTestSupport.clock.advance(Duration.ofSeconds(31));
|
||||
assertThat(BackupsMetrics.renderCached()).contains("plan=\"nightly\"");
|
||||
|
||||
assertThat(BackupsMetrics.escape("a\"b\\c\nd")).isEqualTo("a\\\"b\\\\c\\nd");
|
||||
|
||||
@@ -29,8 +29,6 @@ import com.kingsrook.qbits.backups.manifest.BackupManifest;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.scope.PlanSettings;
|
||||
import com.kingsrook.qbits.backups.scope.ScopeResolver;
|
||||
import com.kingsrook.qbits.backups.scope.TableRole;
|
||||
import com.kingsrook.qbits.backups.scope.TableRoles;
|
||||
import com.kingsrook.qqq.backend.core.actions.async.AsyncJobCallback;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.jobs.plans.JobChunkScope;
|
||||
@@ -39,11 +37,6 @@ import com.kingsrook.qqq.backend.core.jobs.progress.JobLogLevel;
|
||||
import com.kingsrook.qqq.backend.core.jobs.progress.JobPhase;
|
||||
import com.kingsrook.qqq.backend.core.jobs.progress.JobProgress;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
@@ -237,38 +230,6 @@ class BackupEngineTest
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A table on a backend with no query interface (a storage-only module, like
|
||||
** qbit-edi's edi_preview_download) is left out of the scope, even when it is
|
||||
** selected by name, and the run succeeds. Before the fix the reader queried it
|
||||
** and the run FAILED with "Query is not implemented in this module".
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testUnqueryableTableIsExcluded() throws Exception
|
||||
{
|
||||
QInstance qInstance = QContext.getQInstance();
|
||||
qInstance.addBackend(new QBackendMetaData().withName("storageOnly").withBackendType(StorageOnlyBackendModule.class));
|
||||
qInstance.addTable(new QTableMetaData().withName("previewDownload").withBackendName("storageOnly").withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER)));
|
||||
|
||||
BackupRunResult result = run(plan(Map.of()));
|
||||
assertThat(result.getErrorMessage()).isNull();
|
||||
assertThat(result.getStatus()).isEqualTo("SUCCEEDED_WITH_WARNINGS");
|
||||
BackupManifest manifest = BackupManifest.read(BackupsTestUtils.BACKUPS.getBytes(result.getManifestKey()));
|
||||
assertThat(manifest.table("previewDownload")).isNull();
|
||||
assertThat(manifest.table("blogPost")).isNotNull();
|
||||
|
||||
assertThat(TableRoles.roleOf(qInstance, qInstance.getTable("previewDownload"))).isEqualTo(TableRole.EXTERNAL);
|
||||
assertThat(TableRoles.isQueryable(qInstance, qInstance.getTable("blogPost"))).isTrue();
|
||||
|
||||
ScopeResolver.ResolvedScope scope = new ScopeResolver(qInstance).resolve(PlanSettings.adHoc().with("scopeType", "TABLES").with("includeRoles", "DATA,EXTERNAL"),
|
||||
List.of(new ScopeResolver.ScopeRule(true, "TABLE", "previewDownload", null, false, null), new ScopeResolver.ScopeRule(true, "ROLE", "EXTERNAL", null, false, null)));
|
||||
assertThat(scope.names()).doesNotContain("previewDownload");
|
||||
assertThat(scope.excluded()).contains(new ScopeResolver.ExcludedTable("previewDownload", ScopeResolver.NOT_QUERYABLE));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Required snapshots fail before anything is written (no backend can open one yet).
|
||||
*******************************************************************************/
|
||||
@@ -391,14 +352,13 @@ class BackupEngineTest
|
||||
BackupRunRequest request = BackupRunFactory.forPlan(plan(Map.of()), "MANUAL").withProgress(progress)
|
||||
.withReader((table, filter, start, size, transaction, sink) ->
|
||||
{
|
||||
throw (new com.kingsrook.qqq.backend.core.exceptions.QException("https://storage-user:storage-password@host/?secret=storage-secret accessKey=storage-access Detail: Key (email)=(private-row) already exists."));
|
||||
throw (new com.kingsrook.qqq.backend.core.exceptions.QException("https://storage-user:storage-password@host/?secret=storage-secret accessKey=storage-access row=private-row"));
|
||||
});
|
||||
BackupRunResult result = new BackupEngine().execute(request);
|
||||
assertThat(result.getStatus()).isEqualTo("FAILED");
|
||||
assertThat(progress.logs).contains("ERROR:" + result.getErrorMessage());
|
||||
assertThat(progress.getSnapshot().getPhases()).anyMatch(p -> p.getStatus() == JobPhase.Status.FAILED);
|
||||
assertThat(result.getErrorMessage()).startsWith("Backup failed: QException: https://[REDACTED]@host/");
|
||||
assertThat(String.join("\n", progress.logs) + result.getErrorMessage() + result.getErrorDetail()).doesNotContain("storage-user", "storage-password", "storage-secret", "storage-access", "private-row");
|
||||
assertThat(String.join("\n", progress.logs) + result.getErrorMessage()).doesNotContain("storage-user", "storage-password", "storage-secret", "storage-access", "private-row", "https://");
|
||||
}
|
||||
|
||||
|
||||
@@ -448,52 +408,7 @@ class BackupEngineTest
|
||||
BackupRunResult result = new BackupEngine().execute(BackupRunFactory.forPlan(plan(Map.of()), "MANUAL").withProgress(progress).withStorage(storage));
|
||||
assertThat(result.getStatus()).isEqualTo("FAILED");
|
||||
assertThat(progress.logs).contains("ERROR:" + result.getErrorMessage());
|
||||
assertThat(String.join("\n", progress.logs) + result.getErrorMessage() + result.getErrorDetail()).doesNotContain("endpoint-user", "endpoint-password", "access-secret", "secret-secret");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A failed put records what an operator needs on the run row, with no secrets:
|
||||
** error_message is "Backup failed: Class: message" and error_detail has the
|
||||
** class, the cause chain and the root cause frames; the presigned signature,
|
||||
** the access key id and the URL user-info are redacted, the object key is kept.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testFailedPutRecordsSanitizedMessageAndDetail() throws Exception
|
||||
{
|
||||
var storage = new com.kingsrook.qbits.backups.storage.InMemoryBackupStorage()
|
||||
{
|
||||
/*******************************************************************************
|
||||
** A MinIO-style failure with a presigned URL in the message.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public java.io.OutputStream openWrite(String key, com.kingsrook.qbits.backups.storage.WriteOptions options)
|
||||
{
|
||||
throw (new com.kingsrook.qbits.backups.storage.BackupStorageException(com.kingsrook.qbits.backups.storage.BackupStorageException.Reason.ACCESS_DENIED,
|
||||
"AccessDenied on PUT https://minio-user:minio-pass@minio.galaxy.lan/kof22-backups/" + key
|
||||
+ "?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20261016%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Signature=deadbeefcafe0123",
|
||||
new IOException("XMinioStorageFull: Storage backend has reached its minimum free drive threshold. Authorization: AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/x, Signature=feedface")));
|
||||
}
|
||||
};
|
||||
BackupRunResult result = new BackupEngine().execute(BackupRunFactory.forPlan(plan(Map.of()), "MANUAL").withStorage(storage));
|
||||
assertThat(result.getStatus()).isEqualTo("FAILED");
|
||||
|
||||
QRecord run = new BackupCatalog().get(BackupsTables.RUN, result.getRunId()).orElseThrow();
|
||||
String message = run.getValueString("errorMessage");
|
||||
String detail = run.getValueString("errorDetail");
|
||||
assertThat(message).isEqualTo(result.getErrorMessage());
|
||||
assertThat(detail).isEqualTo(result.getErrorDetail());
|
||||
|
||||
assertThat(message).startsWith("Backup failed: BackupStorageException: AccessDenied on PUT https://[REDACTED]@minio.galaxy.lan/kof22-backups/qqq-backups/testapp/nightly/")
|
||||
.doesNotContain("see server logs");
|
||||
assertThat(detail).startsWith("com.kingsrook.qbits.backups.storage.BackupStorageException: AccessDenied on PUT")
|
||||
.contains("\nCaused by: java.io.IOException: XMinioStorageFull: Storage backend has reached its minimum free drive threshold.")
|
||||
.contains("X-Amz-Signature=[REDACTED]")
|
||||
.contains("Root cause stack (top ")
|
||||
.contains("\tat ");
|
||||
assertThat(detail.length()).isLessThanOrEqualTo(FailureText.MAX_DETAIL_CHARS);
|
||||
assertThat(message + detail).doesNotContain("deadbeefcafe0123", "AKIAIOSFODNN7EXAMPLE", "minio-user", "minio-pass", "feedface");
|
||||
assertThat(String.join("\n", progress.logs) + result.getErrorMessage()).doesNotContain("endpoint-user", "endpoint-password", "access-secret", "secret-secret", "https://");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1,186 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.run;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.sql.SQLException;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unit tests of the failure text sanitizer, summary and detail.
|
||||
*******************************************************************************/
|
||||
class FailureTextTest
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
** Null and plain text pass through; object keys and bucket names are kept.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testPlainTextIsKept()
|
||||
{
|
||||
assertThat(FailureText.sanitize(null)).isNull();
|
||||
String plain = "Query is not implemented in this module: PreviewDownloadBackend (bucket kof22-backups, key qqq-backups/kof22/nightly/2026/10/16/run/tables/a/part-00001.jsonl.gz)";
|
||||
assertThat(FailureText.sanitize(plain)).isEqualTo(plain);
|
||||
assertThat(FailureText.sanitize("objectKey=qqq-backups/x/y.gz bucket=kof22-backups")).isEqualTo("objectKey=qqq-backups/x/y.gz bucket=kof22-backups");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** URL user-info and presigned query parameters are redacted; host and path kept.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testUrls()
|
||||
{
|
||||
assertThat(FailureText.sanitize("PUT https://user:pass@minio.lan/bucket/a/b.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2Fx&X-Amz-Signature=abc123&partNumber=2 failed"))
|
||||
.isEqualTo("PUT https://[REDACTED]@minio.lan/bucket/a/b.gz?X-Amz-Algorithm=[REDACTED]&X-Amz-Credential=[REDACTED]&X-Amz-Signature=[REDACTED]&partNumber=2 failed");
|
||||
assertThat(FailureText.sanitize("jdbc:postgresql://app:hunter2@db:5432/app")).isEqualTo("jdbc:postgresql://[REDACTED]@db:5432/app");
|
||||
assertThat(FailureText.sanitize("https://h/p?Signature=s1&Expires=1&AWSAccessKeyId=k&access_token=t&key=v"))
|
||||
.isEqualTo("https://h/p?Signature=[REDACTED]&Expires=1&AWSAccessKeyId=[REDACTED]&access_token=[REDACTED]&key=[REDACTED]");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** key=value and JSON secrets, Authorization and Bearer values.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testAssignmentsAndHeaders()
|
||||
{
|
||||
assertThat(FailureText.sanitize("password=p1 secret=\"s 2\" token='t3' key=k4 accessKey=a5 secretKey=s6 apiKey=x7 db.password = p8, other=ok"))
|
||||
.isEqualTo("password=[REDACTED] secret=[REDACTED] token=[REDACTED] key=[REDACTED] accessKey=[REDACTED] secretKey=[REDACTED] apiKey=[REDACTED] db.password = [REDACTED], other=ok");
|
||||
assertThat(FailureText.sanitize("{\"clientSecret\": \"abc\", \"name\": \"keep\"}")).isEqualTo("{\"clientSecret\": \"[REDACTED]\", \"name\": \"keep\"}");
|
||||
assertThat(FailureText.sanitize("Authorization: Bearer eyJhbGciOi.payload.sig\nnext line")).isEqualTo("Authorization: [REDACTED]\nnext line");
|
||||
assertThat(FailureText.sanitize("sent Bearer abcdefghijk to the API")).isEqualTo("sent Bearer [REDACTED] to the API");
|
||||
assertThat(FailureText.sanitize("id AKIAIOSFODNN7EXAMPLE and ASIAIOSFODNN7EXAMPLE")).isEqualTo("id [REDACTED] and [REDACTED]");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** PEM blocks (terminated or not) and Postgres row values.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testPemAndRowValues()
|
||||
{
|
||||
assertThat(FailureText.sanitize("bad key -----BEGIN PRIVATE KEY-----\nMIIabc\n-----END PRIVATE KEY----- here")).isEqualTo("bad key [REDACTED PEM] here");
|
||||
assertThat(FailureText.sanitize("cut -----BEGIN RSA PRIVATE KEY-----\nMIIabc")).isEqualTo("cut [REDACTED PEM]");
|
||||
assertThat(FailureText.sanitize("duplicate key value violates unique constraint \"u_email\" Key (email)=(ada@example.com) already exists."))
|
||||
.isEqualTo("duplicate key value violates unique constraint \"u_email\" Key (email)=([REDACTED]) already exists.");
|
||||
assertThat(FailureText.sanitize("ERROR: null value in column \"x\"\n Detail: Failing row contains (1, ada, secret-row).\n Where: x"))
|
||||
.isEqualTo("ERROR: null value in column \"x\"\n Detail: [REDACTED]\n Where: x");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The summary is short, sanitized and bounded; no message falls back to the class.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testSummary()
|
||||
{
|
||||
assertThat(FailureText.summary(new IllegalStateException("Query is not implemented in this module: PreviewDownloadBackend")))
|
||||
.isEqualTo("Backup failed: IllegalStateException: Query is not implemented in this module: PreviewDownloadBackend");
|
||||
assertThat(FailureText.summary(new NullPointerException())).isEqualTo("Backup failed: NullPointerException");
|
||||
assertThat(FailureText.summary("Could not create the run record", new IOException(" password=x "))).isEqualTo("Could not create the run record: IOException: password=[REDACTED]");
|
||||
assertThat(FailureText.summary(new IOException("x".repeat(2000)))).hasSize("Backup failed: IOException: ".length() + FailureText.MAX_SUMMARY_MESSAGE_CHARS + 3);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The detail has the class, the cause chain and the root cause frames, bounded.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testDetail()
|
||||
{
|
||||
SQLException root = new SQLException("Key (id)=(4242xyz) already exists.");
|
||||
IOException middle = new IOException("write failed password=pwsecret", root);
|
||||
RuntimeException top = new RuntimeException("Backup of t failed", middle);
|
||||
String detail = FailureText.detail(top);
|
||||
assertThat(detail).startsWith("java.lang.RuntimeException: Backup of t failed\nCaused by: java.io.IOException: write failed password=[REDACTED]\nCaused by: java.sql.SQLException: Key (id)=([REDACTED]) already exists.\nRoot cause stack (top ");
|
||||
assertThat(detail).contains("\tat com.kingsrook.qbits.backups.run.FailureTextTest.testDetail(FailureTextTest.java:");
|
||||
assertThat(detail.split("\n\tat ")).hasSize(FailureText.ROOT_CAUSE_FRAMES + 1);
|
||||
assertThat(detail).doesNotContain("4242xyz", "pwsecret");
|
||||
|
||||
IOException noFrames = new IOException((String) null);
|
||||
noFrames.setStackTrace(new StackTraceElement[0]);
|
||||
assertThat(FailureText.detail(noFrames)).isEqualTo("java.io.IOException");
|
||||
|
||||
String huge = FailureText.detail(new IOException("y".repeat(20_000)));
|
||||
assertThat(huge).hasSize(FailureText.MAX_DETAIL_CHARS).endsWith("... (truncated)");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A cause cycle does not loop.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testCauseCycle()
|
||||
{
|
||||
CyclicException a = new CyclicException("a");
|
||||
CyclicException b = new CyclicException("b");
|
||||
a.setCause(b);
|
||||
b.setCause(a);
|
||||
assertThat(FailureText.detail(a)).startsWith("com.kingsrook.qbits.backups.run.FailureTextTest$CyclicException: a\nCaused by: com.kingsrook.qbits.backups.run.FailureTextTest$CyclicException: b\nRoot cause stack");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** An exception whose cause can be set to form a cycle.
|
||||
***************************************************************************/
|
||||
private static class CyclicException extends Exception
|
||||
{
|
||||
private Throwable cause;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor.
|
||||
*******************************************************************************/
|
||||
CyclicException(String message)
|
||||
{
|
||||
super(message);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Set the cause.
|
||||
*******************************************************************************/
|
||||
void setCause(Throwable cause)
|
||||
{
|
||||
this.cause = cause;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The cause.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public synchronized Throwable getCause()
|
||||
{
|
||||
return (cause);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,86 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.run;
|
||||
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A storage-only backend module (like qbit-edi's PreviewDownloadBackend): it
|
||||
** streams objects but has no query interface, so a table on it cannot be read.
|
||||
*******************************************************************************/
|
||||
public class StorageOnlyBackendModule implements QBackendModuleInterface, QStorageInterface
|
||||
{
|
||||
public static final String TYPE = "test-storage-only";
|
||||
|
||||
static
|
||||
{
|
||||
QBackendModuleDispatcher.registerBackendModule(new StorageOnlyBackendModule());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The backend type.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getBackendType()
|
||||
{
|
||||
return (TYPE);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Storage only.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QStorageInterface getStorageInterface()
|
||||
{
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Nothing is stored.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public InputStream getInputStream(StorageInput input)
|
||||
{
|
||||
return (new ByteArrayInputStream(new byte[0]));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Writes are discarded.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public OutputStream createOutputStream(StorageInput input)
|
||||
{
|
||||
return (new ByteArrayOutputStream());
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user