Compare commits

...
Author SHA1 Message Date
CircleCI 9855521637 Bump version to 0.1.0-RC.8 [skip ci] 2026-10-15 03:36:37 +00:00
james.maesandClaude Opus 5.5 99f74b4b5f Merge develop into release/0.1 for 0.1.0-RC.8 (#23 health checks)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 22:32:35 -05:00
james.maes eb413b6e15 Merge pull request #23 from QRun-IO/feature/health-checks
feat(health): register backups SLA and destination checks with core central health
2026-10-14 22:05:12 -05:00
james.maes 1e17dada29 Merge remote-tracking branch 'origin/develop' into feature/health-checks
# Conflicts:
#	CHANGELOG.md
2026-10-14 22:02:35 -05:00
CircleCI fa9c4c1774 Bump version to 0.1.0-RC.7 [skip ci] 2026-10-15 02:21:57 +00:00
james.maesandClaude Opus 5.5 c8ba79f55b Merge develop into release/0.1 for 0.1.0-RC.7 (#26 QQQ RC.8, #27 help bundle)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 21:18:12 -05:00
james.maesandClaude Opus 5.5 9a2a1dd88e Merge origin/develop into feature/health-checks; pin QQQ 4.1.0-RC.8
RC.8 ships the central health kernel (QRun-IO/qqq#1207).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 21:14:59 -05:00
CircleCI 7830226982 Bump version to 0.1.0-RC.6 [skip ci] 2026-10-14 22:44:27 +00:00
james.maesandClaude Opus 5.5 b1f258a3d5 Merge develop into release/0.1 for 0.1.0-RC.6 (#24)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 17:41:40 -05:00
CircleCI 3df24cda5a Bump version to 0.1.0-RC.5 [skip ci] 2026-10-14 22:22:38 +00:00
james.maesandClaude Opus 5.5 987c20899b Merge develop into release/0.1 for 0.1.0-RC.5 (#25 setup card, default plan, scheduled verify)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 17:19:54 -05:00
james.maesandClaude Opus 5.5 f9767ede83 chore: drop optional qqq-middleware-health dependency; docs for backups health checks
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 14:55:03 -05:00
james.maesandClaude Opus 5.5 8d56613f7d feat(health): register backups SLA and destination checks; drop BackupsHealthIndicator
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 14:55:03 -05:00
james.maesandClaude Opus 5.5 0f9178dc14 feat(health): read-only destination probe mode
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 14:55:02 -05:00
CircleCI 14bb6fa1b4 Bump version to 0.1.0-RC.4 [skip ci] 2026-10-14 12:26:09 +00:00
james.maesandClaude Opus 5.5 b1ff70d7d3 Merge develop into release/0.1 for 0.1.0-RC.4 (#21 schema fingerprint, keyset pager, guardrails)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 07:21:32 -05:00
CircleCI cda334f2fe Bump version to 0.1.0-RC.3 [skip ci] 2026-10-14 00:53:05 +00:00
james.maesandClaude Opus 5.5 590b132fb6 Merge develop into release/0.1 for 0.1.0-RC.3 (core alerts, renamed alert types, #20)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 19:50:34 -05:00
CircleCI f0a8c5000e Bump version to 0.1.0-RC.2 [skip ci] 2026-10-13 16:11:19 +00:00
james.maesandClaude Opus 5.5 8f3f9ec60a Merge develop into release/0.1 for 0.1.0-RC.2
qbit-object-storage 0.1.0-RC.2 (AWS SDK v2 2.41.34, GHSA-443w-3rq3-5m5h; #17).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 11:08:28 -05:00
CircleCI cdaf218201 Bump version to 0.1.0-RC.1 [skip ci] 2026-10-13 00:26:41 +00:00
11 changed files with 913 additions and 280 deletions

No files matched your search

+4
View File
@@ -5,6 +5,7 @@ All notable changes to this project are documented here. The format follows Keep
## [0.1.0] - Unreleased
### Added
- Central `backups.sla` and per-active-plan `backups.destination.<planId>` health checks, with bounded read-only destination diagnostics and secret-free status details.
- Always-durable backup, resume, scheduled backup, restore and verify processes with conservative `NONE` idempotency and stable backup identities on explicit job retry.
- Weighted backup phases, table/file counts, stored bytes, safe per-table/artifact logs, warnings, failures and resume notices through QQQ `JobProgress`.
- Cooperative job cancellation during backup pages, media copying, restore pages and verification scans; local media-reference spooling keeps phase separation from growing heap usage across tables.
@@ -26,3 +27,6 @@ All notable changes to this project are documented here. The format follows Keep
- New manifests identify `qqq-table-schema-v1` and carry core schema descriptors; restore and content verification still accept pre-RC.6 manifests with the original fingerprint algorithm.
- Preserve primary-key ordering, exclusive resume cursors, bounded pages and caller-owned transactions; reject non-orderable primary keys and scope filters with skip/limit.
- Require the core `backups.restore` permission during Review when using a core restore writer, including anonymous-mode hosts; preserve supplied timestamps through `RestoreWriteFlag`.
### Removed
- The legacy `BackupsHealthIndicator` and its optional `qqq-middleware-health` dependency; central health registration replaces manual indicator configuration.
+25
View File
@@ -43,6 +43,11 @@ Follow the QQQ [adoption checklist](https://github.com/QRun-IO/qqq/blob/develop/
See [the RC.6 migration notes](docs/RC6-MIGRATION.md) for fingerprint compatibility,
paging guarantees and restore permissions.
Health checks use the core health API released in QQQ `4.1.0-RC.8`.
The producer registers `backups.sla` and a read-only destination check for each
active plan (`backups.destination.<planId>`). With dependencies cached locally,
run `./mvnw -o verify`; no enforcer skip flags are required.
## In-app help
The producer registers [help/qbit-backups.yaml](src/main/resources/help/qbit-backups.yaml)
@@ -95,3 +100,23 @@ The backup SLA checker still owns expected intervals and grace periods.
RC.5 requires hyphenated alert names: `backups.run.missing-in-storage` and
`backups.plan.owner-invalid` replace the former underscore spellings, including
their deduplication keys. Update host routing rules that used those spellings.
## Health checks
The producer registers these central checks under component `backups`, category `STORAGE`:
| Name | Kind | Interval | Timeout |
| --- | --- | --- | --- |
| `backups.sla` | DEPENDENCY | 900s (15 minutes) | 5000ms |
| `backups.destination.<planId>` | DEPENDENCY | 86400s (24 hours) | 30000ms |
The SLA check reads persisted `slaStatus`: critical active plans that are LATE,
FAILING or MISSING_IN_STORAGE yield DOWN; non-critical failures yield DEGRADED.
Only UNVERIFIED/UNKNOWN plans yield UNKNOWN; no active plans yield UP. Details
contain counts per SLA status. The scheduled `backupCheckSla` still computes SLA state.
Destination checks are supplied for active plans and reuse `DestinationProbe` in
read-only list/head mode, including on unlocked buckets. They never write, delete,
or resolve prune credentials. The Test destination action retains its write tests.
Storage calls are bounded by the central runner, with deadline checks between calls.
Override intervals in `health_check` and disable checks through `health_check.enabled`.
+1 -8
View File
@@ -20,7 +20,7 @@
<properties>
<qbit.guardrails.enforce>true</qbit.guardrails.enforce>
<qbit.guardrails.strictUiApiRoutes>true</qbit.guardrails.strictUiApiRoutes>
<revision>0.1.0-SNAPSHOT</revision>
<revision>0.1.0-RC.8</revision>
<maven.compiler.source>21</maven.compiler.source>
<maven.compiler.target>21</maven.compiler.target>
<jacoco.version>0.8.15</jacoco.version>
@@ -74,13 +74,6 @@
<artifactId>qqq-backend-core</artifactId>
</dependency>
<!-- optional: the BackupsHealthIndicator (apps without qqq-middleware-health never load it) -->
<dependency>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-middleware-health</artifactId>
<optional>true</optional>
</dependency>
<!-- optional: ESB events (apps without qqq-esb never load it) -->
<dependency>
<groupId>com.kingsrook.qqq</groupId>
@@ -17,6 +17,14 @@
package com.kingsrook.qbits.backups;
import com.kingsrook.qbits.backups.health.BackupsDestinationHealthChecks;
import com.kingsrook.qbits.backups.health.BackupsSlaHealthCheck;
import com.kingsrook.qqq.backend.core.health.HealthCategory;
import com.kingsrook.qqq.backend.core.health.HealthCheckKind;
import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData;
import com.kingsrook.qqq.backend.core.health.HealthCheckSupplierMetaData;
import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundles;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaDataProducer;
@@ -164,6 +172,13 @@ public class BackupsQBitProducer implements QBitMetaDataProducer<BackupsQBitConf
// Registration is metadata-only; core guards startup sync when helpContent is absent.
HelpBundles.register(qInstance, "help/qbit-backups.yaml");
HealthChecksMetaData.ofOrWithNew(qInstance)
.withCheck(new HealthCheckMetaData().withName(BackupsSlaHealthCheck.NAME).withLabel("Backup SLA")
.withComponent("backups").withKind(HealthCheckKind.DEPENDENCY).withCategory(HealthCategory.STORAGE)
.withIntervalSeconds(900).withCheck(new QCodeReference(BackupsSlaHealthCheck.class)))
.withSupplier(new HealthCheckSupplierMetaData().withNamePrefix(BackupsDestinationHealthChecks.PREFIX)
.withComponent("backups").withSupplier(new QCodeReference(BackupsDestinationHealthChecks.class)));
// A parent app only lists the apps that are its children, so attach ourselves (for example Admin, then Backups).
com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData parent = qBitConfig.getParentAppName() == null ? null : qInstance.getApp(qBitConfig.getParentAppName());
if(parent != null && (parent.getChildren() == null || parent.getChildren().stream().noneMatch(child -> app.getName().equals(child.getName()))))
@@ -0,0 +1,95 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.health;
import com.kingsrook.qbits.backups.model.BackupsTables;
import com.kingsrook.qbits.backups.plan.DestinationProbe;
import com.kingsrook.qbits.backups.run.BackupCatalog;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.health.HealthCheck;
import com.kingsrook.qqq.backend.core.health.HealthCheckInput;
import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome;
import com.kingsrook.qqq.backend.core.health.HealthDetails;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
/*******************************************************************************
** Reuse the destination diagnostic in read-only mode; never publish its text.
*******************************************************************************/
public class BackupsDestinationHealthCheck implements HealthCheck
{
private final Integer planId;
/*******************************************************************************
** Retain only the id; destination and active state are read on each run.
*******************************************************************************/
public BackupsDestinationHealthCheck(Integer planId)
{
this.planId = planId;
}
/*******************************************************************************
** A removed or disabled plan can remain registered until supplier refresh.
*******************************************************************************/
@Override
public HealthCheckOutcome run(HealthCheckInput input)
{
boolean ownsContext = QContext.getQInstance() == null;
if(ownsContext)
{
QContext.init(input.getQInstance(), new QSystemUserSession(input.getQInstance()));
}
try
{
if(BackupsSlaHealthCheck.expired(input))
{
return (HealthCheckOutcome.down("Backup destination check timed out"));
}
QRecord plan = new BackupCatalog().get(BackupsTables.PLAN, planId).orElse(null);
if(plan == null || !Boolean.TRUE.equals(plan.getValueBoolean("isActive")))
{
return (HealthCheckOutcome.unknown("Backup plan is no longer active"));
}
DestinationProbe.Report report = DestinationProbe.probeReadOnly(plan, input.getDeadline());
if(BackupsSlaHealthCheck.expired(input))
{
return (HealthCheckOutcome.down("Backup destination check timed out"));
}
return ((report.ok() ? HealthCheckOutcome.up("Backup destination is readable") : HealthCheckOutcome.down("Backup destination probe failed"))
.withDetails(new HealthDetails().withValue("planId", planId).withValue("readOnly", true)
.withValue("checks", report.checks().size()).withValue("failedChecks", report.checks().stream().filter(check -> !check.ok()).count())));
}
catch(QException | RuntimeException e)
{
return (HealthCheckOutcome.down("Backup destination could not be checked"));
}
finally
{
if(ownsContext)
{
QContext.clear();
}
}
}
}
@@ -0,0 +1,77 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.health;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import com.kingsrook.qbits.backups.model.BackupsTables;
import com.kingsrook.qbits.backups.run.BackupCatalog;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.health.HealthCategory;
import com.kingsrook.qqq.backend.core.health.HealthCheckKind;
import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData;
import com.kingsrook.qqq.backend.core.health.HealthCheckSupplier;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
/*******************************************************************************
** Discover one daily, read-only destination check per active backup plan.
*******************************************************************************/
public class BackupsDestinationHealthChecks implements HealthCheckSupplier
{
public static final String PREFIX = "backups.destination";
/*******************************************************************************
** Resolve from current catalog data, without opening any storage connections.
*******************************************************************************/
@Override
public List<HealthCheckMetaData> supply(QInstance qInstance) throws QException
{
boolean ownsContext = QContext.getQInstance() == null;
if(ownsContext)
{
QContext.init(qInstance, new QSystemUserSession(qInstance));
}
try
{
List<HealthCheckMetaData> checks = new ArrayList<>();
for(QRecord plan : new BackupCatalog().where(BackupsTables.PLAN, "isActive", true))
{
Integer planId = plan.getValueInteger("id");
checks.add(new HealthCheckMetaData().withName(PREFIX + "." + planId.toString().toLowerCase(Locale.ROOT))
.withLabel("Backup destination " + planId).withComponent("backups")
.withKind(HealthCheckKind.DEPENDENCY).withCategory(HealthCategory.STORAGE)
.withIntervalSeconds(86400).withTimeoutMs(30000).withCheckInstance(new BackupsDestinationHealthCheck(planId)));
}
return (checks);
}
finally
{
if(ownsContext)
{
QContext.clear();
}
}
}
}
@@ -1,133 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.health;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.kingsrook.qbits.backups.model.BackupsTables;
import com.kingsrook.qbits.backups.run.BackupCatalog;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
import com.kingsrook.qqq.middleware.health.HealthIndicator;
import com.kingsrook.qqq.middleware.health.model.metadata.HealthCheckResult;
import com.kingsrook.qqq.middleware.health.model.metadata.HealthStatus;
/*******************************************************************************
** The Backups health indicator (design spec section 10.4), for
** qqq-middleware-health (an optional dependency: an app that does not have that
** module never loads this class). It reports DOWN if any active plan marked
** critical is LATE, FAILING or MISSING_IN_STORAGE, and UP with details otherwise.
** It reads the slaStatus the SLA checker maintains; it never touches storage, so
** a health call is cheap.
**
** It is OFF the liveness path by design. Add it to the indicators of the
** instance's health check (the readiness and monitoring view), and do not point a
** Kubernetes liveness probe at a check that includes it: a backup problem must
** never restart pods.
*******************************************************************************/
public class BackupsHealthIndicator implements HealthIndicator
{
public static final String NAME = "backups";
private static final Set<String> CRITICAL_STATUSES = Set.of("LATE", "FAILING", "MISSING_IN_STORAGE");
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public String getName()
{
return (NAME);
}
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public HealthCheckResult check(QInstance qInstance) throws QException
{
long started = System.currentTimeMillis();
boolean ownsContext = QContext.getQInstance() == null;
if(ownsContext)
{
QContext.init(qInstance, systemSession(qInstance));
}
try
{
List<QRecord> plans = new BackupCatalog().query(BackupsTables.PLAN, new QQueryFilter(new QFilterCriteria("isActive", QCriteriaOperator.EQUALS, true)));
List<String> criticalProblems = new ArrayList<>();
Map<String, Integer> byStatus = new LinkedHashMap<>();
for(QRecord plan : plans)
{
String status = plan.getValueString("slaStatus") == null ? "UNKNOWN" : plan.getValueString("slaStatus");
byStatus.merge(status, 1, Integer::sum);
if(Boolean.TRUE.equals(plan.getValueBoolean("critical")) && CRITICAL_STATUSES.contains(status))
{
criticalProblems.add(plan.getValueString("planKey") + ": " + status);
}
}
Map<String, Object> details = new LinkedHashMap<>();
details.put("activePlans", plans.size());
details.put("plansByStatus", byStatus);
if(!criticalProblems.isEmpty())
{
details.put("criticalPlansWithProblems", criticalProblems);
}
return (new HealthCheckResult()
.withStatus(criticalProblems.isEmpty() ? HealthStatus.UP : HealthStatus.DOWN)
.withDurationMs(System.currentTimeMillis() - started)
.withDetails(details));
}
finally
{
if(ownsContext)
{
QContext.clear();
}
}
}
/*******************************************************************************
** The session for a health request that has no QQQ context. It must be the
** core system user: token-based authentication modules (OAuth2, Auth0)
** reject any other tokenless session, which failed the check on such hosts.
*******************************************************************************/
static QSession systemSession(QInstance qInstance)
{
return (new QSystemUserSession(qInstance));
}
}
@@ -0,0 +1,132 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.health;
import java.time.Instant;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.kingsrook.qbits.backups.model.BackupsTables;
import com.kingsrook.qbits.backups.run.BackupCatalog;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.health.HealthCheck;
import com.kingsrook.qqq.backend.core.health.HealthCheckInput;
import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome;
import com.kingsrook.qqq.backend.core.health.HealthDetails;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
/*******************************************************************************
** Read persisted SLA state using the former indicator's critical-plan rule.
** SLA calculation and storage verification remain the scheduled checker's job.
*******************************************************************************/
public class BackupsSlaHealthCheck implements HealthCheck
{
public static final String NAME = "backups.sla";
private static final Set<String> BAD_STATUSES = Set.of("LATE", "FAILING", "MISSING_IN_STORAGE");
private static final List<String> STATUSES = List.of("OK", "LATE", "FAILING", "MISSING_IN_STORAGE", "UNVERIFIED", "ABANDONED", "OWNER_INVALID", "UNKNOWN");
/*******************************************************************************
** Counts use a fixed vocabulary so stored strings cannot become detail keys.
*******************************************************************************/
@Override
public HealthCheckOutcome run(HealthCheckInput input)
{
boolean ownsContext = QContext.getQInstance() == null;
if(ownsContext)
{
QContext.init(input.getQInstance(), new QSystemUserSession(input.getQInstance()));
}
try
{
if(expired(input))
{
return (HealthCheckOutcome.down("Backup SLA check timed out"));
}
List<QRecord> plans = new BackupCatalog().where(BackupsTables.PLAN, "isActive", true);
Map<String, Integer> counts = new LinkedHashMap<>();
STATUSES.forEach(status -> counts.put(status, 0));
boolean criticalBad = false;
boolean ordinaryBad = false;
for(QRecord plan : plans)
{
String status = plan.getValueString("slaStatus");
status = status != null && STATUSES.contains(status) ? status : "UNKNOWN";
counts.merge(status, 1, Integer::sum);
if(BAD_STATUSES.contains(status))
{
if(Boolean.TRUE.equals(plan.getValueBoolean("critical")))
{
criticalBad = true;
}
else
{
ordinaryBad = true;
}
}
}
if(expired(input))
{
return (HealthCheckOutcome.down("Backup SLA check timed out"));
}
HealthDetails details = new HealthDetails().withValue("activePlans", plans.size());
counts.forEach((status, count) -> details.withValue("slaStatus." + status, count));
HealthCheckOutcome outcome = HealthCheckOutcome.up("Backup SLA has no failures");
if(criticalBad)
{
outcome = HealthCheckOutcome.down("Critical backup plans have SLA failures");
}
else if(ordinaryBad)
{
outcome = HealthCheckOutcome.degraded("Backup plans have SLA failures");
}
else if(!plans.isEmpty() && counts.get("UNVERIFIED") + counts.get("UNKNOWN") == plans.size())
{
outcome = HealthCheckOutcome.unknown("Backup SLA is not verified yet");
}
return (outcome.withDetails(details));
}
catch(QException | RuntimeException e)
{
return (HealthCheckOutcome.down("Backup SLA state could not be read"));
}
finally
{
if(ownsContext)
{
QContext.clear();
}
}
}
/*******************************************************************************
** Stop between catalog operations when the runner's budget is exhausted.
*******************************************************************************/
static boolean expired(HealthCheckInput input)
{
return (Thread.currentThread().isInterrupted() || !Instant.now().isBefore(input.getDeadline()));
}
}
@@ -20,6 +20,7 @@ package com.kingsrook.qbits.backups.plan;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
@@ -81,6 +82,28 @@ public final class DestinationProbe
** Probe the destination of a plan.
*******************************************************************************/
public static Report probe(QRecord plan)
{
return (probe(plan, false, null));
}
/*******************************************************************************
** Health monitoring only lists and heads; it never resolves prune credentials
** or probes capabilities. The runner bounds in-flight storage calls; check the
** deadline between calls because BackupStorage has no per-call timeout API.
*******************************************************************************/
public static Report probeReadOnly(QRecord plan, Instant deadline)
{
return (probe(plan, true, deadline));
}
/*******************************************************************************
** Share storage resolution and the existing locked-bucket read path.
*******************************************************************************/
private static Report probe(QRecord plan, boolean readOnly, Instant deadline)
{
List<Check> checks = new ArrayList<>();
List<String> warnings = new ArrayList<>();
@@ -89,15 +112,23 @@ public final class DestinationProbe
BackupStorage storage;
try
{
requireTime(deadline);
storage = BackupsRuntime.storageFor(alias);
requireTime(deadline);
}
catch(RuntimeException e)
{
checks.add(new Check("Open the backup bucket " + alias, false, e.getMessage()));
return (new Report(false, checks, List.of(), warnings));
return (new Report(readOnly, checks, List.of(), warnings));
}
checks.add(new Check("Open the backup bucket " + alias, true, "found"));
if(readOnly)
{
headOnly(plan, storage, BackupPruner.planPrefix(plan), checks, deadline);
return (new Report(true, checks, List.of(), warnings));
}
BackupStorage deleteSide = storage;
String pruneAlias = plan.getValueString("pruneBucketAlias");
if(pruneAlias == null || pruneAlias.isBlank())
@@ -133,7 +164,7 @@ public final class DestinationProbe
String prefix = BackupPruner.planPrefix(plan);
if(locked)
{
headOnly(plan, storage, prefix, checks);
headOnly(plan, storage, prefix, checks, null);
}
else
{
@@ -147,10 +178,11 @@ public final class DestinationProbe
/*******************************************************************************
** The probe for a locked bucket: list the prefix, and head the newest manifest.
*******************************************************************************/
private static void headOnly(QRecord plan, BackupStorage storage, String prefix, List<Check> checks)
private static void headOnly(QRecord plan, BackupStorage storage, String prefix, List<Check> checks, Instant deadline)
{
checks.add(attempt("Write and delete a probe object", () -> "skipped: object lock is on, a probe object could not be deleted again"));
checks.add(attempt("List the plan prefix", () ->
checks.add(new Check("Write and delete a probe object", true,
deadline == null ? "skipped: object lock is on, a probe object could not be deleted again" : "skipped: read-only probe"));
checks.add(attempt("List the plan prefix", deadline, () ->
{
List<StoredObjectInfo> objects = storage.list(prefix);
return (objects.size() + " objects under " + prefix);
@@ -162,10 +194,11 @@ public final class DestinationProbe
checks.add(new Check("Head the newest manifest", true, "there is no successful run yet"));
return;
}
checks.add(attempt("Head the newest manifest", () ->
checks.add(attempt("Head the newest manifest", deadline, () ->
{
QRecord run = new com.kingsrook.qbits.backups.run.BackupCatalog().get(com.kingsrook.qbits.backups.model.BackupsTables.RUN, lastSuccess).orElseThrow();
String key = run.getValueString("manifestKey");
requireTime(deadline);
if(key == null || storage.head(key).isEmpty())
{
throw (new IllegalStateException("the manifest of the newest successful run is not in the bucket"));
@@ -238,6 +271,9 @@ public final class DestinationProbe
@FunctionalInterface
private interface Attempt
{
/*******************************************************************************
** Execute one diagnostic operation.
*******************************************************************************/
String run() throws Exception;
}
@@ -247,14 +283,40 @@ public final class DestinationProbe
**
*******************************************************************************/
private static Check attempt(String name, Attempt attempt)
{
return (attempt(name, null, attempt));
}
/*******************************************************************************
** Never begin another storage operation after the health budget expires.
*******************************************************************************/
private static Check attempt(String name, Instant deadline, Attempt attempt)
{
try
{
return (new Check(name, true, attempt.run()));
requireTime(deadline);
String detail = attempt.run();
requireTime(deadline);
return (new Check(name, true, detail));
}
catch(Exception e)
{
return (new Check(name, false, e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage()));
}
}
/*******************************************************************************
** Existing on-demand diagnostics have no deadline; health probes always do.
*******************************************************************************/
private static void requireTime(Instant deadline)
{
if(deadline != null && (Thread.currentThread().isInterrupted() || !Instant.now().isBefore(deadline)))
{
throw (new IllegalStateException("Destination probe timed out"));
}
}
}
@@ -0,0 +1,495 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.health;
import java.io.InputStream;
import java.io.OutputStream;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import com.kingsrook.qbits.backups.BackupsRuntime;
import com.kingsrook.qbits.backups.BackupsTestUtils;
import com.kingsrook.qbits.backups.VerifyTestSupport;
import com.kingsrook.qbits.backups.model.BackupsTables;
import com.kingsrook.qbits.backups.plan.DestinationProbe;
import com.kingsrook.qbits.backups.run.BackupCatalog;
import com.kingsrook.qbits.backups.storage.InMemoryBackupStorage;
import com.kingsrook.qbits.backups.storage.StoredObjectInfo;
import com.kingsrook.qbits.backups.storage.WriteOptions;
import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.health.HealthCategory;
import com.kingsrook.qqq.backend.core.health.HealthCheck;
import com.kingsrook.qqq.backend.core.health.HealthCheckInput;
import com.kingsrook.qqq.backend.core.health.HealthCheckKind;
import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData;
import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome;
import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData;
import com.kingsrook.qqq.backend.core.health.HealthStatus;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** Central registration, migrated SLA rules, read-only probes and secret safety.
*******************************************************************************/
class BackupsHealthChecksTest
{
private static final String SECRET = "S3NT1NEL-secret";
private final BackupCatalog catalog = new BackupCatalog();
private ReadOnlyStorage storage;
/*******************************************************************************
** All probes use memory storage; the resolver represents credential injection.
*******************************************************************************/
@BeforeEach
void setUp() throws Exception
{
VerifyTestSupport.setUp(3);
storage = new ReadOnlyStorage(SECRET);
BackupsRuntime.getConfig().withStorageResolver(alias ->
{
assertThat(alias).as("never resolve prune credentials").isNotEqualTo("prune");
return (storage);
});
}
/*******************************************************************************
** Use the actual producer registry, including its reflective supplier loading.
*******************************************************************************/
@Test
void testRegistrationAndDynamicPlans() throws Exception
{
HealthChecksMetaData registry = HealthChecksMetaData.ofOrWithNew(QContext.getQInstance());
assertThat(registry.getChecks()).extracting(HealthCheckMetaData::getName).containsExactly("backups.sla");
assertThat(registry.getSuppliers()).singleElement().satisfies(supplier ->
{
assertThat(supplier.getNamePrefix()).isEqualTo("backups.destination");
assertThat(supplier.getComponent()).isEqualTo("backups");
});
QRecord active = VerifyTestSupport.plan(Map.of());
VerifyTestSupport.plan(Map.of("name", "Off", "planKey", "off", "isActive", false));
List<HealthCheckMetaData> checks = registry.resolveAll(QContext.getQInstance());
assertThat(checks).hasSize(2).allSatisfy(check ->
{
assertThat(check.getComponent()).isEqualTo("backups");
assertThat(check.getKind()).isEqualTo(HealthCheckKind.DEPENDENCY);
assertThat(check.getCategory()).isEqualTo(HealthCategory.STORAGE);
assertThat(check.getIncludeInReadiness()).isFalse();
assertThat(check.getName()).matches("[a-z0-9]+([._-][a-z0-9]+)*");
});
assertThat(checks.get(0).getIntervalSeconds()).isEqualTo(900);
assertThat(checks.get(0).getTimeoutMs()).isEqualTo(5000);
assertThat(QCodeLoader.getAdHoc(HealthCheck.class, checks.get(0).getCheck()).run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
HealthCheckMetaData destination = checks.get(1);
assertThat(destination.getName()).isEqualTo("backups.destination." + active.getValueInteger("id"));
assertThat(destination.getIntervalSeconds()).isEqualTo(86400);
assertThat(destination.getTimeoutMs()).isEqualTo(30000);
assertThat(destination.getCheckInstance().run(input()).getStatus()).isEqualTo(HealthStatus.UP);
catalog.update(BackupsTables.PLAN, active.getValueInteger("id"), Map.of("isActive", false));
assertThat(destination.getCheckInstance().run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
assertThat(registry.resolveAll(QContext.getQInstance())).hasSize(1);
VerifyTestSupport.plan(Map.of("name", "New", "planKey", "new"));
assertThat(registry.resolveAll(QContext.getQInstance())).hasSize(2);
}
/*******************************************************************************
** Carry over the indicator's empty, critical, ordinary and inactive cases.
*******************************************************************************/
@Test
void testSlaUpDownAndDegraded() throws Exception
{
BackupsSlaHealthCheck check = new BackupsSlaHealthCheck();
HealthCheckOutcome empty = check.run(input());
assertThat(empty.getStatus()).isEqualTo(HealthStatus.UP);
assertThat(empty.getDetails().getValues()).containsEntry("activePlans", 0);
QRecord critical = VerifyTestSupport.plan(Map.of("critical", true));
QRecord ordinary = VerifyTestSupport.plan(Map.of("name", "Other", "planKey", "other"));
QRecord inactive = VerifyTestSupport.plan(Map.of("name", "Off", "planKey", "off", "isActive", false, "critical", true));
setStatus(inactive, "LATE");
setStatus(critical, "OK");
setStatus(ordinary, "OK");
assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UP);
for(String status : List.of("LATE", "FAILING", "MISSING_IN_STORAGE"))
{
setStatus(ordinary, status);
assertThat(check.run(input()).getStatus()).as(status).isEqualTo(HealthStatus.DEGRADED);
setStatus(critical, status);
HealthCheckOutcome down = check.run(input());
assertThat(down.getStatus()).as(status).isEqualTo(HealthStatus.DOWN);
assertThat(down.getDetails().getValues()).containsEntry("activePlans", 2).containsEntry("slaStatus." + status, 2);
assertClean(down);
setStatus(critical, "OK");
}
}
/*******************************************************************************
** Unknown or unverified plans do not become a false UP or a critical DOWN.
*******************************************************************************/
@Test
void testSlaUnknownAndMixedStates() throws Exception
{
QRecord plan = VerifyTestSupport.plan(Map.of("critical", true));
BackupsSlaHealthCheck check = new BackupsSlaHealthCheck();
assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
for(String status : List.of("UNVERIFIED", "UNKNOWN"))
{
setStatus(plan, status);
assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
}
QRecord other = VerifyTestSupport.plan(Map.of("name", "Other", "planKey", "other"));
setStatus(other, "UNVERIFIED");
assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
setStatus(other, "OK");
assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UP);
}
/*******************************************************************************
** Preserve the old indicator's no-context behavior and existing sessions.
*******************************************************************************/
@Test
void testWithoutContext() throws Exception
{
QRecord plan = VerifyTestSupport.plan(Map.of());
QInstance instance = QContext.getQInstance();
QSession session = QContext.getQSession();
new BackupsSlaHealthCheck().run(input());
assertThat(QContext.getQSession()).isSameAs(session);
QContext.clear();
try
{
HealthCheckInput input = new HealthCheckInput().withQInstance(instance).withDeadline(Instant.now().plusSeconds(5));
assertThat(new BackupsSlaHealthCheck().run(input).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
assertThat(QContext.getQInstance()).isNull();
assertThat(new BackupsDestinationHealthChecks().supply(instance)).hasSize(1);
assertThat(QContext.getQInstance()).isNull();
BackupsRuntime.getConfig().withStorageResolver(alias ->
{
assertThat(QContext.getQSession()).isInstanceOf(com.kingsrook.qqq.backend.core.model.session.QSystemUserSession.class);
return (storage);
});
assertThat(destination(plan).run(input).getStatus()).isEqualTo(HealthStatus.UP);
assertThat(QContext.getQInstance()).isNull();
}
finally
{
QContext.init(instance, session);
}
}
/*******************************************************************************
** Locked and unlocked plans only list/head; neither reads object contents.
*******************************************************************************/
@Test
void testDestinationReadOnlyAndMissingManifest() throws Exception
{
for(String lockMode : List.of("NONE", "GOVERNANCE"))
{
BackupsRuntime.getConfig().withStorageResolver(alias -> BackupsTestUtils.BACKUPS);
QRecord plan = VerifyTestSupport.plan(Map.of("name", lockMode, "planKey", lockMode.toLowerCase(), "objectLockMode", lockMode, "objectLockDays", 3, "pruneBucketAlias", "prune"));
BackupsRuntime.getConfig().withStorageResolver(alias ->
{
assertThat(alias).as("never resolve prune credentials").isNotEqualTo("prune");
return (storage);
});
Integer runId = catalog.insert(BackupsTables.RUN, Map.of("manifestKey", "manifest.json"));
catalog.update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("lastSuccessRunId", runId));
storage.putBytes("manifest.json", new byte[] { 1 });
assertThat(destination(plan).run(input()).getStatus()).isEqualTo(HealthStatus.UP);
storage.clear();
assertThat(destination(plan).run(input()).getStatus()).isEqualTo(HealthStatus.DOWN);
}
assertThat(storage.lists).isEqualTo(4);
assertThat(storage.heads).isEqualTo(4);
}
/*******************************************************************************
** A deleted plan is UNKNOWN and does not touch storage.
*******************************************************************************/
@Test
void testDeletedPlan() throws Exception
{
QRecord plan = VerifyTestSupport.plan(Map.of());
catalog.deleteByIds(BackupsTables.PLAN, List.of(plan.getValueInteger("id")));
assertThat(destination(plan).run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
assertThat(storage.lists).isZero();
}
/*******************************************************************************
** No work starts with an expired deadline, including direct probe calls.
*******************************************************************************/
@Test
void testExpiredDeadline() throws Exception
{
QRecord plan = VerifyTestSupport.plan(Map.of());
HealthCheckInput expired = input().withDeadline(Instant.now().minusSeconds(1));
assertThat(new BackupsSlaHealthCheck().run(expired).getStatus()).isEqualTo(HealthStatus.DOWN);
assertThat(destination(plan).run(expired).getStatus()).isEqualTo(HealthStatus.DOWN);
assertThat(DestinationProbe.probeReadOnly(plan, expired.getDeadline()).ok()).isFalse();
assertThat(storage.lists).isZero();
assertThat(storage.heads).isZero();
}
/*******************************************************************************
** Runner cancellation after list prevents the following manifest head.
*******************************************************************************/
@Test
void testCancellationBetweenStorageCalls() throws Exception
{
QRecord plan = VerifyTestSupport.plan(Map.of());
Integer runId = catalog.insert(BackupsTables.RUN, Map.of("manifestKey", "manifest.json"));
catalog.update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("lastSuccessRunId", runId));
storage.interruptAfterList = true;
try
{
assertThat(destination(plan).run(input()).getStatus()).isEqualTo(HealthStatus.DOWN);
assertThat(storage.lists).isEqualTo(1);
assertThat(storage.heads).isZero();
}
finally
{
Thread.interrupted();
}
}
/*******************************************************************************
** Poison configuration, persisted state and provider errors. sanitize is
** package-private in core; assert raw outcomes already contain no secrets.
*******************************************************************************/
@Test
void testNoSecretsInSuccessAndFailureOutcomes() throws Exception
{
QRecord plan = VerifyTestSupport.plan(Map.of("name", SECRET, "storageBucketAlias", SECRET, "critical", true));
Integer runId = catalog.insert(BackupsTables.RUN, Map.of("manifestKey", SECRET));
catalog.update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("lastSuccessRunId", runId));
storage.putBytes(SECRET, new byte[] { 1 });
for(String status : List.of("OK", "LATE", "UNVERIFIED", SECRET))
{
setStatus(plan, status);
assertClean(new BackupsSlaHealthCheck().run(input()));
assertClean(destination(plan).run(input()));
}
storage.failList = true;
HealthCheckOutcome failure = destination(plan).run(input());
assertThat(failure.getStatus()).isEqualTo(HealthStatus.DOWN);
assertClean(failure);
storage.failList = false;
storage.failHead = true;
failure = destination(plan).run(input());
assertThat(failure.getStatus()).isEqualTo(HealthStatus.DOWN);
assertClean(failure);
BackupsRuntime.getConfig().withStorageResolver(alias ->
{
throw (new IllegalStateException(SECRET));
});
failure = destination(plan).run(input());
assertThat(failure.getStatus()).isEqualTo(HealthStatus.DOWN);
assertClean(failure);
assertClean(new BackupsSlaHealthCheck().run(input()));
}
/*******************************************************************************
** Catalog failures are DOWN with fixed messages, even if metadata is poisoned.
*******************************************************************************/
@Test
void testCatalogFailureDoesNotExposeSecrets()
{
QContext.getQInstance().getTable(BackupsTables.PLAN).setBackendName(SECRET);
HealthCheckOutcome sla = new BackupsSlaHealthCheck().run(input());
HealthCheckOutcome destination = new BackupsDestinationHealthCheck(1).run(input());
assertThat(sla.getStatus()).isEqualTo(HealthStatus.DOWN);
assertThat(destination.getStatus()).isEqualTo(HealthStatus.DOWN);
assertClean(sla);
assertClean(destination);
}
/*******************************************************************************
** Update persisted SLA state without running the scheduled calculator.
*******************************************************************************/
private void setStatus(QRecord plan, String status) throws Exception
{
catalog.update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("slaStatus", status));
}
/*******************************************************************************
** Fresh bounded input for each call.
*******************************************************************************/
private HealthCheckInput input()
{
return (new HealthCheckInput().withQInstance(QContext.getQInstance()).withDeadline(Instant.now().plusSeconds(5)));
}
/*******************************************************************************
** Construct the same per-plan check as the supplier.
*******************************************************************************/
private BackupsDestinationHealthCheck destination(QRecord plan)
{
return (new BackupsDestinationHealthCheck(plan.getValueInteger("id")));
}
/*******************************************************************************
** Validate the raw public message and flattened detail representation.
*******************************************************************************/
private void assertClean(HealthCheckOutcome outcome)
{
assertThat(outcome.getMessage()).doesNotContain(SECRET, "\n", "\r");
assertThat(outcome.getDetails().toJson()).doesNotContain(SECRET);
}
/*******************************************************************************
** Existing memory fake with read failures and assertions against side effects.
*******************************************************************************/
private static class ReadOnlyStorage extends InMemoryBackupStorage
{
private final String credential;
private boolean failList;
private boolean failHead;
private boolean interruptAfterList;
private int lists;
private int heads;
/*******************************************************************************
** Mimic the credential held by a host-provided storage adapter.
*******************************************************************************/
ReadOnlyStorage(String credential)
{
this.credential = credential;
}
/*******************************************************************************
** Record list calls and inject credential-bearing failures or cancellation.
*******************************************************************************/
@Override
public List<StoredObjectInfo> list(String prefix)
{
lists++;
if(failList)
{
throw (new IllegalStateException(credential));
}
if(interruptAfterList)
{
Thread.currentThread().interrupt();
}
return (super.list(prefix));
}
/*******************************************************************************
** Record head calls and inject credential-bearing failures.
*******************************************************************************/
@Override
public Optional<StoredObjectInfo> head(String key)
{
heads++;
if(failHead)
{
throw (new IllegalStateException(credential));
}
return (super.head(key));
}
/*******************************************************************************
** Health probes must not write, even to an unlocked bucket.
*******************************************************************************/
@Override
public OutputStream openWrite(String key, WriteOptions options)
{
throw (new AssertionError("Health probe wrote an object"));
}
/*******************************************************************************
** Health probes must not fetch backup contents.
*******************************************************************************/
@Override
public InputStream openRead(String key)
{
throw (new AssertionError("Health probe read object contents"));
}
/*******************************************************************************
** Health probes must not delete.
*******************************************************************************/
@Override
public DeleteResult deletePermanent(String key)
{
throw (new AssertionError("Health probe deleted an object"));
}
/*******************************************************************************
** Capability diagnostics can be active probes; health must not invoke them.
*******************************************************************************/
@Override
public Set<StorageCapability> capabilities()
{
throw (new AssertionError("Health probe requested capabilities"));
}
}
}
@@ -1,132 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.health;
import java.util.Map;
import com.kingsrook.qbits.backups.VerifyTestSupport;
import com.kingsrook.qbits.backups.model.BackupsTables;
import com.kingsrook.qbits.backups.run.BackupCatalog;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.middleware.health.model.metadata.HealthCheckResult;
import com.kingsrook.qqq.middleware.health.model.metadata.HealthStatus;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** The health indicator: DOWN only when a critical, active plan is LATE, FAILING or
** MISSING_IN_STORAGE.
*******************************************************************************/
class BackupsHealthIndicatorTest
{
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
void setUp() throws Exception
{
VerifyTestSupport.setUp(3);
}
/*******************************************************************************
**
*******************************************************************************/
private void setStatus(QRecord plan, String status) throws Exception
{
new BackupCatalog().update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("slaStatus", status));
}
/*******************************************************************************
** UP with details when nothing critical is wrong, including a non-critical plan
** that is LATE; DOWN when a critical plan is LATE, FAILING or MISSING_IN_STORAGE.
*******************************************************************************/
@Test
void testUpAndDown() throws Exception
{
BackupsHealthIndicator indicator = new BackupsHealthIndicator();
assertThat(indicator.getName()).isEqualTo("backups");
HealthCheckResult empty = indicator.check(QContext.getQInstance());
assertThat(empty.getStatus()).isEqualTo(HealthStatus.UP);
assertThat(empty.getDetails()).containsEntry("activePlans", 0);
QRecord critical = VerifyTestSupport.plan(Map.of("critical", true));
QRecord ordinary = VerifyTestSupport.plan(Map.of("name", "Other", "planKey", "other"));
VerifyTestSupport.plan(Map.of("name", "Off", "planKey", "off", "isActive", false, "critical", true));
setStatus(critical, "OK");
setStatus(ordinary, "LATE");
HealthCheckResult up = indicator.check(QContext.getQInstance());
assertThat(up.getStatus()).isEqualTo(HealthStatus.UP);
assertThat(up.getDetails()).containsEntry("activePlans", 2).doesNotContainKey("criticalPlansWithProblems");
assertThat(up.getDurationMs()).isNotNull();
for(String status : new String[] { "LATE", "FAILING", "MISSING_IN_STORAGE" })
{
setStatus(critical, status);
HealthCheckResult down = indicator.check(QContext.getQInstance());
assertThat(down.getStatus()).as(status).isEqualTo(HealthStatus.DOWN);
assertThat(down.getDetails().get("criticalPlansWithProblems").toString()).contains("nightly: " + status);
}
setStatus(critical, "UNVERIFIED");
assertThat(indicator.check(QContext.getQInstance()).getStatus()).as("not verified is not down").isEqualTo(HealthStatus.UP);
}
/*******************************************************************************
** On a thread with no QQQ context (a health request) the indicator makes its own
** and cleans up.
*******************************************************************************/
@Test
void testWorksWithoutAContext() throws Exception
{
QInstance qInstance = QContext.getQInstance();
QContext.clear();
try
{
assertThat(new BackupsHealthIndicator().check(qInstance).getStatus()).isEqualTo(HealthStatus.UP);
assertThat(QContext.getQInstance()).isNull();
}
finally
{
QContext.init(qInstance, new com.kingsrook.qqq.backend.core.model.session.QSession());
}
}
/*******************************************************************************
** The indicator's own context runs as the core system user, which token-based
** authentication modules accept without a token.
*******************************************************************************/
@Test
void testOwnContextUsesSystemUserSession()
{
assertThat(BackupsHealthIndicator.systemSession(QContext.getQInstance()))
.isInstanceOf(com.kingsrook.qqq.backend.core.model.session.QSystemUserSession.class);
}
}