From 0f9178dc14a4f250c958b2ee9da8acf5ce274980 Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 14 Oct 2026 14:55:02 -0500 Subject: [PATCH 1/3] feat(health): read-only destination probe mode Co-Authored-By: Claude Opus 5.5 --- .../qbits/backups/plan/DestinationProbe.java | 76 +++++++++++++++++-- 1 file changed, 69 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/kingsrook/qbits/backups/plan/DestinationProbe.java b/src/main/java/com/kingsrook/qbits/backups/plan/DestinationProbe.java index f0e8b21..50ed079 100644 --- a/src/main/java/com/kingsrook/qbits/backups/plan/DestinationProbe.java +++ b/src/main/java/com/kingsrook/qbits/backups/plan/DestinationProbe.java @@ -20,6 +20,7 @@ package com.kingsrook.qbits.backups.plan; import java.io.InputStream; import java.io.OutputStream; import java.nio.charset.StandardCharsets; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.UUID; @@ -81,6 +82,28 @@ public final class DestinationProbe ** Probe the destination of a plan. *******************************************************************************/ public static Report probe(QRecord plan) + { + return (probe(plan, false, null)); + } + + + + /******************************************************************************* + ** Health monitoring only lists and heads; it never resolves prune credentials + ** or probes capabilities. The runner bounds in-flight storage calls; check the + ** deadline between calls because BackupStorage has no per-call timeout API. + *******************************************************************************/ + public static Report probeReadOnly(QRecord plan, Instant deadline) + { + return (probe(plan, true, deadline)); + } + + + + /******************************************************************************* + ** Share storage resolution and the existing locked-bucket read path. + *******************************************************************************/ + private static Report probe(QRecord plan, boolean readOnly, Instant deadline) { List checks = new ArrayList<>(); List warnings = new ArrayList<>(); @@ -89,15 +112,23 @@ public final class DestinationProbe BackupStorage storage; try { + requireTime(deadline); storage = BackupsRuntime.storageFor(alias); + requireTime(deadline); } catch(RuntimeException e) { checks.add(new Check("Open the backup bucket " + alias, false, e.getMessage())); - return (new Report(false, checks, List.of(), warnings)); + return (new Report(readOnly, checks, List.of(), warnings)); } checks.add(new Check("Open the backup bucket " + alias, true, "found")); + if(readOnly) + { + headOnly(plan, storage, BackupPruner.planPrefix(plan), checks, deadline); + return (new Report(true, checks, List.of(), warnings)); + } + BackupStorage deleteSide = storage; String pruneAlias = plan.getValueString("pruneBucketAlias"); if(pruneAlias == null || pruneAlias.isBlank()) @@ -133,7 +164,7 @@ public final class DestinationProbe String prefix = BackupPruner.planPrefix(plan); if(locked) { - headOnly(plan, storage, prefix, checks); + headOnly(plan, storage, prefix, checks, null); } else { @@ -147,10 +178,11 @@ public final class DestinationProbe /******************************************************************************* ** The probe for a locked bucket: list the prefix, and head the newest manifest. *******************************************************************************/ - private static void headOnly(QRecord plan, BackupStorage storage, String prefix, List checks) + private static void headOnly(QRecord plan, BackupStorage storage, String prefix, List checks, Instant deadline) { - checks.add(attempt("Write and delete a probe object", () -> "skipped: object lock is on, a probe object could not be deleted again")); - checks.add(attempt("List the plan prefix", () -> + checks.add(new Check("Write and delete a probe object", true, + deadline == null ? "skipped: object lock is on, a probe object could not be deleted again" : "skipped: read-only probe")); + checks.add(attempt("List the plan prefix", deadline, () -> { List objects = storage.list(prefix); return (objects.size() + " objects under " + prefix); @@ -162,10 +194,11 @@ public final class DestinationProbe checks.add(new Check("Head the newest manifest", true, "there is no successful run yet")); return; } - checks.add(attempt("Head the newest manifest", () -> + checks.add(attempt("Head the newest manifest", deadline, () -> { QRecord run = new com.kingsrook.qbits.backups.run.BackupCatalog().get(com.kingsrook.qbits.backups.model.BackupsTables.RUN, lastSuccess).orElseThrow(); String key = run.getValueString("manifestKey"); + requireTime(deadline); if(key == null || storage.head(key).isEmpty()) { throw (new IllegalStateException("the manifest of the newest successful run is not in the bucket")); @@ -238,6 +271,9 @@ public final class DestinationProbe @FunctionalInterface private interface Attempt { + /******************************************************************************* + ** Execute one diagnostic operation. + *******************************************************************************/ String run() throws Exception; } @@ -247,14 +283,40 @@ public final class DestinationProbe ** *******************************************************************************/ private static Check attempt(String name, Attempt attempt) + { + return (attempt(name, null, attempt)); + } + + + + /******************************************************************************* + ** Never begin another storage operation after the health budget expires. + *******************************************************************************/ + private static Check attempt(String name, Instant deadline, Attempt attempt) { try { - return (new Check(name, true, attempt.run())); + requireTime(deadline); + String detail = attempt.run(); + requireTime(deadline); + return (new Check(name, true, detail)); } catch(Exception e) { return (new Check(name, false, e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage())); } } + + + + /******************************************************************************* + ** Existing on-demand diagnostics have no deadline; health probes always do. + *******************************************************************************/ + private static void requireTime(Instant deadline) + { + if(deadline != null && (Thread.currentThread().isInterrupted() || !Instant.now().isBefore(deadline))) + { + throw (new IllegalStateException("Destination probe timed out")); + } + } } -- 2.54.0 From 8d56613f7d6e6f42535dea9aa5a9c687ad0ff66a Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 14 Oct 2026 14:55:03 -0500 Subject: [PATCH 2/3] feat(health): register backups SLA and destination checks; drop BackupsHealthIndicator Co-Authored-By: Claude Opus 5.5 --- .../qbits/backups/BackupsQBitProducer.java | 15 + .../health/BackupsDestinationHealthCheck.java | 95 ++++ .../BackupsDestinationHealthChecks.java | 77 +++ .../health/BackupsHealthIndicator.java | 120 ----- .../backups/health/BackupsSlaHealthCheck.java | 132 +++++ .../health/BackupsHealthChecksTest.java | 490 ++++++++++++++++++ .../health/BackupsHealthIndicatorTest.java | 119 ----- 7 files changed, 809 insertions(+), 239 deletions(-) create mode 100644 src/main/java/com/kingsrook/qbits/backups/health/BackupsDestinationHealthCheck.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/health/BackupsDestinationHealthChecks.java delete mode 100644 src/main/java/com/kingsrook/qbits/backups/health/BackupsHealthIndicator.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/health/BackupsSlaHealthCheck.java create mode 100644 src/test/java/com/kingsrook/qbits/backups/health/BackupsHealthChecksTest.java delete mode 100644 src/test/java/com/kingsrook/qbits/backups/health/BackupsHealthIndicatorTest.java diff --git a/src/main/java/com/kingsrook/qbits/backups/BackupsQBitProducer.java b/src/main/java/com/kingsrook/qbits/backups/BackupsQBitProducer.java index 66e703a..3272bf7 100644 --- a/src/main/java/com/kingsrook/qbits/backups/BackupsQBitProducer.java +++ b/src/main/java/com/kingsrook/qbits/backups/BackupsQBitProducer.java @@ -17,6 +17,14 @@ package com.kingsrook.qbits.backups; +import com.kingsrook.qbits.backups.health.BackupsDestinationHealthChecks; +import com.kingsrook.qbits.backups.health.BackupsSlaHealthCheck; +import com.kingsrook.qqq.backend.core.health.HealthCategory; +import com.kingsrook.qqq.backend.core.health.HealthCheckKind; +import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData; +import com.kingsrook.qqq.backend.core.health.HealthCheckSupplierMetaData; +import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData; import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaDataProducer; @@ -161,6 +169,13 @@ public class BackupsQBitProducer implements QBitMetaDataProducer app.getName().equals(child.getName())))) diff --git a/src/main/java/com/kingsrook/qbits/backups/health/BackupsDestinationHealthCheck.java b/src/main/java/com/kingsrook/qbits/backups/health/BackupsDestinationHealthCheck.java new file mode 100644 index 0000000..a87f9b8 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/health/BackupsDestinationHealthCheck.java @@ -0,0 +1,95 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.health; + + +import com.kingsrook.qbits.backups.model.BackupsTables; +import com.kingsrook.qbits.backups.plan.DestinationProbe; +import com.kingsrook.qbits.backups.run.BackupCatalog; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.health.HealthCheck; +import com.kingsrook.qqq.backend.core.health.HealthCheckInput; +import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome; +import com.kingsrook.qqq.backend.core.health.HealthDetails; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.session.QSession; + + +/******************************************************************************* + ** Reuse the destination diagnostic in read-only mode; never publish its text. + *******************************************************************************/ +public class BackupsDestinationHealthCheck implements HealthCheck +{ + private final Integer planId; + + + + /******************************************************************************* + ** Retain only the id; destination and active state are read on each run. + *******************************************************************************/ + public BackupsDestinationHealthCheck(Integer planId) + { + this.planId = planId; + } + + + + /******************************************************************************* + ** A removed or disabled plan can remain registered until supplier refresh. + *******************************************************************************/ + @Override + public HealthCheckOutcome run(HealthCheckInput input) + { + boolean ownsContext = QContext.getQInstance() == null; + if(ownsContext) + { + QContext.init(input.getQInstance(), new QSession()); + } + try + { + if(BackupsSlaHealthCheck.expired(input)) + { + return (HealthCheckOutcome.down("Backup destination check timed out")); + } + QRecord plan = new BackupCatalog().get(BackupsTables.PLAN, planId).orElse(null); + if(plan == null || !Boolean.TRUE.equals(plan.getValueBoolean("isActive"))) + { + return (HealthCheckOutcome.unknown("Backup plan is no longer active")); + } + DestinationProbe.Report report = DestinationProbe.probeReadOnly(plan, input.getDeadline()); + if(BackupsSlaHealthCheck.expired(input)) + { + return (HealthCheckOutcome.down("Backup destination check timed out")); + } + return ((report.ok() ? HealthCheckOutcome.up("Backup destination is readable") : HealthCheckOutcome.down("Backup destination probe failed")) + .withDetails(new HealthDetails().withValue("planId", planId).withValue("readOnly", true) + .withValue("checks", report.checks().size()).withValue("failedChecks", report.checks().stream().filter(check -> !check.ok()).count()))); + } + catch(QException | RuntimeException e) + { + return (HealthCheckOutcome.down("Backup destination could not be checked")); + } + finally + { + if(ownsContext) + { + QContext.clear(); + } + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/health/BackupsDestinationHealthChecks.java b/src/main/java/com/kingsrook/qbits/backups/health/BackupsDestinationHealthChecks.java new file mode 100644 index 0000000..6834c07 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/health/BackupsDestinationHealthChecks.java @@ -0,0 +1,77 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.health; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import com.kingsrook.qbits.backups.model.BackupsTables; +import com.kingsrook.qbits.backups.run.BackupCatalog; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.health.HealthCategory; +import com.kingsrook.qqq.backend.core.health.HealthCheckKind; +import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData; +import com.kingsrook.qqq.backend.core.health.HealthCheckSupplier; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.session.QSession; + + +/******************************************************************************* + ** Discover one daily, read-only destination check per active backup plan. + *******************************************************************************/ +public class BackupsDestinationHealthChecks implements HealthCheckSupplier +{ + public static final String PREFIX = "backups.destination"; + + + + /******************************************************************************* + ** Resolve from current catalog data, without opening any storage connections. + *******************************************************************************/ + @Override + public List supply(QInstance qInstance) throws QException + { + boolean ownsContext = QContext.getQInstance() == null; + if(ownsContext) + { + QContext.init(qInstance, new QSession()); + } + try + { + List checks = new ArrayList<>(); + for(QRecord plan : new BackupCatalog().where(BackupsTables.PLAN, "isActive", true)) + { + Integer planId = plan.getValueInteger("id"); + checks.add(new HealthCheckMetaData().withName(PREFIX + "." + planId.toString().toLowerCase(Locale.ROOT)) + .withLabel("Backup destination " + planId).withComponent("backups") + .withKind(HealthCheckKind.DEPENDENCY).withCategory(HealthCategory.STORAGE) + .withIntervalSeconds(86400).withTimeoutMs(30000).withCheckInstance(new BackupsDestinationHealthCheck(planId))); + } + return (checks); + } + finally + { + if(ownsContext) + { + QContext.clear(); + } + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/health/BackupsHealthIndicator.java b/src/main/java/com/kingsrook/qbits/backups/health/BackupsHealthIndicator.java deleted file mode 100644 index 26511f1..0000000 --- a/src/main/java/com/kingsrook/qbits/backups/health/BackupsHealthIndicator.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright 2026 Kingsrook, LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.kingsrook.qbits.backups.health; - - -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import com.kingsrook.qbits.backups.model.BackupsTables; -import com.kingsrook.qbits.backups.run.BackupCatalog; -import com.kingsrook.qqq.backend.core.context.QContext; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.session.QSession; -import com.kingsrook.qqq.middleware.health.HealthIndicator; -import com.kingsrook.qqq.middleware.health.model.metadata.HealthCheckResult; -import com.kingsrook.qqq.middleware.health.model.metadata.HealthStatus; - - -/******************************************************************************* - ** The Backups health indicator (design spec section 10.4), for - ** qqq-middleware-health (an optional dependency: an app that does not have that - ** module never loads this class). It reports DOWN if any active plan marked - ** critical is LATE, FAILING or MISSING_IN_STORAGE, and UP with details otherwise. - ** It reads the slaStatus the SLA checker maintains; it never touches storage, so - ** a health call is cheap. - ** - ** It is OFF the liveness path by design. Add it to the indicators of the - ** instance's health check (the readiness and monitoring view), and do not point a - ** Kubernetes liveness probe at a check that includes it: a backup problem must - ** never restart pods. - *******************************************************************************/ -public class BackupsHealthIndicator implements HealthIndicator -{ - public static final String NAME = "backups"; - - private static final Set CRITICAL_STATUSES = Set.of("LATE", "FAILING", "MISSING_IN_STORAGE"); - - - - /******************************************************************************* - ** {@inheritDoc} - *******************************************************************************/ - @Override - public String getName() - { - return (NAME); - } - - - - /******************************************************************************* - ** {@inheritDoc} - *******************************************************************************/ - @Override - public HealthCheckResult check(QInstance qInstance) throws QException - { - long started = System.currentTimeMillis(); - boolean ownsContext = QContext.getQInstance() == null; - if(ownsContext) - { - QContext.init(qInstance, new QSession()); - } - try - { - List plans = new BackupCatalog().query(BackupsTables.PLAN, new QQueryFilter(new QFilterCriteria("isActive", QCriteriaOperator.EQUALS, true))); - - List criticalProblems = new ArrayList<>(); - Map byStatus = new LinkedHashMap<>(); - for(QRecord plan : plans) - { - String status = plan.getValueString("slaStatus") == null ? "UNKNOWN" : plan.getValueString("slaStatus"); - byStatus.merge(status, 1, Integer::sum); - if(Boolean.TRUE.equals(plan.getValueBoolean("critical")) && CRITICAL_STATUSES.contains(status)) - { - criticalProblems.add(plan.getValueString("planKey") + ": " + status); - } - } - - Map details = new LinkedHashMap<>(); - details.put("activePlans", plans.size()); - details.put("plansByStatus", byStatus); - if(!criticalProblems.isEmpty()) - { - details.put("criticalPlansWithProblems", criticalProblems); - } - return (new HealthCheckResult() - .withStatus(criticalProblems.isEmpty() ? HealthStatus.UP : HealthStatus.DOWN) - .withDurationMs(System.currentTimeMillis() - started) - .withDetails(details)); - } - finally - { - if(ownsContext) - { - QContext.clear(); - } - } - } -} diff --git a/src/main/java/com/kingsrook/qbits/backups/health/BackupsSlaHealthCheck.java b/src/main/java/com/kingsrook/qbits/backups/health/BackupsSlaHealthCheck.java new file mode 100644 index 0000000..24b84a5 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/health/BackupsSlaHealthCheck.java @@ -0,0 +1,132 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.health; + + +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qbits.backups.model.BackupsTables; +import com.kingsrook.qbits.backups.run.BackupCatalog; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.health.HealthCheck; +import com.kingsrook.qqq.backend.core.health.HealthCheckInput; +import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome; +import com.kingsrook.qqq.backend.core.health.HealthDetails; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.session.QSession; + + +/******************************************************************************* + ** Read persisted SLA state using the former indicator's critical-plan rule. + ** SLA calculation and storage verification remain the scheduled checker's job. + *******************************************************************************/ +public class BackupsSlaHealthCheck implements HealthCheck +{ + public static final String NAME = "backups.sla"; + + private static final Set BAD_STATUSES = Set.of("LATE", "FAILING", "MISSING_IN_STORAGE"); + private static final List STATUSES = List.of("OK", "LATE", "FAILING", "MISSING_IN_STORAGE", "UNVERIFIED", "ABANDONED", "OWNER_INVALID", "UNKNOWN"); + + + + /******************************************************************************* + ** Counts use a fixed vocabulary so stored strings cannot become detail keys. + *******************************************************************************/ + @Override + public HealthCheckOutcome run(HealthCheckInput input) + { + boolean ownsContext = QContext.getQInstance() == null; + if(ownsContext) + { + QContext.init(input.getQInstance(), new QSession()); + } + try + { + if(expired(input)) + { + return (HealthCheckOutcome.down("Backup SLA check timed out")); + } + List plans = new BackupCatalog().where(BackupsTables.PLAN, "isActive", true); + Map counts = new LinkedHashMap<>(); + STATUSES.forEach(status -> counts.put(status, 0)); + boolean criticalBad = false; + boolean ordinaryBad = false; + for(QRecord plan : plans) + { + String status = plan.getValueString("slaStatus"); + status = status != null && STATUSES.contains(status) ? status : "UNKNOWN"; + counts.merge(status, 1, Integer::sum); + if(BAD_STATUSES.contains(status)) + { + if(Boolean.TRUE.equals(plan.getValueBoolean("critical"))) + { + criticalBad = true; + } + else + { + ordinaryBad = true; + } + } + } + if(expired(input)) + { + return (HealthCheckOutcome.down("Backup SLA check timed out")); + } + HealthDetails details = new HealthDetails().withValue("activePlans", plans.size()); + counts.forEach((status, count) -> details.withValue("slaStatus." + status, count)); + HealthCheckOutcome outcome = HealthCheckOutcome.up("Backup SLA has no failures"); + if(criticalBad) + { + outcome = HealthCheckOutcome.down("Critical backup plans have SLA failures"); + } + else if(ordinaryBad) + { + outcome = HealthCheckOutcome.degraded("Backup plans have SLA failures"); + } + else if(!plans.isEmpty() && counts.get("UNVERIFIED") + counts.get("UNKNOWN") == plans.size()) + { + outcome = HealthCheckOutcome.unknown("Backup SLA is not verified yet"); + } + return (outcome.withDetails(details)); + } + catch(QException | RuntimeException e) + { + return (HealthCheckOutcome.down("Backup SLA state could not be read")); + } + finally + { + if(ownsContext) + { + QContext.clear(); + } + } + } + + + + /******************************************************************************* + ** Stop between catalog operations when the runner's budget is exhausted. + *******************************************************************************/ + static boolean expired(HealthCheckInput input) + { + return (Thread.currentThread().isInterrupted() || !Instant.now().isBefore(input.getDeadline())); + } +} diff --git a/src/test/java/com/kingsrook/qbits/backups/health/BackupsHealthChecksTest.java b/src/test/java/com/kingsrook/qbits/backups/health/BackupsHealthChecksTest.java new file mode 100644 index 0000000..2b537d0 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/backups/health/BackupsHealthChecksTest.java @@ -0,0 +1,490 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.health; + + +import java.io.InputStream; +import java.io.OutputStream; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import com.kingsrook.qbits.backups.BackupsRuntime; +import com.kingsrook.qbits.backups.BackupsTestUtils; +import com.kingsrook.qbits.backups.VerifyTestSupport; +import com.kingsrook.qbits.backups.model.BackupsTables; +import com.kingsrook.qbits.backups.plan.DestinationProbe; +import com.kingsrook.qbits.backups.run.BackupCatalog; +import com.kingsrook.qbits.backups.storage.InMemoryBackupStorage; +import com.kingsrook.qbits.backups.storage.StoredObjectInfo; +import com.kingsrook.qbits.backups.storage.WriteOptions; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.health.HealthCategory; +import com.kingsrook.qqq.backend.core.health.HealthCheck; +import com.kingsrook.qqq.backend.core.health.HealthCheckInput; +import com.kingsrook.qqq.backend.core.health.HealthCheckKind; +import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData; +import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome; +import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData; +import com.kingsrook.qqq.backend.core.health.HealthStatus; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Central registration, migrated SLA rules, read-only probes and secret safety. + *******************************************************************************/ +class BackupsHealthChecksTest +{ + private static final String SECRET = "S3NT1NEL-secret"; + + private final BackupCatalog catalog = new BackupCatalog(); + private ReadOnlyStorage storage; + + + + /******************************************************************************* + ** All probes use memory storage; the resolver represents credential injection. + *******************************************************************************/ + @BeforeEach + void setUp() throws Exception + { + VerifyTestSupport.setUp(3); + storage = new ReadOnlyStorage(SECRET); + BackupsRuntime.getConfig().withStorageResolver(alias -> + { + assertThat(alias).as("never resolve prune credentials").isNotEqualTo("prune"); + return (storage); + }); + } + + + + /******************************************************************************* + ** Use the actual producer registry, including its reflective supplier loading. + *******************************************************************************/ + @Test + void testRegistrationAndDynamicPlans() throws Exception + { + HealthChecksMetaData registry = HealthChecksMetaData.ofOrWithNew(QContext.getQInstance()); + assertThat(registry.getChecks()).extracting(HealthCheckMetaData::getName).containsExactly("backups.sla"); + assertThat(registry.getSuppliers()).singleElement().satisfies(supplier -> + { + assertThat(supplier.getNamePrefix()).isEqualTo("backups.destination"); + assertThat(supplier.getComponent()).isEqualTo("backups"); + }); + QRecord active = VerifyTestSupport.plan(Map.of()); + VerifyTestSupport.plan(Map.of("name", "Off", "planKey", "off", "isActive", false)); + List checks = registry.resolveAll(QContext.getQInstance()); + assertThat(checks).hasSize(2).allSatisfy(check -> + { + assertThat(check.getComponent()).isEqualTo("backups"); + assertThat(check.getKind()).isEqualTo(HealthCheckKind.DEPENDENCY); + assertThat(check.getCategory()).isEqualTo(HealthCategory.STORAGE); + assertThat(check.getIncludeInReadiness()).isFalse(); + assertThat(check.getName()).matches("[a-z0-9]+([._-][a-z0-9]+)*"); + }); + assertThat(checks.get(0).getIntervalSeconds()).isEqualTo(900); + assertThat(checks.get(0).getTimeoutMs()).isEqualTo(5000); + assertThat(QCodeLoader.getAdHoc(HealthCheck.class, checks.get(0).getCheck()).run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN); + HealthCheckMetaData destination = checks.get(1); + assertThat(destination.getName()).isEqualTo("backups.destination." + active.getValueInteger("id")); + assertThat(destination.getIntervalSeconds()).isEqualTo(86400); + assertThat(destination.getTimeoutMs()).isEqualTo(30000); + assertThat(destination.getCheckInstance().run(input()).getStatus()).isEqualTo(HealthStatus.UP); + catalog.update(BackupsTables.PLAN, active.getValueInteger("id"), Map.of("isActive", false)); + assertThat(destination.getCheckInstance().run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN); + assertThat(registry.resolveAll(QContext.getQInstance())).hasSize(1); + VerifyTestSupport.plan(Map.of("name", "New", "planKey", "new")); + assertThat(registry.resolveAll(QContext.getQInstance())).hasSize(2); + } + + + + /******************************************************************************* + ** Carry over the indicator's empty, critical, ordinary and inactive cases. + *******************************************************************************/ + @Test + void testSlaUpDownAndDegraded() throws Exception + { + BackupsSlaHealthCheck check = new BackupsSlaHealthCheck(); + HealthCheckOutcome empty = check.run(input()); + assertThat(empty.getStatus()).isEqualTo(HealthStatus.UP); + assertThat(empty.getDetails().getValues()).containsEntry("activePlans", 0); + QRecord critical = VerifyTestSupport.plan(Map.of("critical", true)); + QRecord ordinary = VerifyTestSupport.plan(Map.of("name", "Other", "planKey", "other")); + QRecord inactive = VerifyTestSupport.plan(Map.of("name", "Off", "planKey", "off", "isActive", false, "critical", true)); + setStatus(inactive, "LATE"); + setStatus(critical, "OK"); + setStatus(ordinary, "OK"); + assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UP); + for(String status : List.of("LATE", "FAILING", "MISSING_IN_STORAGE")) + { + setStatus(ordinary, status); + assertThat(check.run(input()).getStatus()).as(status).isEqualTo(HealthStatus.DEGRADED); + setStatus(critical, status); + HealthCheckOutcome down = check.run(input()); + assertThat(down.getStatus()).as(status).isEqualTo(HealthStatus.DOWN); + assertThat(down.getDetails().getValues()).containsEntry("activePlans", 2).containsEntry("slaStatus." + status, 2); + assertClean(down); + setStatus(critical, "OK"); + } + } + + + + /******************************************************************************* + ** Unknown or unverified plans do not become a false UP or a critical DOWN. + *******************************************************************************/ + @Test + void testSlaUnknownAndMixedStates() throws Exception + { + QRecord plan = VerifyTestSupport.plan(Map.of("critical", true)); + BackupsSlaHealthCheck check = new BackupsSlaHealthCheck(); + assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN); + for(String status : List.of("UNVERIFIED", "UNKNOWN")) + { + setStatus(plan, status); + assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN); + } + QRecord other = VerifyTestSupport.plan(Map.of("name", "Other", "planKey", "other")); + setStatus(other, "UNVERIFIED"); + assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN); + setStatus(other, "OK"); + assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UP); + } + + + + /******************************************************************************* + ** Preserve the old indicator's no-context behavior and existing sessions. + *******************************************************************************/ + @Test + void testWithoutContext() throws Exception + { + QRecord plan = VerifyTestSupport.plan(Map.of()); + QInstance instance = QContext.getQInstance(); + QSession session = QContext.getQSession(); + new BackupsSlaHealthCheck().run(input()); + assertThat(QContext.getQSession()).isSameAs(session); + QContext.clear(); + try + { + HealthCheckInput input = new HealthCheckInput().withQInstance(instance).withDeadline(Instant.now().plusSeconds(5)); + assertThat(new BackupsSlaHealthCheck().run(input).getStatus()).isEqualTo(HealthStatus.UNKNOWN); + assertThat(QContext.getQInstance()).isNull(); + assertThat(new BackupsDestinationHealthChecks().supply(instance)).hasSize(1); + assertThat(QContext.getQInstance()).isNull(); + assertThat(destination(plan).run(input).getStatus()).isEqualTo(HealthStatus.UP); + assertThat(QContext.getQInstance()).isNull(); + } + finally + { + QContext.init(instance, session); + } + } + + + + /******************************************************************************* + ** Locked and unlocked plans only list/head; neither reads object contents. + *******************************************************************************/ + @Test + void testDestinationReadOnlyAndMissingManifest() throws Exception + { + for(String lockMode : List.of("NONE", "GOVERNANCE")) + { + BackupsRuntime.getConfig().withStorageResolver(alias -> BackupsTestUtils.BACKUPS); + QRecord plan = VerifyTestSupport.plan(Map.of("name", lockMode, "planKey", lockMode.toLowerCase(), "objectLockMode", lockMode, "objectLockDays", 3, "pruneBucketAlias", "prune")); + BackupsRuntime.getConfig().withStorageResolver(alias -> + { + assertThat(alias).as("never resolve prune credentials").isNotEqualTo("prune"); + return (storage); + }); + Integer runId = catalog.insert(BackupsTables.RUN, Map.of("manifestKey", "manifest.json")); + catalog.update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("lastSuccessRunId", runId)); + storage.putBytes("manifest.json", new byte[] { 1 }); + assertThat(destination(plan).run(input()).getStatus()).isEqualTo(HealthStatus.UP); + storage.clear(); + assertThat(destination(plan).run(input()).getStatus()).isEqualTo(HealthStatus.DOWN); + } + assertThat(storage.lists).isEqualTo(4); + assertThat(storage.heads).isEqualTo(4); + } + + + + /******************************************************************************* + ** A deleted plan is UNKNOWN and does not touch storage. + *******************************************************************************/ + @Test + void testDeletedPlan() throws Exception + { + QRecord plan = VerifyTestSupport.plan(Map.of()); + catalog.deleteByIds(BackupsTables.PLAN, List.of(plan.getValueInteger("id"))); + assertThat(destination(plan).run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN); + assertThat(storage.lists).isZero(); + } + + + + /******************************************************************************* + ** No work starts with an expired deadline, including direct probe calls. + *******************************************************************************/ + @Test + void testExpiredDeadline() throws Exception + { + QRecord plan = VerifyTestSupport.plan(Map.of()); + HealthCheckInput expired = input().withDeadline(Instant.now().minusSeconds(1)); + assertThat(new BackupsSlaHealthCheck().run(expired).getStatus()).isEqualTo(HealthStatus.DOWN); + assertThat(destination(plan).run(expired).getStatus()).isEqualTo(HealthStatus.DOWN); + assertThat(DestinationProbe.probeReadOnly(plan, expired.getDeadline()).ok()).isFalse(); + assertThat(storage.lists).isZero(); + assertThat(storage.heads).isZero(); + } + + + + /******************************************************************************* + ** Runner cancellation after list prevents the following manifest head. + *******************************************************************************/ + @Test + void testCancellationBetweenStorageCalls() throws Exception + { + QRecord plan = VerifyTestSupport.plan(Map.of()); + Integer runId = catalog.insert(BackupsTables.RUN, Map.of("manifestKey", "manifest.json")); + catalog.update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("lastSuccessRunId", runId)); + storage.interruptAfterList = true; + try + { + assertThat(destination(plan).run(input()).getStatus()).isEqualTo(HealthStatus.DOWN); + assertThat(storage.lists).isEqualTo(1); + assertThat(storage.heads).isZero(); + } + finally + { + Thread.interrupted(); + } + } + + + + /******************************************************************************* + ** Poison configuration, persisted state and provider errors. sanitize is + ** package-private in core; assert raw outcomes already contain no secrets. + *******************************************************************************/ + @Test + void testNoSecretsInSuccessAndFailureOutcomes() throws Exception + { + QRecord plan = VerifyTestSupport.plan(Map.of("name", SECRET, "storageBucketAlias", SECRET, "critical", true)); + Integer runId = catalog.insert(BackupsTables.RUN, Map.of("manifestKey", SECRET)); + catalog.update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("lastSuccessRunId", runId)); + storage.putBytes(SECRET, new byte[] { 1 }); + for(String status : List.of("OK", "LATE", "UNVERIFIED", SECRET)) + { + setStatus(plan, status); + assertClean(new BackupsSlaHealthCheck().run(input())); + assertClean(destination(plan).run(input())); + } + storage.failList = true; + HealthCheckOutcome failure = destination(plan).run(input()); + assertThat(failure.getStatus()).isEqualTo(HealthStatus.DOWN); + assertClean(failure); + storage.failList = false; + storage.failHead = true; + failure = destination(plan).run(input()); + assertThat(failure.getStatus()).isEqualTo(HealthStatus.DOWN); + assertClean(failure); + BackupsRuntime.getConfig().withStorageResolver(alias -> + { + throw (new IllegalStateException(SECRET)); + }); + failure = destination(plan).run(input()); + assertThat(failure.getStatus()).isEqualTo(HealthStatus.DOWN); + assertClean(failure); + assertClean(new BackupsSlaHealthCheck().run(input())); + } + + + + /******************************************************************************* + ** Catalog failures are DOWN with fixed messages, even if metadata is poisoned. + *******************************************************************************/ + @Test + void testCatalogFailureDoesNotExposeSecrets() + { + QContext.getQInstance().getTable(BackupsTables.PLAN).setBackendName(SECRET); + HealthCheckOutcome sla = new BackupsSlaHealthCheck().run(input()); + HealthCheckOutcome destination = new BackupsDestinationHealthCheck(1).run(input()); + assertThat(sla.getStatus()).isEqualTo(HealthStatus.DOWN); + assertThat(destination.getStatus()).isEqualTo(HealthStatus.DOWN); + assertClean(sla); + assertClean(destination); + } + + + + /******************************************************************************* + ** Update persisted SLA state without running the scheduled calculator. + *******************************************************************************/ + private void setStatus(QRecord plan, String status) throws Exception + { + catalog.update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("slaStatus", status)); + } + + + + /******************************************************************************* + ** Fresh bounded input for each call. + *******************************************************************************/ + private HealthCheckInput input() + { + return (new HealthCheckInput().withQInstance(QContext.getQInstance()).withDeadline(Instant.now().plusSeconds(5))); + } + + + + /******************************************************************************* + ** Construct the same per-plan check as the supplier. + *******************************************************************************/ + private BackupsDestinationHealthCheck destination(QRecord plan) + { + return (new BackupsDestinationHealthCheck(plan.getValueInteger("id"))); + } + + + + /******************************************************************************* + ** Validate the raw public message and flattened detail representation. + *******************************************************************************/ + private void assertClean(HealthCheckOutcome outcome) + { + assertThat(outcome.getMessage()).doesNotContain(SECRET, "\n", "\r"); + assertThat(outcome.getDetails().toJson()).doesNotContain(SECRET); + } + + + + /******************************************************************************* + ** Existing memory fake with read failures and assertions against side effects. + *******************************************************************************/ + private static class ReadOnlyStorage extends InMemoryBackupStorage + { + private final String credential; + private boolean failList; + private boolean failHead; + private boolean interruptAfterList; + private int lists; + private int heads; + + + + /******************************************************************************* + ** Mimic the credential held by a host-provided storage adapter. + *******************************************************************************/ + ReadOnlyStorage(String credential) + { + this.credential = credential; + } + + + + /******************************************************************************* + ** Record list calls and inject credential-bearing failures or cancellation. + *******************************************************************************/ + @Override + public List list(String prefix) + { + lists++; + if(failList) + { + throw (new IllegalStateException(credential)); + } + if(interruptAfterList) + { + Thread.currentThread().interrupt(); + } + return (super.list(prefix)); + } + + + + /******************************************************************************* + ** Record head calls and inject credential-bearing failures. + *******************************************************************************/ + @Override + public Optional head(String key) + { + heads++; + if(failHead) + { + throw (new IllegalStateException(credential)); + } + return (super.head(key)); + } + + + + /******************************************************************************* + ** Health probes must not write, even to an unlocked bucket. + *******************************************************************************/ + @Override + public OutputStream openWrite(String key, WriteOptions options) + { + throw (new AssertionError("Health probe wrote an object")); + } + + + + /******************************************************************************* + ** Health probes must not fetch backup contents. + *******************************************************************************/ + @Override + public InputStream openRead(String key) + { + throw (new AssertionError("Health probe read object contents")); + } + + + + /******************************************************************************* + ** Health probes must not delete. + *******************************************************************************/ + @Override + public DeleteResult deletePermanent(String key) + { + throw (new AssertionError("Health probe deleted an object")); + } + + + + /******************************************************************************* + ** Capability diagnostics can be active probes; health must not invoke them. + *******************************************************************************/ + @Override + public Set capabilities() + { + throw (new AssertionError("Health probe requested capabilities")); + } + } +} diff --git a/src/test/java/com/kingsrook/qbits/backups/health/BackupsHealthIndicatorTest.java b/src/test/java/com/kingsrook/qbits/backups/health/BackupsHealthIndicatorTest.java deleted file mode 100644 index 9ba10c6..0000000 --- a/src/test/java/com/kingsrook/qbits/backups/health/BackupsHealthIndicatorTest.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright 2026 Kingsrook, LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.kingsrook.qbits.backups.health; - - -import java.util.Map; -import com.kingsrook.qbits.backups.VerifyTestSupport; -import com.kingsrook.qbits.backups.model.BackupsTables; -import com.kingsrook.qbits.backups.run.BackupCatalog; -import com.kingsrook.qqq.backend.core.context.QContext; -import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.middleware.health.model.metadata.HealthCheckResult; -import com.kingsrook.qqq.middleware.health.model.metadata.HealthStatus; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import static org.assertj.core.api.Assertions.assertThat; - - -/******************************************************************************* - ** The health indicator: DOWN only when a critical, active plan is LATE, FAILING or - ** MISSING_IN_STORAGE. - *******************************************************************************/ -class BackupsHealthIndicatorTest -{ - /******************************************************************************* - ** - *******************************************************************************/ - @BeforeEach - void setUp() throws Exception - { - VerifyTestSupport.setUp(3); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private void setStatus(QRecord plan, String status) throws Exception - { - new BackupCatalog().update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("slaStatus", status)); - } - - - - /******************************************************************************* - ** UP with details when nothing critical is wrong, including a non-critical plan - ** that is LATE; DOWN when a critical plan is LATE, FAILING or MISSING_IN_STORAGE. - *******************************************************************************/ - @Test - void testUpAndDown() throws Exception - { - BackupsHealthIndicator indicator = new BackupsHealthIndicator(); - assertThat(indicator.getName()).isEqualTo("backups"); - - HealthCheckResult empty = indicator.check(QContext.getQInstance()); - assertThat(empty.getStatus()).isEqualTo(HealthStatus.UP); - assertThat(empty.getDetails()).containsEntry("activePlans", 0); - - QRecord critical = VerifyTestSupport.plan(Map.of("critical", true)); - QRecord ordinary = VerifyTestSupport.plan(Map.of("name", "Other", "planKey", "other")); - VerifyTestSupport.plan(Map.of("name", "Off", "planKey", "off", "isActive", false, "critical", true)); - - setStatus(critical, "OK"); - setStatus(ordinary, "LATE"); - HealthCheckResult up = indicator.check(QContext.getQInstance()); - assertThat(up.getStatus()).isEqualTo(HealthStatus.UP); - assertThat(up.getDetails()).containsEntry("activePlans", 2).doesNotContainKey("criticalPlansWithProblems"); - assertThat(up.getDurationMs()).isNotNull(); - - for(String status : new String[] { "LATE", "FAILING", "MISSING_IN_STORAGE" }) - { - setStatus(critical, status); - HealthCheckResult down = indicator.check(QContext.getQInstance()); - assertThat(down.getStatus()).as(status).isEqualTo(HealthStatus.DOWN); - assertThat(down.getDetails().get("criticalPlansWithProblems").toString()).contains("nightly: " + status); - } - - setStatus(critical, "UNVERIFIED"); - assertThat(indicator.check(QContext.getQInstance()).getStatus()).as("not verified is not down").isEqualTo(HealthStatus.UP); - } - - - - /******************************************************************************* - ** On a thread with no QQQ context (a health request) the indicator makes its own - ** and cleans up. - *******************************************************************************/ - @Test - void testWorksWithoutAContext() throws Exception - { - QInstance qInstance = QContext.getQInstance(); - QContext.clear(); - try - { - assertThat(new BackupsHealthIndicator().check(qInstance).getStatus()).isEqualTo(HealthStatus.UP); - assertThat(QContext.getQInstance()).isNull(); - } - finally - { - QContext.init(qInstance, new com.kingsrook.qqq.backend.core.model.session.QSession()); - } - } -} -- 2.54.0 From f9767ede83d7b252186a8030f00b9c9689d0c6e2 Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 14 Oct 2026 14:55:03 -0500 Subject: [PATCH 3/3] chore: drop optional qqq-middleware-health dependency; docs for backups health checks Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 ++++ README.md | 27 ++++++++++++++++++++++++--- pom.xml | 7 ------- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 697db61..e2a9ca0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ All notable changes to this project are documented here. The format follows Keep ## [0.1.0] - Unreleased ### Added +- Central `backups.sla` and per-active-plan `backups.destination.` health checks, with bounded read-only destination diagnostics and secret-free status details. - Repository scaffold: `BackupsQBitConfig` and `BackupsQBitProducer` (table prefix `backups_`, permissions `backups.*`), build, CI and repository standard files. ### Changed @@ -15,3 +16,6 @@ All notable changes to this project are documented here. The format follows Keep - New manifests identify `qqq-table-schema-v1` and carry core schema descriptors; restore and content verification still accept pre-RC.6 manifests with the original fingerprint algorithm. - Preserve primary-key ordering, exclusive resume cursors, bounded pages and caller-owned transactions; reject non-orderable primary keys and scope filters with skip/limit. - Require the core `backups.restore` permission during Review when using a core restore writer, including anonymous-mode hosts; preserve supplied timestamps through `RestoreWriteFlag`. + +### Removed +- The legacy `BackupsHealthIndicator` and its optional `qqq-middleware-health` dependency; central health registration replaces manual indicator configuration. diff --git a/README.md b/README.md index bae86d2..b918b8e 100644 --- a/README.md +++ b/README.md @@ -36,13 +36,14 @@ Follow the QQQ [adoption checklist](https://github.com/QRun-IO/qqq/blob/develop/ ./mvnw verify ``` -Before RC.6 is released, verify against the locally installed core build only: +Health checks require the unreleased core health SPI, installed locally as `4.1.0-SNAPSHOT`: ```bash -./mvnw -B verify -Dqqq.snapshot.version=4.1.0-ARCHRC6-SNAPSHOT +./mvnw -o verify -Dqqq.snapshot.version=4.1.0-SNAPSHOT -Denforcer.skipRules=requireReleaseDeps -Dtest='!ObjectStorageBackupStorageMinioTest' ``` -The POM stays on RC.6. See [the RC.6 migration notes](docs/RC6-MIGRATION.md) for +This skips only the release-only dependency rule for the required local SNAPSHOT +and excludes the Docker/MinIO tests to keep verification offline. The POM stays on RC.6. See [the RC.6 migration notes](docs/RC6-MIGRATION.md) for fingerprint compatibility, paging guarantees and restore permissions. Always use the Maven wrapper (Maven 3.9.11); Maven 3.10 cannot read the parent POM. @@ -65,3 +66,23 @@ The backup SLA checker still owns expected intervals and grace periods. RC.5 requires hyphenated alert names: `backups.run.missing-in-storage` and `backups.plan.owner-invalid` replace the former underscore spellings, including their deduplication keys. Update host routing rules that used those spellings. + +## Health checks + +The producer registers these central checks under component `backups`, category `STORAGE`: + +| Name | Kind | Interval | Timeout | +| --- | --- | --- | --- | +| `backups.sla` | DEPENDENCY | 900s (15 minutes) | 5000ms | +| `backups.destination.` | DEPENDENCY | 86400s (24 hours) | 30000ms | + +The SLA check reads persisted `slaStatus`: critical active plans that are LATE, +FAILING or MISSING_IN_STORAGE yield DOWN; non-critical failures yield DEGRADED. +Only UNVERIFIED/UNKNOWN plans yield UNKNOWN; no active plans yield UP. Details +contain counts per SLA status. The scheduled `backupCheckSla` still computes SLA state. + +Destination checks are supplied for active plans and reuse `DestinationProbe` in +read-only list/head mode, including on unlocked buckets. They never write, delete, +or resolve prune credentials. The Test destination action retains its write tests. +Storage calls are bounded by the central runner, with deadline checks between calls. +Override intervals in `health_check` and disable checks through `health_check.enabled`. diff --git a/pom.xml b/pom.xml index 832f58d..0168619 100644 --- a/pom.xml +++ b/pom.xml @@ -74,13 +74,6 @@ qqq-backend-core - - - com.kingsrook.qqq - qqq-middleware-health - true - - com.kingsrook.qqq -- 2.54.0