From e57ab621931e17ec8e684f3a55a8f22f37e45ac9 Mon Sep 17 00:00:00 2001 From: James Maes Date: Thu, 15 Oct 2026 12:11:11 -0500 Subject: [PATCH 1/2] feat(cluster): register backups caches with ClusterCaches The cached Prometheus text is shared as qbit-backups.metrics and cleared on every node when a run finishes or a plan is saved or deleted (the plan customizer now also runs post insert/update/delete); the 60 s TTL stays as the safety net. SlaChecker's manifest-head and bucket-audit throttles and the process-permission map are node-local. Builds against QQQ 4.1.0-SNAPSHOT (qqq.snapshot.version) until the QQQ RC with ClusterCaches is published; a release must pin that RC. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 + pom.xml | 2 +- .../qbits/backups/BackupsRuntime.java | 4 +- .../qbits/backups/metrics/BackupsMetrics.java | 45 +++-- .../qbits/backups/model/BackupsTables.java | 17 +- .../plan/BackupPlanTableCustomizer.java | 43 ++++- .../qbits/backups/run/BackupEngine.java | 2 + .../qbits/backups/sla/SlaChecker.java | 8 +- .../backups/BackupsClusterCachesTest.java | 164 ++++++++++++++++++ .../backups/metrics/BackupsMetricsTest.java | 9 +- 10 files changed, 273 insertions(+), 23 deletions(-) create mode 100644 src/test/java/com/kingsrook/qbits/backups/BackupsClusterCachesTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 81ba497..17de33b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ All notable changes to this project are documented here. The format follows Keep - Repository scaffold: `BackupsQBitConfig` and `BackupsQBitProducer` (table prefix `backups_`, permissions `backups.*`), build, CI and repository standard files. ### Changed +- Multi-node deployments: the cached Prometheus text (`BackupsMetrics`) is registered with QQQ `ClusterCaches` as `qbit-backups.metrics` and cleared on every node when a run finishes or a plan is inserted, updated or deleted; the 60 s TTL remains the safety net. `SlaChecker`'s manifest-head and bucket-audit throttles and `BackupsRuntime`'s process-permission map are registered as node-local, with reasons. `BackupsClusterCachesTest` audits the package for unregistered static caches and covers a remote invalidation. +- Builds against QQQ `4.1.0-SNAPSHOT` (`qqq.snapshot.version`) for `ClusterCaches` until the next QQQ RC ships; a release must pin that RC. - Read the QBit metadata version from the Maven artifact version so release builds report their released version. - Target QQQ `4.1.0-RC.8` (job progress). - Persist controlled backup failure messages instead of untrusted provider exception text. diff --git a/pom.xml b/pom.xml index 7927a27..7768b08 100644 --- a/pom.xml +++ b/pom.xml @@ -216,7 +216,7 @@ true - 4.1.0-RC.8 + 4.1.0-SNAPSHOT diff --git a/src/main/java/com/kingsrook/qbits/backups/BackupsRuntime.java b/src/main/java/com/kingsrook/qbits/backups/BackupsRuntime.java index 6c629e7..218c7e1 100644 --- a/src/main/java/com/kingsrook/qbits/backups/BackupsRuntime.java +++ b/src/main/java/com/kingsrook/qbits/backups/BackupsRuntime.java @@ -25,6 +25,7 @@ import com.kingsrook.qbits.backups.alerts.BackupAlertSink; import com.kingsrook.qbits.backups.alerts.NoOpBackupAlertSink; import com.kingsrook.qbits.backups.storage.BackupStorage; import com.kingsrook.qbits.backups.storage.BackupStorageException; +import com.kingsrook.qqq.backend.core.cluster.ClusterCaches; /******************************************************************************* @@ -34,7 +35,8 @@ import com.kingsrook.qbits.backups.storage.BackupStorageException; *******************************************************************************/ public final class BackupsRuntime { - private static final Map PROCESS_PERMISSIONS = new ConcurrentHashMap<>(); + private static final Map PROCESS_PERMISSIONS = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(), + "built from the qbit config at startup, the same on every node"); private static volatile BackupsQBitConfig config; private static volatile Clock clock = Clock.systemUTC(); diff --git a/src/main/java/com/kingsrook/qbits/backups/metrics/BackupsMetrics.java b/src/main/java/com/kingsrook/qbits/backups/metrics/BackupsMetrics.java index 55abc26..028e76c 100644 --- a/src/main/java/com/kingsrook/qbits/backups/metrics/BackupsMetrics.java +++ b/src/main/java/com/kingsrook/qbits/backups/metrics/BackupsMetrics.java @@ -20,10 +20,12 @@ package com.kingsrook.qbits.backups.metrics; import java.time.Duration; import java.time.Instant; import java.util.List; +import java.util.concurrent.atomic.AtomicReference; import com.kingsrook.qbits.backups.BackupsQBitConfig; import com.kingsrook.qbits.backups.BackupsRuntime; import com.kingsrook.qbits.backups.model.BackupsTables; import com.kingsrook.qbits.backups.run.BackupCatalog; +import com.kingsrook.qqq.backend.core.cluster.ClusterCaches; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; @@ -56,10 +58,21 @@ public final class BackupsMetrics { public static final String ENDPOINT_PATH = "/metrics/backups"; + /** Cluster cache name of the rendered text: cleared on every node when a run finishes or a plan changes. */ + public static final String CACHE_NAME = "qbit-backups.metrics"; + private static final Duration CACHE = Duration.ofSeconds(60); - private static String cachedText; - private static Instant cachedAt; + private static final AtomicReference CACHED = ClusterCaches.shared(CACHE_NAME, new AtomicReference<>(), BackupsMetrics::resetCache, null); + + + + /******************************************************************************* + ** The rendered text and when it was rendered. + *******************************************************************************/ + private record Rendered(String text, Instant at) + { + } @@ -84,12 +97,21 @@ public final class BackupsMetrics /******************************************************************************* - ** Forget the cached text (tests). + ** Forget the cached text on this node. *******************************************************************************/ - public static synchronized void resetCache() + public static void resetCache() { - cachedText = null; - cachedAt = null; + CACHED.set(null); + } + + + + /******************************************************************************* + ** Forget the cached text on every node (the backup data changed). + *******************************************************************************/ + public static void invalidate() + { + ClusterCaches.invalidate(CACHE_NAME); } @@ -99,13 +121,14 @@ public final class BackupsMetrics *******************************************************************************/ public static synchronized String renderCached() throws QException { - Instant now = BackupsRuntime.getClock().instant(); - if(cachedText == null || cachedAt == null || cachedAt.plus(CACHE).isBefore(now)) + Instant now = BackupsRuntime.getClock().instant(); + Rendered cached = CACHED.get(); + if(cached == null || cached.at().plus(CACHE).isBefore(now)) { - cachedText = render(); - cachedAt = now; + cached = new Rendered(render(), now); + CACHED.set(cached); } - return (cachedText); + return (cached.text()); } diff --git a/src/main/java/com/kingsrook/qbits/backups/model/BackupsTables.java b/src/main/java/com/kingsrook/qbits/backups/model/BackupsTables.java index f509571..c8d6b02 100644 --- a/src/main/java/com/kingsrook/qbits/backups/model/BackupsTables.java +++ b/src/main/java/com/kingsrook/qbits/backups/model/BackupsTables.java @@ -413,15 +413,22 @@ public final class BackupsTables QCodeReference validation = new QCodeReference(BackupPlanTableCustomizer.class); table.withCustomizer(TableCustomizers.PRE_INSERT_RECORD, validation); table.withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, validation); + + ///////////////////////////////////////////////////////////////////////////// + // after a save or delete, the plan customizer clears the cached metrics on // + // every node // + ///////////////////////////////////////////////////////////////////////////// + QCodeReference post = validation; if(StringUtils.hasContent(config.getSchedulerName())) { - QCodeReferenceWithProperties sync = MultiCustomizer.of( + post = MultiCustomizer.of( BaseSyncToScheduledJobTableCustomizer.makeCodeReference(table, new BackupPlanSyncToScheduledJobProcess()), - BaseSyncToScheduledJobTableCustomizer.makeCodeReference(table, new BackupPlanVerifySyncToScheduledJobProcess())); - table.withCustomizer(TableCustomizers.POST_INSERT_RECORD, sync); - table.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, sync); - table.withCustomizer(TableCustomizers.POST_DELETE_RECORD, sync); + BaseSyncToScheduledJobTableCustomizer.makeCodeReference(table, new BackupPlanVerifySyncToScheduledJobProcess()), + validation); } + table.withCustomizer(TableCustomizers.POST_INSERT_RECORD, post); + table.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, post); + table.withCustomizer(TableCustomizers.POST_DELETE_RECORD, post); return (table); } diff --git a/src/main/java/com/kingsrook/qbits/backups/plan/BackupPlanTableCustomizer.java b/src/main/java/com/kingsrook/qbits/backups/plan/BackupPlanTableCustomizer.java index c4e32b0..240834e 100644 --- a/src/main/java/com/kingsrook/qbits/backups/plan/BackupPlanTableCustomizer.java +++ b/src/main/java/com/kingsrook/qbits/backups/plan/BackupPlanTableCustomizer.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; +import com.kingsrook.qbits.backups.metrics.BackupsMetrics; import com.kingsrook.qbits.backups.model.BackupsTables; import com.kingsrook.qbits.backups.run.BackupCatalog; import com.kingsrook.qbits.backups.run.BackupRunFactory; @@ -31,6 +32,9 @@ import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterfa import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; @@ -43,7 +47,8 @@ import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage; ** configuration (the processes write the last run and SLA fields) is not ** validated again. ** - ** The saving user becomes the plan owner when none is given. + ** The saving user becomes the plan owner when none is given. After an insert, + ** update or delete, the cached Prometheus text is cleared on every node. *******************************************************************************/ public class BackupPlanTableCustomizer implements TableCustomizerInterface { @@ -122,6 +127,42 @@ public class BackupPlanTableCustomizer implements TableCustomizerInterface + /******************************************************************************* + ** A saved plan changes the metrics on every node. + *******************************************************************************/ + @Override + public List postInsert(InsertInput insertInput, List records) throws QException + { + BackupsMetrics.invalidate(); + return (records); + } + + + + /******************************************************************************* + ** A saved plan changes the metrics on every node. + *******************************************************************************/ + @Override + public List postUpdate(UpdateInput updateInput, List records, Optional> oldRecordList) throws QException + { + BackupsMetrics.invalidate(); + return (records); + } + + + + /******************************************************************************* + ** A deleted plan changes the metrics on every node. + *******************************************************************************/ + @Override + public List postDelete(DeleteInput deleteInput, List records) throws QException + { + BackupsMetrics.invalidate(); + return (records); + } + + + /******************************************************************************* ** The scope rules to resolve: the ones sent with the plan (associated rows), or ** the saved ones. diff --git a/src/main/java/com/kingsrook/qbits/backups/run/BackupEngine.java b/src/main/java/com/kingsrook/qbits/backups/run/BackupEngine.java index fcc55fe..fd06b03 100644 --- a/src/main/java/com/kingsrook/qbits/backups/run/BackupEngine.java +++ b/src/main/java/com/kingsrook/qbits/backups/run/BackupEngine.java @@ -47,6 +47,7 @@ import com.kingsrook.qbits.backups.files.FileReference; import com.kingsrook.qbits.backups.files.FileReferenceProvider; import com.kingsrook.qbits.backups.manifest.BackupManifest; import com.kingsrook.qbits.backups.manifest.TableFingerprints; +import com.kingsrook.qbits.backups.metrics.BackupsMetrics; import com.kingsrook.qbits.backups.model.BackupsTables; import com.kingsrook.qbits.backups.scope.ScopeResolver; import com.kingsrook.qbits.backups.storage.BackupStorage; @@ -774,6 +775,7 @@ public class BackupEngine result.setStatus(status); result.setErrorMessage(message); signal(request, result, status, message); + BackupsMetrics.invalidate(); } diff --git a/src/main/java/com/kingsrook/qbits/backups/sla/SlaChecker.java b/src/main/java/com/kingsrook/qbits/backups/sla/SlaChecker.java index b79195b..94b9978 100644 --- a/src/main/java/com/kingsrook/qbits/backups/sla/SlaChecker.java +++ b/src/main/java/com/kingsrook/qbits/backups/sla/SlaChecker.java @@ -35,6 +35,7 @@ import com.kingsrook.qbits.backups.model.BackupsTables; import com.kingsrook.qbits.backups.run.BackupCatalog; import com.kingsrook.qbits.backups.storage.BackupStorage; import com.kingsrook.qbits.backups.storage.StoredObjectInfo; +import com.kingsrook.qqq.backend.core.cluster.ClusterCaches; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; @@ -95,8 +96,11 @@ public class SlaChecker private static final Set FAILED_STATUSES = Set.of("FAILED", "TIMED_OUT", "ABANDONED"); private static final Set FINISHED_STATUSES = Set.of("SUCCEEDED", "SUCCEEDED_WITH_WARNINGS", "FAILED", "TIMED_OUT", "ABANDONED"); - private static final Map MANIFEST_OK_AT = new ConcurrentHashMap<>(); - private static final Map LAST_AUDIT_AT = new ConcurrentHashMap<>(); + private static final Map MANIFEST_OK_AT = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(), + "throttles this node's storage HEAD of a manifest to once per MANIFEST_CACHE; it caches storage state, not editable data, " + + "and a pruned run is skipped by its prunedDate, so each node is as fresh as a single node"); + private static final Map LAST_AUDIT_AT = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(), + "throttles this node's bucket audit to once per BUCKET_AUDIT_INTERVAL; another node auditing again is only repeated work"); private final BackupCatalog catalog = new BackupCatalog(); diff --git a/src/test/java/com/kingsrook/qbits/backups/BackupsClusterCachesTest.java b/src/test/java/com/kingsrook/qbits/backups/BackupsClusterCachesTest.java new file mode 100644 index 0000000..43b4ae1 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/backups/BackupsClusterCachesTest.java @@ -0,0 +1,164 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups; + + +import java.util.EnumSet; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.Consumer; +import com.kingsrook.qbits.backups.metrics.BackupsMetrics; +import com.kingsrook.qbits.backups.model.BackupsTables; +import com.kingsrook.qbits.backups.run.BackupCatalog; +import com.kingsrook.qqq.backend.core.cluster.ClusterBus; +import com.kingsrook.qqq.backend.core.cluster.ClusterBusInterface; +import com.kingsrook.qqq.backend.core.cluster.ClusterBusMetaData; +import com.kingsrook.qqq.backend.core.cluster.ClusterCacheAudit; +import com.kingsrook.qqq.backend.core.cluster.ClusterEvent; +import com.kingsrook.qqq.backend.core.cluster.QNodeRoles; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** The Backups caches are correct on several nodes: every static cache is + ** registered with ClusterCaches, and the cached metrics are cleared on every + ** node when a plan changes. + *******************************************************************************/ +class BackupsClusterCachesTest +{ + /** the "wire": events every node's transport sees */ + static final List> WIRE = new CopyOnWriteArrayList<>(); + + + + /******************************************************************************* + ** A transport over the static WIRE, so a "remote" publish reaches us. + *******************************************************************************/ + public static class WireBus implements ClusterBusInterface + { + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void publish(ClusterEvent event) + { + WIRE.forEach(listener -> listener.accept(event)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public AutoCloseable subscribe(Consumer listener) + { + WIRE.add(listener); + return (() -> WIRE.remove(listener)); + } + } + + + + /******************************************************************************* + ** A backed-up test instance, connected to the wire as node-a. + *******************************************************************************/ + @BeforeEach + void setUp() throws Exception + { + VerifyTestSupport.setUp(1); + QInstance instance = QContext.getQInstance(); + instance.withSupplementalMetaData(new ClusterBusMetaData().withBus(new QCodeReference(WireBus.class))); + QNodeRoles.set(new QNodeRoles.Settings(EnumSet.allOf(QNodeRoles.Role.class), true, true, true, null, "node-a")); + ClusterBus.connect(instance); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + ClusterBus.close(QContext.getQInstance()); + WIRE.clear(); + QNodeRoles.set(null); + } + + + + /******************************************************************************* + ** Every static Memoization in the QBit is registered (shared or node-local). + *******************************************************************************/ + @Test + void testNoUnregisteredStaticCaches() throws Exception + { + assertThat(ClusterCacheAudit.unregisteredStaticCaches("com.kingsrook.qbits.backups")).isEmpty(); + } + + + + /******************************************************************************* + ** Another node's invalidation clears this node's cached metrics. + *******************************************************************************/ + @Test + void testRemoteInvalidationClearsMetricsCache() throws Exception + { + String cached = BackupsMetrics.renderCached(); + assertThat(BackupsMetrics.renderCached()).isSameAs(cached); + + ClusterEvent remote = new ClusterEvent(ClusterEvent.TYPE_MEMO_INVALIDATE, BackupsMetrics.CACHE_NAME, null); + remote.setOriginNodeId("node-b"); + WIRE.forEach(listener -> listener.accept(remote)); + + assertThat(BackupsMetrics.renderCached()).isNotSameAs(cached); + } + + + + /******************************************************************************* + ** A plan edit here clears the metrics here and tells the other nodes. + *******************************************************************************/ + @Test + void testPlanEditPublishesMetricsInvalidation() throws Exception + { + QRecord plan = VerifyTestSupport.plan(Map.of()); + String cached = BackupsMetrics.renderCached(); + + List sent = new CopyOnWriteArrayList<>(); + WIRE.add(sent::add); + + new BackupCatalog().update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("slaHours", 48)); + + assertThat(BackupsMetrics.renderCached()).isNotSameAs(cached); + assertThat(sent).anySatisfy(event -> + { + assertThat(event.getType()).isEqualTo(ClusterEvent.TYPE_MEMO_INVALIDATE); + assertThat(event.getName()).isEqualTo(BackupsMetrics.CACHE_NAME); + assertThat(event.getOriginNodeId()).isEqualTo("node-a"); + }); + } +} diff --git a/src/test/java/com/kingsrook/qbits/backups/metrics/BackupsMetricsTest.java b/src/test/java/com/kingsrook/qbits/backups/metrics/BackupsMetricsTest.java index 91b7d68..c2275f4 100644 --- a/src/test/java/com/kingsrook/qbits/backups/metrics/BackupsMetricsTest.java +++ b/src/test/java/com/kingsrook/qbits/backups/metrics/BackupsMetricsTest.java @@ -87,10 +87,15 @@ class BackupsMetricsTest String empty = BackupsMetrics.renderCached(); assertThat(empty).doesNotContain("{instance_name"); - VerifyTestSupport.backup(VerifyTestSupport.plan(Map.of())); VerifyTestSupport.clock.advance(Duration.ofSeconds(30)); - assertThat(BackupsMetrics.renderCached()).as("cached").isEqualTo(empty); + assertThat(BackupsMetrics.renderCached()).as("cached").isSameAs(empty); VerifyTestSupport.clock.advance(Duration.ofSeconds(31)); + assertThat(BackupsMetrics.renderCached()).as("expired").isNotSameAs(empty); + + ///////////////////////////////////////////////////////////// + // a saved plan and a finished run clear the cache at once // + ///////////////////////////////////////////////////////////// + VerifyTestSupport.backup(VerifyTestSupport.plan(Map.of())); assertThat(BackupsMetrics.renderCached()).contains("plan=\"nightly\""); assertThat(BackupsMetrics.escape("a\"b\\c\nd")).isEqualTo("a\\\"b\\\\c\\nd"); -- 2.54.0 From 2f1964ddd1d03ca4e4e3bb946426427c7139e9da Mon Sep 17 00:00:00 2001 From: James Maes Date: Thu, 15 Oct 2026 14:05:06 -0500 Subject: [PATCH 2/2] build: QQQ 4.1.0-RC.14 (ships ClusterCaches) Co-Authored-By: Claude Opus 5.5 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7768b08..c62e6e3 100644 --- a/pom.xml +++ b/pom.xml @@ -216,7 +216,7 @@ true - 4.1.0-SNAPSHOT + 4.1.0-RC.14 -- 2.54.0