refactor(alerts): direct core QAlerts binding on QQQ RC.5 (#19 part 1) #20

Merged
KofTwentyTwo merged 1 commits from feature/arch-alerts-core into develop 2026-10-14 00:37:02 +00:00
20 changed files with 268 additions and 646 deletions

No files matched your search

+1 -1
View File
@@ -35,7 +35,7 @@ This file provides guidance to Codex when working with code in this repository.
## Dependencies
- Java 21.
- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). The default-active `qqq-snapshot` profile imports `qqq-bom-pom:4.1.0-RC.4` ahead of it, as in qbit-quick-search (ADR-0007). Remove that activation when the parent moves to 4.1.
- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). The default-active `qqq-snapshot` profile imports `qqq-bom-pom:4.1.0-RC.5` ahead of it, as in qbit-quick-search (ADR-0007). Remove that activation when the parent moves to 4.1.
- Tests: JUnit 5, AssertJ.
## Release
+6
View File
@@ -6,3 +6,9 @@ All notable changes to this project are documented here. The format follows Keep
### Added
- Repository scaffold: `BackupsQBitConfig` and `BackupsQBitProducer` (table prefix `backups_`, permissions `backups.*`), build, CI and repository standard files.
### Changed
- Build against the released QQQ BOM 4.1.0-RC.5; replace reflective alerts binding with core QAlerts and real sink integration tests.
- Register backup alert types for every instance while preserving custom backup sinks and no-op behavior without a configured core sink.
- Use RC.5-valid alert names `backups.run.missing-in-storage` and `backups.plan.owner-invalid` (and matching deduplication keys). Hosts using the old underscore names must update routing rules.
- Mark fingerprint, keyset reader, and restore bridges for removal in RC.6.
+16 -1
View File
@@ -10,7 +10,7 @@
## Requirements
- Java 21.
- QQQ 4.1 (built against `4.1.0-RC.4`).
- QQQ 4.1 (built against `4.1.0-RC.5`).
## Install
@@ -41,3 +41,18 @@ Always use the Maven wrapper (Maven 3.9.11); Maven 3.10 cannot read the parent P
## License
Apache License 2.0. See [LICENSE](LICENSE) and [NOTICE](NOTICE).
### Alerts integration
Backups binds directly to core QAlerts in QQQ 4.1.0-RC.5 and declares its nine
alert types on each instance. Hosts install an `AlertSinkInterface` through
`AlertsInstanceMetaData.withSink(QCodeReference)`. Without a configured sink,
backup alert calls remain no-ops (they do not use core's logging fallback).
Custom `BackupAlertSink` bindings are preserved. Core handles dispatch failures;
adapter failures are caught and logged once, so alerting cannot fail a backup.
Raise and resolve follow core's transaction-aware dispatch; heartbeats run immediately.
The backup SLA checker still owns expected intervals and grace periods.
RC.5 requires hyphenated alert names: `backups.run.missing-in-storage` and
`backups.plan.owner-invalid` replace the former underscore spellings, including
their deduplication keys. Update host routing rules that used those spellings.
+2 -2
View File
@@ -192,14 +192,14 @@
<profiles>
<!-- Built against qqq 4.1 (QBitMetaDataProducer). Until qbit-build-parent -->
<!-- imports the released 4.1 BOM, this profile is active by default and -->
<!-- imports qqq-bom-pom 4.1.0-RC.4 ahead of the parent's 4.0.0 BOM. -->
<!-- imports qqq-bom-pom 4.1.0-RC.5 ahead of the parent's 4.0.0 BOM. -->
<profile>
<id>qqq-snapshot</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<qqq.snapshot.version>4.1.0-RC.4</qqq.snapshot.version>
<qqq.snapshot.version>4.1.0-RC.5</qqq.snapshot.version>
</properties>
<dependencyManagement>
<dependencies>
@@ -119,13 +119,8 @@ public class BackupsQBitProducer implements QBitMetaDataProducer<BackupsQBitConf
qInstance.addQBit(qBit);
BackupsRuntime.install(qBitConfig);
///////////////////////////////////////////////////////////////////////////////
// bind the Alerts qbit producer SPI when it is on the class path (optional) //
///////////////////////////////////////////////////////////////////////////////
if(com.kingsrook.qbits.backups.alerts.AlertsQBitAlertSink.bindIfPresent())
{
new com.kingsrook.qbits.backups.alerts.AlertsQBitAlertSink().registerAlertTypes(qInstance);
}
com.kingsrook.qbits.backups.alerts.AlertsQBitAlertSink.bindIfAbsent();
new com.kingsrook.qbits.backups.alerts.AlertsQBitAlertSink().registerAlertTypes(qInstance);
com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput output = new com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput();
for(var possibleValueSource : com.kingsrook.qbits.backups.model.BackupsTables.possibleValueSources())
@@ -17,105 +17,39 @@
package com.kingsrook.qbits.backups.alerts;
import java.lang.reflect.Method;
import java.util.Map;
import java.util.stream.Collectors;
import com.kingsrook.qbits.backups.BackupsRuntime;
import com.kingsrook.qqq.backend.core.alerts.AlertSeverity;
import com.kingsrook.qqq.backend.core.alerts.AlertSignal;
import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData;
import com.kingsrook.qqq.backend.core.alerts.QAlertTypeMetaData;
import com.kingsrook.qqq.backend.core.alerts.QAlerts;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
** The BackupAlertSink adapter for the Alerts qbit producer SPI (QRun-IO/qqq
** PR #1108, com.kingsrook.qqq.backend.core.alerts: QAlerts, AlertSignal,
** AlertResolution, AlertSeverity, QAlertTypeMetaData; design spec
** docs/specs/2026-10-12-qbit-alerts-design.md).
**
** STAND-IN: the SPI is an open core PR and is not in QQQ 4.1.0-RC.4, so this
** adapter is bound REFLECTIVELY and optionally. It compiles against nothing in
** the SPI; it looks the classes up by name, and every call is guarded:
** <ul>
** <li>no SPI classes on the class path: {@link #bindIfPresent()} binds nothing
** and the sink stays the no-op;</li>
** <li>the SPI is present but the instance has no alerts meta-data (the Alerts
** qbit is not installed): every call is a no-op;</li>
** <li>any reflective or SPI failure is logged once and swallowed, because an
** alerting problem must never fail a backup.</li>
** </ul>
** Replace with a direct binding (QAlerts.raise and friends) when RC.5 ships.
**
** Mapping: raise to QAlerts.raise(AlertSignal), resolve to QAlerts.resolve(type,
** dedupKey, note), heartbeat to QAlerts.heartbeat(key). The SPI has no per-plan
** heartbeat registration (monitors are declared on the alert type), so the
** expected interval and grace are not passed on; the SLA checker raises
** backups.plan.missed itself, independent of any heartbeat (spec 10.4).
** Direct adapter to core QAlerts. Calls are no-ops without a configured alerts
** sink, including when only the backup alert types have been registered.
** QAlerts catches and logs dispatch failures; adapter failures are logged once.
** Heartbeat intervals remain the responsibility of the backup SLA checker.
*******************************************************************************/
public class AlertsQBitAlertSink implements BackupAlertSink
{
private static final QLogger LOG = QLogger.getLogger(AlertsQBitAlertSink.class);
/** The package of the Alerts producer SPI in qqq-backend-core. */
public static final String DEFAULT_API_PACKAGE = "com.kingsrook.qqq.backend.core.alerts";
private final String apiPackage;
private boolean warned = false;
private Class<?> qAlerts;
private Class<?> alertSignal;
private Class<?> alertSeverity;
private Class<?> instanceMetaData;
/*******************************************************************************
** Constructor for the real SPI package.
** Bind the core adapter unless the host already supplied a backup sink.
*******************************************************************************/
public AlertsQBitAlertSink()
public static boolean bindIfAbsent()
{
this(DEFAULT_API_PACKAGE);
}
/*******************************************************************************
** Constructor for another package (a test double of the SPI shape).
*******************************************************************************/
public AlertsQBitAlertSink(String apiPackage)
{
this.apiPackage = apiPackage;
}
/*******************************************************************************
** Whether the SPI classes are on the class path.
*******************************************************************************/
public static boolean isPresent(String apiPackage)
{
try
{
Class.forName(apiPackage + ".QAlerts");
Class.forName(apiPackage + ".AlertSignal");
return (true);
}
catch(ClassNotFoundException | LinkageError e)
{
return (false);
}
}
/*******************************************************************************
** Bind this adapter as the runtime's alert sink when the SPI is present and no
** other sink was bound already. Returns whether it bound.
*******************************************************************************/
public static boolean bindIfPresent()
{
if(!(BackupsRuntime.getAlertSink() instanceof NoOpBackupAlertSink) || !isPresent(DEFAULT_API_PACKAGE))
if(!(BackupsRuntime.getAlertSink() instanceof NoOpBackupAlertSink))
{
return (false);
}
@@ -126,38 +60,32 @@ public class AlertsQBitAlertSink implements BackupAlertSink
/*******************************************************************************
** Declare the backup alert types with the Alerts qbit (QAlertTypeMetaData), when
** the SPI is present. Returns how many were added.
** Declare missing backup alert types on each instance, independently of sink
** installation order. Returns how many were added.
*******************************************************************************/
public int registerAlertTypes(QInstance qInstance)
{
int added = 0;
try
for(BackupAlerts.AlertType alertType : BackupAlerts.alertTypes())
{
Class<?> typeClass = Class.forName(apiPackage + ".QAlertTypeMetaData");
load();
for(BackupAlerts.AlertType alertType : BackupAlerts.alertTypes())
try
{
try
if(QAlertTypeMetaData.of(qInstance, alertType.name()) == null)
{
Object type = typeClass.getConstructor().newInstance();
typeClass.getMethod("withName", String.class).invoke(type, alertType.name());
typeClass.getMethod("withLabel", String.class).invoke(type, alertType.label());
typeClass.getMethod("withGroupName", String.class).invoke(type, "Backups");
typeClass.getMethod("withHowToFix", String.class).invoke(type, alertType.howToFix());
typeClass.getMethod("withDefaultSeverity", alertSeverity).invoke(type, severity(alertType.defaultSeverity()));
typeClass.getMethod("addSelfToInstance", QInstance.class).invoke(type, qInstance);
new QAlertTypeMetaData()
.withName(alertType.name())
.withLabel(alertType.label())
.withGroupName("Backups")
.withHowToFix(alertType.howToFix())
.withDefaultSeverity(AlertSeverity.valueOf(alertType.defaultSeverity().name()))
.addSelfToInstance(qInstance);
added++;
}
catch(ReflectiveOperationException | RuntimeException e)
{
warnOnce("Could not declare the alert type " + alertType.name(), e);
}
}
}
catch(ReflectiveOperationException | LinkageError e)
{
warnOnce("The alerts SPI is not available", e);
catch(RuntimeException | LinkageError e)
{
warnOnce("Could not declare the alert type " + alertType.name(), e);
}
}
return (added);
}
@@ -170,28 +98,24 @@ public class AlertsQBitAlertSink implements BackupAlertSink
@Override
public void raise(String type, String dedupeKey, Severity severity, String title, Map<String, String> detail)
{
if(!isAlertsQBitInstalled())
{
return;
}
try
{
Object signal = alertSignal.getConstructor(String.class).newInstance(type);
alertSignal.getMethod("setDedupKey", String.class).invoke(signal, dedupeKey);
alertSignal.getMethod("setSeverity", alertSeverity).invoke(signal, severity(severity));
alertSignal.getMethod("setTitle", String.class).invoke(signal, title);
if(!isAlertsQBitInstalled())
{
return;
}
AlertSignal signal = new AlertSignal(type)
.withDedupKey(dedupeKey)
.withSeverity(AlertSeverity.valueOf(severity.name()))
.withTitle(title);
if(detail != null && !detail.isEmpty())
{
alertSignal.getMethod("setSummary", String.class).invoke(signal, detail.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(", ")));
Method withAttribute = alertSignal.getMethod("withAttribute", String.class, String.class);
for(Map.Entry<String, String> entry : detail.entrySet())
{
withAttribute.invoke(signal, entry.getKey(), entry.getValue());
}
signal.withSummary(detail.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(", ")));
detail.forEach(signal::withAttribute);
}
qAlerts.getMethod("raise", alertSignal).invoke(null, signal);
QAlerts.raise(signal);
}
catch(ReflectiveOperationException | RuntimeException e)
catch(RuntimeException | LinkageError e)
{
warnOnce("Could not raise an alert", e);
}
@@ -205,15 +129,14 @@ public class AlertsQBitAlertSink implements BackupAlertSink
@Override
public void resolve(String type, String dedupeKey)
{
if(!isAlertsQBitInstalled())
{
return;
}
try
{
qAlerts.getMethod("resolve", String.class, String.class, String.class).invoke(null, type, dedupeKey, "Cleared by the Backups qbit");
if(isAlertsQBitInstalled())
{
QAlerts.resolve(type, dedupeKey, "Cleared by the Backups qbit");
}
}
catch(ReflectiveOperationException | RuntimeException e)
catch(RuntimeException | LinkageError e)
{
warnOnce("Could not resolve an alert", e);
}
@@ -227,15 +150,14 @@ public class AlertsQBitAlertSink implements BackupAlertSink
@Override
public void heartbeat(String monitorKey, long expectedEverySeconds, long graceSeconds)
{
if(!isAlertsQBitInstalled())
{
return;
}
try
{
qAlerts.getMethod("heartbeat", String.class).invoke(null, monitorKey);
if(isAlertsQBitInstalled())
{
QAlerts.heartbeat(monitorKey);
}
}
catch(ReflectiveOperationException | RuntimeException e)
catch(RuntimeException | LinkageError e)
{
warnOnce("Could not check in a heartbeat", e);
}
@@ -244,67 +166,26 @@ public class AlertsQBitAlertSink implements BackupAlertSink
/*******************************************************************************
** The SPI classes load, and the instance has alerts meta-data (the Alerts qbit
** is installed). Checked on every call so the order qbits are produced in
** does not matter.
** Check on every call so installing the alerts sink after this qbit works.
** Type declarations alone do not mean an alerts implementation is installed.
*******************************************************************************/
boolean isAlertsQBitInstalled()
{
try
{
load();
QInstance qInstance = QContext.getQInstance();
if(qInstance == null)
{
return (false);
}
return (instanceMetaData.getMethod("of", QInstance.class).invoke(null, qInstance) != null);
}
catch(ReflectiveOperationException | RuntimeException | LinkageError e)
{
warnOnce("The alerts SPI is not usable", e);
return (false);
}
AlertsInstanceMetaData metaData = AlertsInstanceMetaData.of(QContext.getQInstance());
return (metaData != null && metaData.getSink() != null);
}
/*******************************************************************************
**
** Log adapter failures once; dispatch failures are already logged by core.
*******************************************************************************/
private synchronized void load() throws ClassNotFoundException
{
if(qAlerts == null)
{
qAlerts = Class.forName(apiPackage + ".QAlerts");
alertSignal = Class.forName(apiPackage + ".AlertSignal");
alertSeverity = Class.forName(apiPackage + ".AlertSeverity");
instanceMetaData = Class.forName(apiPackage + ".AlertsInstanceMetaData");
}
}
/*******************************************************************************
** Map our severity to the SPI's enum constant of the same name.
*******************************************************************************/
@SuppressWarnings({ "unchecked", "rawtypes" })
private Object severity(Severity severity)
{
return (Enum.valueOf((Class<Enum>) alertSeverity, severity.name()));
}
/*******************************************************************************
**
*******************************************************************************/
private void warnOnce(String message, Throwable t)
private synchronized void warnOnce(String message, Throwable t)
{
if(!warned)
{
warned = true;
LOG.warn(message + " (the Alerts qbit adapter will keep failing quietly)", t, logPair("apiPackage", apiPackage));
LOG.warn(message + " (the Alerts qbit adapter will keep failing quietly)", t);
}
}
}
@@ -36,11 +36,11 @@ public final class BackupAlerts
{
public static final String PLAN_MISSED = "backups.plan.missed";
public static final String RUN_FAILED = "backups.run.failed";
public static final String RUN_MISSING = "backups.run.missing_in_storage";
public static final String RUN_MISSING = "backups.run.missing-in-storage";
public static final String FILES_MISSING = "backups.files.missing";
public static final String VERIFY_FAILED = "backups.verify.failed";
public static final String VERIFY_OVERDUE = "backups.verify.overdue";
public static final String PLAN_OWNER_INVALID = "backups.plan.owner_invalid";
public static final String PLAN_OWNER_INVALID = "backups.plan.owner-invalid";
public static final String STORAGE_DEBRIS = "backups.storage.debris";
public static final String RESTORE_COMPLETED = "backups.restore.completed";
@@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
/*******************************************************************************
** STAND-IN(core=QRun-IO/qqq#1113, removeBy=4.1.0-RC.6)
** A stable schema descriptor and SHA-256 fingerprint of a table (spec 5.5).
**
** This is an internal stand-in for the QQQ core TableSchemaFingerprint (core PR
@@ -45,6 +45,8 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
** STAND-IN(core=QRun-IO/qqq#1111, removeBy=4.1.0-RC.6)
** STAND-IN(core=QRun-IO/qqq#1112, removeBy=4.1.0-RC.6)
** The default RestoreWriter: the standard Insert, Update and Delete actions.
**
** Stand-ins (replace with the core API when QQQ 4.1.0-RC.5 ships):
@@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.utils.JsonUtils;
/*******************************************************************************
** STAND-IN(core=QRun-IO/qqq#1114, removeBy=4.1.0-RC.6)
** Reads the rows of one table in primary key order, in keyset pages (WHERE pk >
** lastKey ORDER BY pk LIMIT n), so the cost stays flat with depth and a read can
** resume from a key. A read raw: hidden fields, passwords and heavy fields are
@@ -50,7 +50,7 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
** run is a full-instance, system-session backup (securityScope FULL, requested by
** system:scheduler), which is why the plan owner is validated first: a plan whose
** owner has gone, or lost backups.admin, is not run silently. It raises
** backups.plan.owner_invalid, marks the plan OWNER_INVALID, and stops.
** backups.plan.owner-invalid, marks the plan OWNER_INVALID, and stops.
**
** Everything goes through the run coordinator, so a trigger that finds the plan
** already running is skipped visibly, and a full instance queues the run.
@@ -141,11 +141,11 @@ public class BackupScheduledRunProcess implements BackendStep
if(problem != null)
{
catalog.update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("slaStatus", "OWNER_INVALID"));
BackupsRuntime.getAlertSink().raise("backups.plan.owner_invalid", "backups.plan.owner_invalid:" + plan.getValueString("planKey"), BackupAlertSink.Severity.WARNING,
BackupsRuntime.getAlertSink().raise("backups.plan.owner-invalid", "backups.plan.owner-invalid:" + plan.getValueString("planKey"), BackupAlertSink.Severity.WARNING,
"A scheduled backup did not run: " + problem, Map.of("plan", plan.getValueString("planKey")));
throw (new QUserFacingException("The scheduled backup did not run: " + problem + "."));
}
BackupsRuntime.getAlertSink().resolve("backups.plan.owner_invalid", "backups.plan.owner_invalid:" + plan.getValueString("planKey"));
BackupsRuntime.getAlertSink().resolve("backups.plan.owner-invalid", "backups.plan.owner-invalid:" + plan.getValueString("planKey"));
BackupRunResult result = new BackupRunCoordinator().runPlan(plan, "SCHEDULED");
output.addValue("status", result.getStatus());
@@ -18,16 +18,24 @@ package com.kingsrook.qbits.backups.alerts;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qbits.backups.BackupsRuntime;
import com.kingsrook.qbits.backups.VerifyTestSupport;
import com.kingsrook.qbits.backups.alerts.fakeapi.AlertsInstanceMetaData;
import com.kingsrook.qbits.backups.alerts.fakeapi.QAlertTypeMetaData;
import com.kingsrook.qbits.backups.alerts.fakeapi.QAlerts;
import com.kingsrook.qbits.backups.sla.SlaChecker;
import com.kingsrook.qqq.backend.core.alerts.AlertResolution;
import com.kingsrook.qqq.backend.core.alerts.AlertSignal;
import com.kingsrook.qqq.backend.core.alerts.AlertSinkInterface;
import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData;
import com.kingsrook.qqq.backend.core.alerts.QAlerts;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -35,14 +43,10 @@ import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** The Alerts qbit adapter, bound reflectively. The SPI is not in QQQ 4.1.0-RC.4,
** so these tests run it against a test double with the same shape as the real
** com.kingsrook.qqq.backend.core.alerts package (QRun-IO/qqq PR #1108).
** The backup adapter against the released core alerts API and a registered sink.
*******************************************************************************/
class AlertsQBitAlertSinkTest
{
private static final String FAKE_API = "com.kingsrook.qbits.backups.alerts.fakeapi";
/*******************************************************************************
@@ -52,9 +56,11 @@ class AlertsQBitAlertSinkTest
void setUp() throws Exception
{
VerifyTestSupport.setUp(3);
QAlerts.reset();
QAlertTypeMetaData.ADDED.clear();
AlertsInstanceMetaData.installed = true;
RecordingSink.raised.clear();
RecordingSink.resolved.clear();
RecordingSink.heartbeats.clear();
RecordingSink.failWith = null;
AlertsInstanceMetaData.ofOrWithNew(QContext.getQInstance()).withSink(new QCodeReference(RecordingSink.class));
}
@@ -65,8 +71,8 @@ class AlertsQBitAlertSinkTest
@AfterEach
void tearDown()
{
AlertsInstanceMetaData.installed = false;
QAlerts.reset();
BackupsRuntime.reset();
QContext.clear();
}
@@ -78,7 +84,7 @@ class AlertsQBitAlertSinkTest
@Test
void testRaiseResolveHeartbeat()
{
AlertsQBitAlertSink sink = new AlertsQBitAlertSink(FAKE_API);
AlertsQBitAlertSink sink = new AlertsQBitAlertSink();
Map<String, String> detail = new LinkedHashMap<>();
detail.put("plan", "nightly");
detail.put("slaHours", "26");
@@ -87,60 +93,90 @@ class AlertsQBitAlertSinkTest
sink.resolve("backups.plan.missed", "backups.plan.missed:nightly");
sink.heartbeat("backups.plan:nightly", 93600, 0);
assertThat(QAlerts.RAISED).hasSize(1);
var signal = QAlerts.RAISED.get(0);
assertThat(signal.alertTypeName).isEqualTo("backups.plan.missed");
assertThat(signal.dedupKey).isEqualTo("backups.plan.missed:nightly");
assertThat(signal.severity.name()).isEqualTo("CRITICAL");
assertThat(signal.title).isEqualTo("A backup did not run in time");
assertThat(signal.summary).isEqualTo("plan=nightly, slaHours=26");
assertThat(signal.attributes).containsEntry("plan", "nightly").containsEntry("slaHours", "26");
assertThat(QAlerts.RESOLVED).containsExactly("backups.plan.missed|backups.plan.missed:nightly|Cleared by the Backups qbit");
assertThat(QAlerts.HEARTBEATS).containsExactly("backups.plan:nightly");
assertThat(RecordingSink.raised).hasSize(1);
var signal = RecordingSink.raised.get(0);
assertThat(signal.getAlertTypeName()).isEqualTo("backups.plan.missed");
assertThat(signal.getDedupKey()).isEqualTo("backups.plan.missed:nightly");
assertThat(signal.getSeverity().name()).isEqualTo("CRITICAL");
assertThat(signal.getTitle()).isEqualTo("A backup did not run in time");
assertThat(signal.getSummary()).isEqualTo("plan=nightly, slaHours=26");
assertThat(signal.getAttributes()).containsEntry("plan", "nightly").containsEntry("slaHours", "26");
assertThat(RecordingSink.resolved).singleElement().satisfies(resolution ->
{
assertThat(resolution.getAlertTypeName()).isEqualTo("backups.plan.missed");
assertThat(resolution.getDedupKey()).isEqualTo("backups.plan.missed:nightly");
assertThat(resolution.getNote()).isEqualTo("Cleared by the Backups qbit");
});
assertThat(RecordingSink.heartbeats).containsExactly("backups.plan:nightly");
sink.raise("backups.storage.debris", "k", BackupAlertSink.Severity.INFO, "no detail", null);
assertThat(QAlerts.RAISED.get(1).summary).isNull();
assertThat(RecordingSink.raised.get(1).getSummary()).isNull();
}
/*******************************************************************************
** Without the Alerts qbit installed (no alerts meta-data on the instance) every
** Without a configured sink, even with alert type metadata present, every
** call is a no-op.
*******************************************************************************/
@Test
void testNoOpWithoutTheAlertsQBit()
{
AlertsInstanceMetaData.installed = false;
AlertsQBitAlertSink sink = new AlertsQBitAlertSink(FAKE_API);
sink.raise("t", "k", BackupAlertSink.Severity.INFO, "x", Map.of());
sink.resolve("t", "k");
sink.heartbeat("m", 1, 1);
assertThat(QAlerts.RAISED).isEmpty();
assertThat(QAlerts.RESOLVED).isEmpty();
assertThat(QAlerts.HEARTBEATS).isEmpty();
AlertsInstanceMetaData.of(QContext.getQInstance()).withSink(null);
AlertsQBitAlertSink sink = new AlertsQBitAlertSink();
long failures = QAlerts.getProducerFailureCount();
sink.raise(null, "k", BackupAlertSink.Severity.INFO, "x", Map.of());
sink.resolve(null, "k");
sink.heartbeat(null, 1, 1);
assertThat(QAlerts.getProducerFailureCount()).isEqualTo(failures);
assertThat(RecordingSink.raised).isEmpty();
assertThat(RecordingSink.resolved).isEmpty();
assertThat(RecordingSink.heartbeats).isEmpty();
}
/*******************************************************************************
** A broken or missing SPI never throws into the backup.
** The same adapter follows the current instance and a sink installed later.
*******************************************************************************/
@Test
void testLateInstallationAndInstanceSwitch()
{
var original = QContext.getQInstance();
QContext.setQInstance(new QInstance());
AlertsQBitAlertSink sink = new AlertsQBitAlertSink();
assertThat(sink.isAlertsQBitInstalled()).isFalse();
sink.raise("backups.plan.missed", "k", BackupAlertSink.Severity.INFO, "x", Map.of());
assertThat(RecordingSink.raised).isEmpty();
sink.registerAlertTypes(QContext.getQInstance());
assertThat(sink.isAlertsQBitInstalled()).isFalse();
AlertsInstanceMetaData.of(QContext.getQInstance()).withSink(new QCodeReference(RecordingSink.class));
sink.raise("backups.plan.missed", "k", BackupAlertSink.Severity.INFO, "x", Map.of());
assertThat(RecordingSink.raised).hasSize(1);
assertThat(RecordingSink.raised.get(0).getSummary()).isNull();
QContext.setQInstance(original);
assertThat(sink.isAlertsQBitInstalled()).isTrue();
}
/*******************************************************************************
** Sink and adapter failures never throw into the backup.
*******************************************************************************/
@Test
void testFailuresAreSwallowed()
{
AlertsQBitAlertSink missing = new AlertsQBitAlertSink("no.such.package");
missing.raise("t", "k", BackupAlertSink.Severity.INFO, "x", Map.of());
missing.resolve("t", "k");
missing.heartbeat("m", 1, 1);
assertThat(missing.registerAlertTypes(QContext.getQInstance())).isZero();
QAlerts.failWith = new IllegalStateException("alerts down");
AlertsQBitAlertSink broken = new AlertsQBitAlertSink(FAKE_API);
long failures = QAlerts.getProducerFailureCount();
RecordingSink.failWith = new IllegalStateException("alerts down");
AlertsQBitAlertSink broken = new AlertsQBitAlertSink();
broken.raise("t", "k", BackupAlertSink.Severity.INFO, "x", Map.of());
broken.resolve("t", "k");
broken.heartbeat("m", 1, 1);
assertThat(QAlerts.RAISED).isEmpty();
assertThat(RecordingSink.raised).isEmpty();
assertThat(QAlerts.getProducerFailureCount()).isEqualTo(failures + 3);
broken.raise("t", "k", null, "bad severity", Map.of());
broken.raise("t", "k", null, "bad severity again", Map.of());
assertThat(broken.registerAlertTypes(null)).isZero();
}
@@ -149,41 +185,41 @@ class AlertsQBitAlertSinkTest
** The backup alert types are declared with the Alerts qbit.
*******************************************************************************/
@Test
void testRegisterAlertTypes()
void testRegisterAlertTypes() throws Exception
{
int added = new AlertsQBitAlertSink(FAKE_API).registerAlertTypes(QContext.getQInstance());
QInstance instance = new QInstance();
AlertsQBitAlertSink sink = new AlertsQBitAlertSink();
int added = sink.registerAlertTypes(instance);
var types = new ArrayList<>(AlertsInstanceMetaData.of(instance).getAlertTypes().values());
assertThat(added).isEqualTo(BackupAlerts.alertTypes().size()).isEqualTo(9);
assertThat(QAlertTypeMetaData.ADDED).extracting(t -> t.name).containsExactly(
"backups.plan.missed", "backups.run.failed", "backups.run.missing_in_storage", "backups.files.missing", "backups.verify.failed",
"backups.verify.overdue", "backups.plan.owner_invalid", "backups.storage.debris", "backups.restore.completed");
assertThat(QAlertTypeMetaData.ADDED).allSatisfy(t ->
assertThat(types).extracting(t -> t.getName()).containsExactly(
"backups.plan.missed", "backups.run.failed", "backups.run.missing-in-storage", "backups.files.missing", "backups.verify.failed",
"backups.verify.overdue", "backups.plan.owner-invalid", "backups.storage.debris", "backups.restore.completed");
assertThat(types).allSatisfy(t ->
{
assertThat(t.groupName).isEqualTo("Backups");
assertThat(t.howToFix).isNotBlank();
assertThat(t.label).isNotBlank();
assertThat(t.getGroupName()).isEqualTo("Backups");
assertThat(t.getHowToFix()).isNotBlank();
assertThat(t.getLabel()).isNotBlank();
});
assertThat(QAlertTypeMetaData.ADDED.get(2).defaultSeverity.name()).isEqualTo("CRITICAL");
assertThat(types.get(2).getDefaultSeverity().name()).isEqualTo("CRITICAL");
assertThat(sink.registerAlertTypes(instance)).isZero();
new QInstanceValidator().validate(QContext.getQInstance());
}
/*******************************************************************************
** Detection: the SPI is present or not by class name; the real SPI is not in
** RC.4, so nothing is bound by default there.
** Default binding preserves a host-provided backup sink.
*******************************************************************************/
@Test
void testDetectionAndBinding()
{
assertThat(AlertsQBitAlertSink.isPresent(FAKE_API)).isTrue();
assertThat(AlertsQBitAlertSink.isPresent("no.such.package")).isFalse();
boolean realSpiPresent = AlertsQBitAlertSink.isPresent(AlertsQBitAlertSink.DEFAULT_API_PACKAGE);
assertThat(AlertsQBitAlertSink.bindIfPresent()).isEqualTo(realSpiPresent);
if(realSpiPresent)
{
assertThat(BackupsRuntime.getAlertSink()).isInstanceOf(AlertsQBitAlertSink.class);
assertThat(AlertsQBitAlertSink.bindIfPresent()).as("an already bound sink is not replaced").isFalse();
}
assertThat(AlertsQBitAlertSink.bindIfAbsent()).isFalse();
assertThat(BackupsRuntime.getAlertSink()).isSameAs(VerifyTestSupport.sink);
BackupsRuntime.setAlertSink(null);
assertThat(AlertsQBitAlertSink.bindIfAbsent()).isTrue();
assertThat(BackupsRuntime.getAlertSink()).isInstanceOf(AlertsQBitAlertSink.class);
assertThat(AlertsQBitAlertSink.bindIfAbsent()).isFalse();
}
@@ -194,22 +230,80 @@ class AlertsQBitAlertSinkTest
@Test
void testSlaAlertsThroughTheAdapter() throws Exception
{
BackupsRuntime.setAlertSink(new AlertsQBitAlertSink(FAKE_API));
BackupsRuntime.setAlertSink(new AlertsQBitAlertSink());
QRecord plan = VerifyTestSupport.plan(Map.of());
VerifyTestSupport.backup(plan);
assertThat(QAlerts.HEARTBEATS).contains("backups.plan:nightly");
assertThat(RecordingSink.heartbeats).contains("backups.plan:nightly");
VerifyTestSupport.clock.advance(Duration.ofHours(30));
new SlaChecker().check(VerifyTestSupport.reload(plan));
assertThat(QAlerts.RAISED).anySatisfy(s ->
assertThat(RecordingSink.raised).anySatisfy(s ->
{
assertThat(s.alertTypeName).isEqualTo("backups.plan.missed");
assertThat(s.dedupKey).isEqualTo("backups.plan.missed:nightly");
assertThat(s.severity.name()).isEqualTo("ERROR");
assertThat(s.getAlertTypeName()).isEqualTo("backups.plan.missed");
assertThat(s.getDedupKey()).isEqualTo("backups.plan.missed:nightly");
assertThat(s.getSeverity().name()).isEqualTo("ERROR");
});
VerifyTestSupport.backup(plan);
new SlaChecker().check(VerifyTestSupport.reload(plan));
assertThat(QAlerts.RESOLVED).anyMatch(r -> r.startsWith("backups.plan.missed|backups.plan.missed:nightly|"));
assertThat(RecordingSink.resolved).anyMatch(r -> r.getAlertTypeName().equals("backups.plan.missed") && r.getDedupKey().equals("backups.plan.missed:nightly"));
}
/*******************************************************************************
** Sink loaded by the real QCodeLoader through AlertsInstanceMetaData.
*******************************************************************************/
public static class RecordingSink implements AlertSinkInterface
{
static final List<AlertSignal> raised = new ArrayList<>();
static final List<AlertResolution> resolved = new ArrayList<>();
static final List<String> heartbeats = new ArrayList<>();
static RuntimeException failWith;
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public void raise(AlertSignal signal)
{
if(failWith != null)
{
throw (failWith);
}
raised.add(signal);
}
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public void resolve(AlertResolution resolution)
{
if(failWith != null)
{
throw (failWith);
}
resolved.add(resolution);
}
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public void heartbeat(String heartbeatKey, Instant at)
{
if(failWith != null)
{
throw (failWith);
}
heartbeats.add(heartbeatKey);
}
}
}
@@ -70,8 +70,8 @@ class BackupAlertsTest
void testTheSpecTable()
{
Set<String> names = BackupAlerts.alertTypes().stream().map(BackupAlerts.AlertType::name).collect(Collectors.toSet());
assertThat(names).containsExactlyInAnyOrder("backups.plan.missed", "backups.run.failed", "backups.run.missing_in_storage", "backups.files.missing",
"backups.verify.failed", "backups.verify.overdue", "backups.plan.owner_invalid", "backups.storage.debris", "backups.restore.completed");
assertThat(names).containsExactlyInAnyOrder("backups.plan.missed", "backups.run.failed", "backups.run.missing-in-storage", "backups.files.missing",
"backups.verify.failed", "backups.verify.overdue", "backups.plan.owner-invalid", "backups.storage.debris", "backups.restore.completed");
assertThat(BackupAlerts.alertTypes()).allSatisfy(t -> assertThat(t.howToFix()).isNotBlank());
assertThat(BackupAlerts.alertTypes().stream().filter(t -> t.name().equals(BackupAlerts.RUN_MISSING)).findFirst().orElseThrow().defaultSeverity())
.isEqualTo(BackupAlertSink.Severity.CRITICAL);
@@ -1,30 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.alerts.fakeapi;
/*******************************************************************************
** Test double of the Alerts producer SPI (QRun-IO/qqq PR #1108): the same shape as
** com.kingsrook.qqq.backend.core.alerts.AlertSeverity.
*******************************************************************************/
public enum AlertSeverity
{
INFO,
WARNING,
ERROR,
CRITICAL
}
@@ -1,96 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.alerts.fakeapi;
import java.util.LinkedHashMap;
import java.util.Map;
/*******************************************************************************
** Test double of the Alerts producer SPI AlertSignal (only what the adapter uses).
*******************************************************************************/
public class AlertSignal
{
public final String alertTypeName;
public String dedupKey;
public AlertSeverity severity;
public String title;
public String summary;
public final Map<String, String> attributes = new LinkedHashMap<>();
/*******************************************************************************
** Constructor.
*******************************************************************************/
public AlertSignal(String alertTypeName)
{
this.alertTypeName = alertTypeName;
}
/*******************************************************************************
** Setter.
*******************************************************************************/
public void setDedupKey(String dedupKey)
{
this.dedupKey = dedupKey;
}
/*******************************************************************************
** Setter.
*******************************************************************************/
public void setSeverity(AlertSeverity severity)
{
this.severity = severity;
}
/*******************************************************************************
** Setter.
*******************************************************************************/
public void setTitle(String title)
{
this.title = title;
}
/*******************************************************************************
** Setter.
*******************************************************************************/
public void setSummary(String summary)
{
this.summary = summary;
}
/*******************************************************************************
** Fluent attribute.
*******************************************************************************/
public AlertSignal withAttribute(String key, String value)
{
attributes.put(key, value);
return (this);
}
}
@@ -1,40 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.alerts.fakeapi;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
/*******************************************************************************
** Test double of the Alerts producer SPI AlertsInstanceMetaData: of(instance) is
** non-null only when a test says the Alerts qbit is installed.
*******************************************************************************/
public class AlertsInstanceMetaData
{
public static boolean installed = false;
/*******************************************************************************
** The alerts meta-data of an instance, or null.
*******************************************************************************/
public static AlertsInstanceMetaData of(QInstance qInstance)
{
return (installed ? new AlertsInstanceMetaData() : null);
}
}
@@ -1,103 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.alerts.fakeapi;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
/*******************************************************************************
** Test double of the Alerts producer SPI QAlertTypeMetaData: records the types
** added to an instance.
*******************************************************************************/
public class QAlertTypeMetaData
{
public static final List<QAlertTypeMetaData> ADDED = new ArrayList<>();
public String name;
public String label;
public String groupName;
public String howToFix;
public AlertSeverity defaultSeverity;
/*******************************************************************************
** Fluent.
*******************************************************************************/
public QAlertTypeMetaData withName(String name)
{
this.name = name;
return (this);
}
/*******************************************************************************
** Fluent.
*******************************************************************************/
public QAlertTypeMetaData withLabel(String label)
{
this.label = label;
return (this);
}
/*******************************************************************************
** Fluent.
*******************************************************************************/
public QAlertTypeMetaData withGroupName(String groupName)
{
this.groupName = groupName;
return (this);
}
/*******************************************************************************
** Fluent.
*******************************************************************************/
public QAlertTypeMetaData withHowToFix(String howToFix)
{
this.howToFix = howToFix;
return (this);
}
/*******************************************************************************
** Fluent.
*******************************************************************************/
public QAlertTypeMetaData withDefaultSeverity(AlertSeverity defaultSeverity)
{
this.defaultSeverity = defaultSeverity;
return (this);
}
/*******************************************************************************
** Add to the instance (recorded).
*******************************************************************************/
public void addSelfToInstance(QInstance qInstance)
{
ADDED.add(this);
}
}
@@ -1,104 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.backups.alerts.fakeapi;
import java.util.ArrayList;
import java.util.List;
/*******************************************************************************
** Test double of the Alerts producer SPI QAlerts: static calls that record what
** they were given.
*******************************************************************************/
public final class QAlerts
{
public static final List<AlertSignal> RAISED = new ArrayList<>();
public static final List<String> RESOLVED = new ArrayList<>();
public static final List<String> HEARTBEATS = new ArrayList<>();
/** When set, every call throws it (the SPI is meant not to, but the adapter must survive). */
public static RuntimeException failWith;
/*******************************************************************************
** Utility class.
*******************************************************************************/
private QAlerts()
{
}
/*******************************************************************************
** Forget everything.
*******************************************************************************/
public static void reset()
{
RAISED.clear();
RESOLVED.clear();
HEARTBEATS.clear();
failWith = null;
}
/*******************************************************************************
** Raise.
*******************************************************************************/
public static void raise(AlertSignal signal)
{
maybeFail();
RAISED.add(signal);
}
/*******************************************************************************
** Resolve.
*******************************************************************************/
public static void resolve(String alertTypeName, String dedupKey, String note)
{
maybeFail();
RESOLVED.add(alertTypeName + "|" + dedupKey + "|" + note);
}
/*******************************************************************************
** Heartbeat.
*******************************************************************************/
public static void heartbeat(String heartbeatKey)
{
maybeFail();
HEARTBEATS.add(heartbeatKey);
}
/*******************************************************************************
**
*******************************************************************************/
private static void maybeFail()
{
if(failWith != null)
{
throw (failWith);
}
}
}
@@ -306,7 +306,7 @@ class BackupPlanScheduleTest
assertThat(catalog.where(BackupsTables.RUN, "backupPlanId", planId)).isEmpty();
assertThat(BackupsTestUtils.BACKUPS.list("")).isEmpty();
assertThat(catalog.get(BackupsTables.PLAN, planId).orElseThrow().getValueString("slaStatus")).isEqualTo("OWNER_INVALID");
assertThat(alerts.raised).containsExactly("backups.plan.owner_invalid|backups.plan.owner_invalid:nightly");
assertThat(alerts.raised).containsExactly("backups.plan.owner-invalid|backups.plan.owner-invalid:nightly");
BackupScheduledRunProcess.setOwnerHasAdminCheck(null);
Integer noOwner = insertPlan(Map.of("name", "Ownerless", "planKey", "ownerless", "isActive", true));
@@ -180,12 +180,12 @@ class SlaCheckerTest
clock.advance(Duration.ofMinutes(55));
assertThat(check(plan).status()).isEqualTo(SlaChecker.MISSING_IN_STORAGE);
BackupAlertsAssert.assertOpen("backups.run.missing_in_storage:nightly", BackupAlertSink.Severity.CRITICAL);
BackupAlertsAssert.assertOpen("backups.run.missing-in-storage:nightly", BackupAlertSink.Severity.CRITICAL);
assertThat(check(plan).status()).as("a negative answer is never cached").isEqualTo(SlaChecker.MISSING_IN_STORAGE);
BackupsTestUtils.BACKUPS.putBytes(result.getManifestKey(), bytes);
assertThat(check(plan).status()).isEqualTo(SlaChecker.OK);
assertThat(sink.isOpen("backups.run.missing_in_storage:nightly")).isFalse();
assertThat(sink.isOpen("backups.run.missing-in-storage:nightly")).isFalse();
}
@@ -260,11 +260,11 @@ class SlaCheckerTest
BackupsRuntime.getConfig().withPlanOwnerValidator(owner -> false);
assertThat(check(plan).status()).isEqualTo(SlaChecker.OWNER_INVALID);
BackupAlertsAssert.assertOpen("backups.plan.owner_invalid:nightly", BackupAlertSink.Severity.WARNING);
BackupAlertsAssert.assertOpen("backups.plan.owner-invalid:nightly", BackupAlertSink.Severity.WARNING);
BackupsRuntime.getConfig().withPlanOwnerValidator(owner -> "bob".equals(owner));
assertThat(check(plan).status()).isEqualTo(SlaChecker.OK);
assertThat(sink.isOpen("backups.plan.owner_invalid:nightly")).isFalse();
assertThat(sink.isOpen("backups.plan.owner-invalid:nightly")).isFalse();
BackupsRuntime.getConfig().withPlanOwnerValidator(owner ->
{
@@ -287,7 +287,7 @@ class SlaCheckerTest
clock.advance(Duration.ofDays(2));
assertThat(check(plan).status()).isEqualTo(SlaChecker.MISSING_IN_STORAGE);
assertThat(sink.isOpen("backups.plan.missed:nightly")).as("the alerts are independent of the status shown").isTrue();
assertThat(sink.isOpen("backups.run.missing_in_storage:nightly")).isTrue();
assertThat(sink.isOpen("backups.run.missing-in-storage:nightly")).isTrue();
}