feat(verify): backup verification, SLA checker, alerts, health, events #5
No files matched your search
@@ -72,6 +72,20 @@
|
||||
<artifactId>qqq-backend-core</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- optional: the BackupsHealthIndicator (apps without qqq-middleware-health never load it) -->
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
<artifactId>qqq-middleware-health</artifactId>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
|
||||
<!-- optional: ESB events (apps without qqq-esb never load it) -->
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
<artifactId>qqq-esb</artifactId>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter</artifactId>
|
||||
|
||||
@@ -19,6 +19,9 @@ package com.kingsrook.qbits.backups;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.Predicate;
|
||||
import com.kingsrook.qbits.backups.codec.ArtifactTransform;
|
||||
import com.kingsrook.qbits.backups.verify.ScratchTarget;
|
||||
import com.kingsrook.qbits.backups.storage.BackupStorage;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig;
|
||||
@@ -52,6 +55,12 @@ public class BackupsQBitConfig implements QBitConfig
|
||||
|
||||
private Function<String, BackupStorage> storageResolver;
|
||||
|
||||
private Integer fileReverifyDays = 30;
|
||||
private Function<String, ArtifactTransform> artifactTransformResolver;
|
||||
private ScratchTarget scratchTarget;
|
||||
private Predicate<String> planOwnerValidator;
|
||||
private String esbDestinationName;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@@ -315,4 +324,109 @@ public class BackupsQBitConfig implements QBitConfig
|
||||
return (this);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for fileReverifyDays: CONTENT verification skips file-store objects
|
||||
** verified more recently than this.
|
||||
*******************************************************************************/
|
||||
public Integer getFileReverifyDays()
|
||||
{
|
||||
return (fileReverifyDays);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for fileReverifyDays
|
||||
*******************************************************************************/
|
||||
public BackupsQBitConfig withFileReverifyDays(Integer fileReverifyDays)
|
||||
{
|
||||
this.fileReverifyDays = fileReverifyDays;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for artifactTransformResolver (the decrypt hook for verify)
|
||||
*******************************************************************************/
|
||||
public Function<String, ArtifactTransform> getArtifactTransformResolver()
|
||||
{
|
||||
return (artifactTransformResolver);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for artifactTransformResolver: encryption key reference (the name
|
||||
** of a key in a key provider) to the transform that decrypts what that key wrote.
|
||||
*******************************************************************************/
|
||||
public BackupsQBitConfig withArtifactTransformResolver(Function<String, ArtifactTransform> artifactTransformResolver)
|
||||
{
|
||||
this.artifactTransformResolver = artifactTransformResolver;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for scratchTarget (where TEST_RESTORE restores to; null means MEMORY)
|
||||
*******************************************************************************/
|
||||
public ScratchTarget getScratchTarget()
|
||||
{
|
||||
return (scratchTarget);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for scratchTarget
|
||||
*******************************************************************************/
|
||||
public BackupsQBitConfig withScratchTarget(ScratchTarget scratchTarget)
|
||||
{
|
||||
this.scratchTarget = scratchTarget;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for planOwnerValidator (true when the owner still exists and holds backups.admin)
|
||||
*******************************************************************************/
|
||||
public Predicate<String> getPlanOwnerValidator()
|
||||
{
|
||||
return (planOwnerValidator);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for planOwnerValidator: with none, plan owners are not checked.
|
||||
*******************************************************************************/
|
||||
public BackupsQBitConfig withPlanOwnerValidator(Predicate<String> planOwnerValidator)
|
||||
{
|
||||
this.planOwnerValidator = planOwnerValidator;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for esbDestinationName (ESB events are published here, when set)
|
||||
*******************************************************************************/
|
||||
public String getEsbDestinationName()
|
||||
{
|
||||
return (esbDestinationName);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for esbDestinationName
|
||||
*******************************************************************************/
|
||||
public BackupsQBitConfig withEsbDestinationName(String esbDestinationName)
|
||||
{
|
||||
this.esbDestinationName = esbDestinationName;
|
||||
return (this);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -119,6 +119,14 @@ public class BackupsQBitProducer implements QBitMetaDataProducer<BackupsQBitConf
|
||||
qInstance.addQBit(qBit);
|
||||
BackupsRuntime.install(qBitConfig);
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// bind the Alerts qbit producer SPI when it is on the class path (optional) //
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
if(com.kingsrook.qbits.backups.alerts.AlertsQBitAlertSink.bindIfPresent())
|
||||
{
|
||||
new com.kingsrook.qbits.backups.alerts.AlertsQBitAlertSink().registerAlertTypes(qInstance);
|
||||
}
|
||||
|
||||
com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput output = new com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput();
|
||||
for(var possibleValueSource : com.kingsrook.qbits.backups.model.BackupsTables.possibleValueSources())
|
||||
{
|
||||
|
||||
@@ -0,0 +1,310 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.alerts;
|
||||
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
import com.kingsrook.qbits.backups.BackupsRuntime;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The BackupAlertSink adapter for the Alerts qbit producer SPI (QRun-IO/qqq
|
||||
** PR #1108, com.kingsrook.qqq.backend.core.alerts: QAlerts, AlertSignal,
|
||||
** AlertResolution, AlertSeverity, QAlertTypeMetaData; design spec
|
||||
** docs/specs/2026-10-12-qbit-alerts-design.md).
|
||||
**
|
||||
** STAND-IN: the SPI is an open core PR and is not in QQQ 4.1.0-RC.4, so this
|
||||
** adapter is bound REFLECTIVELY and optionally. It compiles against nothing in
|
||||
** the SPI; it looks the classes up by name, and every call is guarded:
|
||||
** <ul>
|
||||
** <li>no SPI classes on the class path: {@link #bindIfPresent()} binds nothing
|
||||
** and the sink stays the no-op;</li>
|
||||
** <li>the SPI is present but the instance has no alerts meta-data (the Alerts
|
||||
** qbit is not installed): every call is a no-op;</li>
|
||||
** <li>any reflective or SPI failure is logged once and swallowed, because an
|
||||
** alerting problem must never fail a backup.</li>
|
||||
** </ul>
|
||||
** Replace with a direct binding (QAlerts.raise and friends) when RC.5 ships.
|
||||
**
|
||||
** Mapping: raise to QAlerts.raise(AlertSignal), resolve to QAlerts.resolve(type,
|
||||
** dedupKey, note), heartbeat to QAlerts.heartbeat(key). The SPI has no per-plan
|
||||
** heartbeat registration (monitors are declared on the alert type), so the
|
||||
** expected interval and grace are not passed on; the SLA checker raises
|
||||
** backups.plan.missed itself, independent of any heartbeat (spec 10.4).
|
||||
*******************************************************************************/
|
||||
public class AlertsQBitAlertSink implements BackupAlertSink
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(AlertsQBitAlertSink.class);
|
||||
|
||||
/** The package of the Alerts producer SPI in qqq-backend-core. */
|
||||
public static final String DEFAULT_API_PACKAGE = "com.kingsrook.qqq.backend.core.alerts";
|
||||
|
||||
private final String apiPackage;
|
||||
|
||||
private boolean warned = false;
|
||||
|
||||
private Class<?> qAlerts;
|
||||
private Class<?> alertSignal;
|
||||
private Class<?> alertSeverity;
|
||||
private Class<?> instanceMetaData;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor for the real SPI package.
|
||||
*******************************************************************************/
|
||||
public AlertsQBitAlertSink()
|
||||
{
|
||||
this(DEFAULT_API_PACKAGE);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor for another package (a test double of the SPI shape).
|
||||
*******************************************************************************/
|
||||
public AlertsQBitAlertSink(String apiPackage)
|
||||
{
|
||||
this.apiPackage = apiPackage;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether the SPI classes are on the class path.
|
||||
*******************************************************************************/
|
||||
public static boolean isPresent(String apiPackage)
|
||||
{
|
||||
try
|
||||
{
|
||||
Class.forName(apiPackage + ".QAlerts");
|
||||
Class.forName(apiPackage + ".AlertSignal");
|
||||
return (true);
|
||||
}
|
||||
catch(ClassNotFoundException | LinkageError e)
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Bind this adapter as the runtime's alert sink when the SPI is present and no
|
||||
** other sink was bound already. Returns whether it bound.
|
||||
*******************************************************************************/
|
||||
public static boolean bindIfPresent()
|
||||
{
|
||||
if(!(BackupsRuntime.getAlertSink() instanceof NoOpBackupAlertSink) || !isPresent(DEFAULT_API_PACKAGE))
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
BackupsRuntime.setAlertSink(new AlertsQBitAlertSink());
|
||||
return (true);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Declare the backup alert types with the Alerts qbit (QAlertTypeMetaData), when
|
||||
** the SPI is present. Returns how many were added.
|
||||
*******************************************************************************/
|
||||
public int registerAlertTypes(QInstance qInstance)
|
||||
{
|
||||
int added = 0;
|
||||
try
|
||||
{
|
||||
Class<?> typeClass = Class.forName(apiPackage + ".QAlertTypeMetaData");
|
||||
load();
|
||||
for(BackupAlerts.AlertType alertType : BackupAlerts.alertTypes())
|
||||
{
|
||||
try
|
||||
{
|
||||
Object type = typeClass.getConstructor().newInstance();
|
||||
typeClass.getMethod("withName", String.class).invoke(type, alertType.name());
|
||||
typeClass.getMethod("withLabel", String.class).invoke(type, alertType.label());
|
||||
typeClass.getMethod("withGroupName", String.class).invoke(type, "Backups");
|
||||
typeClass.getMethod("withHowToFix", String.class).invoke(type, alertType.howToFix());
|
||||
typeClass.getMethod("withDefaultSeverity", alertSeverity).invoke(type, severity(alertType.defaultSeverity()));
|
||||
typeClass.getMethod("addSelfToInstance", QInstance.class).invoke(type, qInstance);
|
||||
added++;
|
||||
}
|
||||
catch(ReflectiveOperationException | RuntimeException e)
|
||||
{
|
||||
warnOnce("Could not declare the alert type " + alertType.name(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch(ReflectiveOperationException | LinkageError e)
|
||||
{
|
||||
warnOnce("The alerts SPI is not available", e);
|
||||
}
|
||||
return (added);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void raise(String type, String dedupeKey, Severity severity, String title, Map<String, String> detail)
|
||||
{
|
||||
if(!isAlertsQBitInstalled())
|
||||
{
|
||||
return;
|
||||
}
|
||||
try
|
||||
{
|
||||
Object signal = alertSignal.getConstructor(String.class).newInstance(type);
|
||||
alertSignal.getMethod("setDedupKey", String.class).invoke(signal, dedupeKey);
|
||||
alertSignal.getMethod("setSeverity", alertSeverity).invoke(signal, severity(severity));
|
||||
alertSignal.getMethod("setTitle", String.class).invoke(signal, title);
|
||||
if(detail != null && !detail.isEmpty())
|
||||
{
|
||||
alertSignal.getMethod("setSummary", String.class).invoke(signal, detail.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(", ")));
|
||||
Method withAttribute = alertSignal.getMethod("withAttribute", String.class, String.class);
|
||||
for(Map.Entry<String, String> entry : detail.entrySet())
|
||||
{
|
||||
withAttribute.invoke(signal, entry.getKey(), entry.getValue());
|
||||
}
|
||||
}
|
||||
qAlerts.getMethod("raise", alertSignal).invoke(null, signal);
|
||||
}
|
||||
catch(ReflectiveOperationException | RuntimeException e)
|
||||
{
|
||||
warnOnce("Could not raise an alert", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void resolve(String type, String dedupeKey)
|
||||
{
|
||||
if(!isAlertsQBitInstalled())
|
||||
{
|
||||
return;
|
||||
}
|
||||
try
|
||||
{
|
||||
qAlerts.getMethod("resolve", String.class, String.class, String.class).invoke(null, type, dedupeKey, "Cleared by the Backups qbit");
|
||||
}
|
||||
catch(ReflectiveOperationException | RuntimeException e)
|
||||
{
|
||||
warnOnce("Could not resolve an alert", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void heartbeat(String monitorKey, long expectedEverySeconds, long graceSeconds)
|
||||
{
|
||||
if(!isAlertsQBitInstalled())
|
||||
{
|
||||
return;
|
||||
}
|
||||
try
|
||||
{
|
||||
qAlerts.getMethod("heartbeat", String.class).invoke(null, monitorKey);
|
||||
}
|
||||
catch(ReflectiveOperationException | RuntimeException e)
|
||||
{
|
||||
warnOnce("Could not check in a heartbeat", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The SPI classes load, and the instance has alerts meta-data (the Alerts qbit
|
||||
** is installed). Checked on every call so the order qbits are produced in
|
||||
** does not matter.
|
||||
*******************************************************************************/
|
||||
boolean isAlertsQBitInstalled()
|
||||
{
|
||||
try
|
||||
{
|
||||
load();
|
||||
QInstance qInstance = QContext.getQInstance();
|
||||
if(qInstance == null)
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
return (instanceMetaData.getMethod("of", QInstance.class).invoke(null, qInstance) != null);
|
||||
}
|
||||
catch(ReflectiveOperationException | RuntimeException | LinkageError e)
|
||||
{
|
||||
warnOnce("The alerts SPI is not usable", e);
|
||||
return (false);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private synchronized void load() throws ClassNotFoundException
|
||||
{
|
||||
if(qAlerts == null)
|
||||
{
|
||||
qAlerts = Class.forName(apiPackage + ".QAlerts");
|
||||
alertSignal = Class.forName(apiPackage + ".AlertSignal");
|
||||
alertSeverity = Class.forName(apiPackage + ".AlertSeverity");
|
||||
instanceMetaData = Class.forName(apiPackage + ".AlertsInstanceMetaData");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Map our severity to the SPI's enum constant of the same name.
|
||||
*******************************************************************************/
|
||||
@SuppressWarnings({ "unchecked", "rawtypes" })
|
||||
private Object severity(Severity severity)
|
||||
{
|
||||
return (Enum.valueOf((Class<Enum>) alertSeverity, severity.name()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private void warnOnce(String message, Throwable t)
|
||||
{
|
||||
if(!warned)
|
||||
{
|
||||
warned = true;
|
||||
LOG.warn(message + " (the Alerts qbit adapter will keep failing quietly)", t, logPair("apiPackage", apiPackage));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.alerts;
|
||||
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.backups.BackupsRuntime;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every alert the Backups qbit raises (design spec section 10.4), with stable
|
||||
** dedupe keys, and helpers that send them through the bound BackupAlertSink.
|
||||
**
|
||||
** A dedupe key is the alert type plus a plan key (or a run key, or a plan key
|
||||
** and a verify level), so a repeated condition refreshes one alert and a cleared
|
||||
** condition resolves exactly that alert. Details carry counts and names only:
|
||||
** never row values, object keys with secrets in them, or presigned URLs.
|
||||
*******************************************************************************/
|
||||
public final class BackupAlerts
|
||||
{
|
||||
public static final String PLAN_MISSED = "backups.plan.missed";
|
||||
public static final String RUN_FAILED = "backups.run.failed";
|
||||
public static final String RUN_MISSING = "backups.run.missing_in_storage";
|
||||
public static final String FILES_MISSING = "backups.files.missing";
|
||||
public static final String VERIFY_FAILED = "backups.verify.failed";
|
||||
public static final String VERIFY_OVERDUE = "backups.verify.overdue";
|
||||
public static final String PLAN_OWNER_INVALID = "backups.plan.owner_invalid";
|
||||
public static final String STORAGE_DEBRIS = "backups.storage.debris";
|
||||
public static final String RESTORE_COMPLETED = "backups.restore.completed";
|
||||
|
||||
/** The prefix of the heartbeat monitor key of a plan. */
|
||||
public static final String HEARTBEAT_PREFIX = "backups.plan:";
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** A declared alert type, for the Alerts qbit's type registry.
|
||||
**
|
||||
** @param name the alert type name
|
||||
** @param label a short label
|
||||
** @param defaultSeverity the severity when a signal does not say
|
||||
** @param howToFix one sentence for the person looking at the alert
|
||||
***************************************************************************/
|
||||
public record AlertType(String name, String label, BackupAlertSink.Severity defaultSeverity, String howToFix)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private BackupAlerts()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The alert types of the spec table (section 10.4), in table order.
|
||||
*******************************************************************************/
|
||||
public static List<AlertType> alertTypes()
|
||||
{
|
||||
return (List.of(
|
||||
new AlertType(PLAN_MISSED, "Backup did not run", BackupAlertSink.Severity.ERROR,
|
||||
"Open the plan and its latest runs, check the schedule and the scheduler, then run the backup by hand."),
|
||||
new AlertType(RUN_FAILED, "Backup run failed", BackupAlertSink.Severity.WARNING,
|
||||
"Open the failed run for the error, fix the cause, and run the plan again."),
|
||||
new AlertType(RUN_MISSING, "Backup is missing from storage", BackupAlertSink.Severity.CRITICAL,
|
||||
"The catalog says the run succeeded but its manifest is gone from the bucket. Check the bucket, its lifecycle rules and the credentials, then run a new backup."),
|
||||
new AlertType(FILES_MISSING, "Backed up files were missing at the source", BackupAlertSink.Severity.WARNING,
|
||||
"Open the run's file index: records point at files that no longer exist."),
|
||||
new AlertType(VERIFY_FAILED, "Backup verification failed", BackupAlertSink.Severity.ERROR,
|
||||
"Open the verification for the mismatches. Treat the run as unusable and take a new backup."),
|
||||
new AlertType(VERIFY_OVERDUE, "Backup verification is overdue", BackupAlertSink.Severity.WARNING,
|
||||
"Run Verify backup on a recent run of the plan, or set a verify schedule on the plan."),
|
||||
new AlertType(PLAN_OWNER_INVALID, "Backup plan owner is not valid", BackupAlertSink.Severity.WARNING,
|
||||
"Set an owner who exists and holds backups.admin on the plan."),
|
||||
new AlertType(STORAGE_DEBRIS, "Backup storage has debris or orphans", BackupAlertSink.Severity.INFO,
|
||||
"Prune clears debris. Orphan prefixes can be imported into the catalog or deleted."),
|
||||
new AlertType(RESTORE_COMPLETED, "A restore finished", BackupAlertSink.Severity.INFO,
|
||||
"Nothing to fix unless the restore failed. Open the restore run.")));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The stable dedupe key for a type and an id (plan key, run key...).
|
||||
*******************************************************************************/
|
||||
public static String key(String type, String id)
|
||||
{
|
||||
return (type + ":" + id);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The dedupe key of a verify failure: it includes the level so a pass at a
|
||||
** higher level clears the lower failures, and a cheap pass never hides a
|
||||
** content failure.
|
||||
*******************************************************************************/
|
||||
public static String verifyKey(String planKey, String level)
|
||||
{
|
||||
return (VERIFY_FAILED + ":" + planKey + ":" + level);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Raise (or refresh) an alert. Never throws: an alerting problem must not fail
|
||||
** a backup or a check.
|
||||
*******************************************************************************/
|
||||
public static void raise(String type, String id, BackupAlertSink.Severity severity, String title, String... detailPairs)
|
||||
{
|
||||
try
|
||||
{
|
||||
Map<String, String> detail = new LinkedHashMap<>();
|
||||
for(int i = 0; i + 1 < detailPairs.length; i += 2)
|
||||
{
|
||||
detail.put(detailPairs[i], detailPairs[i + 1]);
|
||||
}
|
||||
BackupsRuntime.getAlertSink().raise(type, key(type, id), severity, title, detail);
|
||||
}
|
||||
catch(RuntimeException e)
|
||||
{
|
||||
// fire and forget
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Raise with an explicit dedupe key.
|
||||
*******************************************************************************/
|
||||
public static void raiseWithKey(String type, String dedupeKey, BackupAlertSink.Severity severity, String title, Map<String, String> detail)
|
||||
{
|
||||
try
|
||||
{
|
||||
BackupsRuntime.getAlertSink().raise(type, dedupeKey, severity, title, detail);
|
||||
}
|
||||
catch(RuntimeException e)
|
||||
{
|
||||
// fire and forget
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve the alert of a type and id. Never throws.
|
||||
*******************************************************************************/
|
||||
public static void resolve(String type, String id)
|
||||
{
|
||||
resolveKey(type, key(type, id));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve an alert by its dedupe key. Never throws.
|
||||
*******************************************************************************/
|
||||
public static void resolveKey(String type, String dedupeKey)
|
||||
{
|
||||
try
|
||||
{
|
||||
BackupsRuntime.getAlertSink().resolve(type, dedupeKey);
|
||||
}
|
||||
catch(RuntimeException e)
|
||||
{
|
||||
// fire and forget
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A restore finished: an info alert, so restores are never silent (error when
|
||||
** it failed). The Alerts qbit clears it after 7 days; nothing resolves it here.
|
||||
*******************************************************************************/
|
||||
public static void restoreCompleted(String restoreRunKey, String status, String planKey, long rowsRestored)
|
||||
{
|
||||
boolean failed = "FAILED".equals(status);
|
||||
raise(RESTORE_COMPLETED, restoreRunKey, failed ? BackupAlertSink.Severity.ERROR : BackupAlertSink.Severity.INFO,
|
||||
failed ? "A restore failed" : "A restore finished",
|
||||
"restoreRun", restoreRunKey, "status", String.valueOf(status), "plan", String.valueOf(planKey), "rows", String.valueOf(rowsRestored));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.events;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.function.BiConsumer;
|
||||
import com.kingsrook.qbits.backups.BackupsRuntime;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** ESB events of the Backups qbit (design spec section 10.4): qqq.backups.run.succeeded,
|
||||
** qqq.backups.run.failed and qqq.backups.restore.completed. They are for
|
||||
** integrations, not alerting, and are FIRE AND FORGET: a failed publish is logged
|
||||
** at most once and never fails a backup or a restore.
|
||||
**
|
||||
** Events go out only when BackupsQBitConfig.esbDestinationName is set and the ESB
|
||||
** (qqq-esb, an optional dependency) is on the class path; otherwise publish() does
|
||||
** nothing. A test, or an app with its own bus, can bind a publisher with
|
||||
** {@link #setPublisher}.
|
||||
*******************************************************************************/
|
||||
public final class BackupEvents
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(BackupEvents.class);
|
||||
|
||||
public static final String RUN_SUCCEEDED = "qqq.backups.run.succeeded";
|
||||
public static final String RUN_FAILED = "qqq.backups.run.failed";
|
||||
public static final String RESTORE_COMPLETED = "qqq.backups.restore.completed";
|
||||
|
||||
private static volatile BiConsumer<String, Map<String, Serializable>> publisher;
|
||||
private static volatile boolean warned = false;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private BackupEvents()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Bind a publisher (event type, data), or null to go back to the ESB default.
|
||||
*******************************************************************************/
|
||||
public static void setPublisher(BiConsumer<String, Map<String, Serializable>> newPublisher)
|
||||
{
|
||||
publisher = newPublisher;
|
||||
warned = false;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A run finished. Succeeded and warnings publish run.succeeded; failed, timed
|
||||
** out and abandoned publish run.failed; cancelled publishes nothing.
|
||||
*******************************************************************************/
|
||||
public static void runFinished(String planKey, String runKey, String status, long rows, long bytesStored)
|
||||
{
|
||||
String type;
|
||||
if("SUCCEEDED".equals(status) || "SUCCEEDED_WITH_WARNINGS".equals(status))
|
||||
{
|
||||
type = RUN_SUCCEEDED;
|
||||
}
|
||||
else if("CANCELLED".equals(status))
|
||||
{
|
||||
return;
|
||||
}
|
||||
else
|
||||
{
|
||||
type = RUN_FAILED;
|
||||
}
|
||||
Map<String, Serializable> data = new LinkedHashMap<>();
|
||||
data.put("plan", planKey);
|
||||
data.put("run", runKey);
|
||||
data.put("status", status);
|
||||
data.put("rows", rows);
|
||||
data.put("bytesStored", bytesStored);
|
||||
publish(type, data);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A restore finished (any status).
|
||||
*******************************************************************************/
|
||||
public static void restoreCompleted(String restoreRunKey, String status, long rowsRestored)
|
||||
{
|
||||
Map<String, Serializable> data = new LinkedHashMap<>();
|
||||
data.put("restoreRun", restoreRunKey);
|
||||
data.put("status", status);
|
||||
data.put("rows", rowsRestored);
|
||||
publish(RESTORE_COMPLETED, data);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Publish an event. Never throws.
|
||||
*******************************************************************************/
|
||||
public static void publish(String type, Map<String, Serializable> data)
|
||||
{
|
||||
try
|
||||
{
|
||||
BiConsumer<String, Map<String, Serializable>> bound = publisher;
|
||||
if(bound != null)
|
||||
{
|
||||
bound.accept(type, data);
|
||||
return;
|
||||
}
|
||||
|
||||
String destination = BackupsRuntime.getConfig() == null ? null : BackupsRuntime.getConfig().getEsbDestinationName();
|
||||
if(destination == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
EsbBackupEventPublisher.publish(destination, type, data);
|
||||
}
|
||||
catch(Exception | LinkageError e)
|
||||
{
|
||||
if(!warned)
|
||||
{
|
||||
warned = true;
|
||||
LOG.warn("A backup event could not be published (events are best effort)", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.events;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.esb.publish.EsbPublishAction;
|
||||
import com.kingsrook.qqq.esb.publish.EsbPublishInput;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Publishes a backup event on the QQQ ESB. This is the only class that touches
|
||||
** qqq-esb (an optional dependency); BackupEvents calls it inside a guard that
|
||||
** swallows a missing class (NoClassDefFoundError), so apps without the ESB never
|
||||
** notice.
|
||||
*******************************************************************************/
|
||||
final class EsbBackupEventPublisher
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private EsbBackupEventPublisher()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Publish one event to a destination.
|
||||
*******************************************************************************/
|
||||
static void publish(String destinationName, String type, Map<String, Serializable> data)
|
||||
{
|
||||
new EsbPublishAction().execute(new EsbPublishInput()
|
||||
.withDestinationName(destinationName)
|
||||
.withType(type)
|
||||
.withSourcePath("/backups")
|
||||
.withData(data));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.health;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.middleware.health.HealthIndicator;
|
||||
import com.kingsrook.qqq.middleware.health.model.metadata.HealthCheckResult;
|
||||
import com.kingsrook.qqq.middleware.health.model.metadata.HealthStatus;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The Backups health indicator (design spec section 10.4), for
|
||||
** qqq-middleware-health (an optional dependency: an app that does not have that
|
||||
** module never loads this class). It reports DOWN if any active plan marked
|
||||
** critical is LATE, FAILING or MISSING_IN_STORAGE, and UP with details otherwise.
|
||||
** It reads the slaStatus the SLA checker maintains; it never touches storage, so
|
||||
** a health call is cheap.
|
||||
**
|
||||
** It is OFF the liveness path by design. Add it to the indicators of the
|
||||
** instance's health check (the readiness and monitoring view), and do not point a
|
||||
** Kubernetes liveness probe at a check that includes it: a backup problem must
|
||||
** never restart pods.
|
||||
*******************************************************************************/
|
||||
public class BackupsHealthIndicator implements HealthIndicator
|
||||
{
|
||||
public static final String NAME = "backups";
|
||||
|
||||
private static final Set<String> CRITICAL_STATUSES = Set.of("LATE", "FAILING", "MISSING_IN_STORAGE");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getName()
|
||||
{
|
||||
return (NAME);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public HealthCheckResult check(QInstance qInstance) throws QException
|
||||
{
|
||||
long started = System.currentTimeMillis();
|
||||
boolean ownsContext = QContext.getQInstance() == null;
|
||||
if(ownsContext)
|
||||
{
|
||||
QContext.init(qInstance, new QSession());
|
||||
}
|
||||
try
|
||||
{
|
||||
List<QRecord> plans = new BackupCatalog().query(BackupsTables.PLAN, new QQueryFilter(new QFilterCriteria("isActive", QCriteriaOperator.EQUALS, true)));
|
||||
|
||||
List<String> criticalProblems = new ArrayList<>();
|
||||
Map<String, Integer> byStatus = new LinkedHashMap<>();
|
||||
for(QRecord plan : plans)
|
||||
{
|
||||
String status = plan.getValueString("slaStatus") == null ? "UNKNOWN" : plan.getValueString("slaStatus");
|
||||
byStatus.merge(status, 1, Integer::sum);
|
||||
if(Boolean.TRUE.equals(plan.getValueBoolean("critical")) && CRITICAL_STATUSES.contains(status))
|
||||
{
|
||||
criticalProblems.add(plan.getValueString("planKey") + ": " + status);
|
||||
}
|
||||
}
|
||||
|
||||
Map<String, Object> details = new LinkedHashMap<>();
|
||||
details.put("activePlans", plans.size());
|
||||
details.put("plansByStatus", byStatus);
|
||||
if(!criticalProblems.isEmpty())
|
||||
{
|
||||
details.put("criticalPlansWithProblems", criticalProblems);
|
||||
}
|
||||
return (new HealthCheckResult()
|
||||
.withStatus(criticalProblems.isEmpty() ? HealthStatus.UP : HealthStatus.DOWN)
|
||||
.withDurationMs(System.currentTimeMillis() - started)
|
||||
.withDetails(details));
|
||||
}
|
||||
finally
|
||||
{
|
||||
if(ownsContext)
|
||||
{
|
||||
QContext.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.metrics;
|
||||
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.backups.BackupsQBitConfig;
|
||||
import com.kingsrook.qbits.backups.BackupsRuntime;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The optional Prometheus text for operators who already run Alertmanager (design
|
||||
** spec section 10.4), computed from the tables and cached for 60 seconds. Gauges,
|
||||
** all labeled {instance_name, plan}:
|
||||
** <ul>
|
||||
** <li>qqq_backup_last_success_timestamp_seconds</li>
|
||||
** <li>qqq_backup_last_run_failed (1 when the latest finished run failed)</li>
|
||||
** <li>qqq_backup_sla_seconds</li>
|
||||
** <li>qqq_backup_last_verify_success_timestamp_seconds</li>
|
||||
** <li>qqq_backup_last_success_bytes</li>
|
||||
** </ul>
|
||||
** The rule worth having even alongside the Alerts qbit is
|
||||
** absent_over_time(qqq_backup_last_success_timestamp_seconds[30m]): the one signal
|
||||
** that still fires when the whole app, and so the Alerts qbit with it, is down.
|
||||
**
|
||||
** ENDPOINT STUB: this class renders the text. Serving it at GET /metrics/backups
|
||||
** (when BackupsQBitConfig.withPrometheusEndpoint(true)) is a route in
|
||||
** qqq-middleware-javalin, which this qbit does not depend on; the app's Javalin
|
||||
** route provider calls {@link #renderCached()}. {@link #isEnabled()} says whether
|
||||
** the config asks for the endpoint.
|
||||
*******************************************************************************/
|
||||
public final class BackupsMetrics
|
||||
{
|
||||
public static final String ENDPOINT_PATH = "/metrics/backups";
|
||||
|
||||
private static final Duration CACHE = Duration.ofSeconds(60);
|
||||
|
||||
private static String cachedText;
|
||||
private static Instant cachedAt;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private BackupsMetrics()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether the config asks for the endpoint.
|
||||
*******************************************************************************/
|
||||
public static boolean isEnabled()
|
||||
{
|
||||
BackupsQBitConfig config = BackupsRuntime.getConfig();
|
||||
return (config != null && Boolean.TRUE.equals(config.getPrometheusEndpoint()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Forget the cached text (tests).
|
||||
*******************************************************************************/
|
||||
public static synchronized void resetCache()
|
||||
{
|
||||
cachedText = null;
|
||||
cachedAt = null;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The text, from the cache when it is under 60 seconds old.
|
||||
*******************************************************************************/
|
||||
public static synchronized String renderCached() throws QException
|
||||
{
|
||||
Instant now = BackupsRuntime.getClock().instant();
|
||||
if(cachedText == null || cachedAt == null || cachedAt.plus(CACHE).isBefore(now))
|
||||
{
|
||||
cachedText = render();
|
||||
cachedAt = now;
|
||||
}
|
||||
return (cachedText);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Render the text format now.
|
||||
*******************************************************************************/
|
||||
public static String render() throws QException
|
||||
{
|
||||
BackupCatalog catalog = new BackupCatalog();
|
||||
BackupsQBitConfig config = BackupsRuntime.getConfig();
|
||||
String instance = config != null && config.getInstanceName() != null ? config.getInstanceName() : "qqq";
|
||||
|
||||
StringBuilder lastSuccess = header("qqq_backup_last_success_timestamp_seconds", "Unix time of the last successful backup run");
|
||||
StringBuilder lastFailed = header("qqq_backup_last_run_failed", "1 when the latest finished run of the plan failed");
|
||||
StringBuilder sla = header("qqq_backup_sla_seconds", "Maximum age of the last successful run before the plan is late");
|
||||
StringBuilder lastVerify = header("qqq_backup_last_verify_success_timestamp_seconds", "Unix time of the last passed content or test-restore verification");
|
||||
StringBuilder lastBytes = header("qqq_backup_last_success_bytes", "Bytes stored by the last successful run");
|
||||
|
||||
List<QRecord> plans = catalog.query(BackupsTables.PLAN, new QQueryFilter(new QFilterCriteria("isActive", QCriteriaOperator.EQUALS, true)));
|
||||
for(QRecord plan : plans)
|
||||
{
|
||||
String labels = "{instance_name=\"" + escape(instance) + "\",plan=\"" + escape(plan.getValueString("planKey")) + "\"}";
|
||||
|
||||
if(plan.getValueInstant("lastSuccessDate") != null)
|
||||
{
|
||||
lastSuccess.append("qqq_backup_last_success_timestamp_seconds").append(labels).append(' ').append(plan.getValueInstant("lastSuccessDate").getEpochSecond()).append('\n');
|
||||
}
|
||||
List<QRecord> runs = catalog.where(BackupsTables.RUN, "backupPlanId", plan.getValueInteger("id"));
|
||||
QRecord latest = runs.stream().filter(r -> List.of("SUCCEEDED", "SUCCEEDED_WITH_WARNINGS", "FAILED", "TIMED_OUT", "ABANDONED").contains(r.getValueString("status"))).findFirst().orElse(null);
|
||||
if(latest != null)
|
||||
{
|
||||
boolean failed = List.of("FAILED", "TIMED_OUT", "ABANDONED").contains(latest.getValueString("status"));
|
||||
lastFailed.append("qqq_backup_last_run_failed").append(labels).append(' ').append(failed ? 1 : 0).append('\n');
|
||||
}
|
||||
Integer slaHours = plan.getValueInteger("slaHours");
|
||||
if(slaHours != null)
|
||||
{
|
||||
sla.append("qqq_backup_sla_seconds").append(labels).append(' ').append(slaHours * 3600L).append('\n');
|
||||
}
|
||||
if(plan.getValueInstant("lastVerifiedDate") != null)
|
||||
{
|
||||
lastVerify.append("qqq_backup_last_verify_success_timestamp_seconds").append(labels).append(' ').append(plan.getValueInstant("lastVerifiedDate").getEpochSecond()).append('\n');
|
||||
}
|
||||
if(plan.getValueInteger("lastSuccessRunId") != null)
|
||||
{
|
||||
QRecord run = catalog.get(BackupsTables.RUN, plan.getValueInteger("lastSuccessRunId")).orElse(null);
|
||||
if(run != null && run.getValueLong("bytesStored") != null)
|
||||
{
|
||||
lastBytes.append("qqq_backup_last_success_bytes").append(labels).append(' ').append(run.getValueLong("bytesStored")).append('\n');
|
||||
}
|
||||
}
|
||||
}
|
||||
return (lastSuccess.toString() + lastFailed + sla + lastVerify + lastBytes);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static StringBuilder header(String name, String help)
|
||||
{
|
||||
return (new StringBuilder("# HELP ").append(name).append(' ').append(help).append('\n').append("# TYPE ").append(name).append(" gauge\n"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Escape a label value for the text format.
|
||||
*******************************************************************************/
|
||||
static String escape(String value)
|
||||
{
|
||||
return (value == null ? "" : value.replace("\\", "\\\\").replace("\"", "\\\"").replace("\n", "\\n"));
|
||||
}
|
||||
}
|
||||
@@ -319,6 +319,7 @@ public final class BackupsTables
|
||||
.withField(roInt("scheduledJobId"))
|
||||
.withField(new QFieldMetaData("slaHours", QFieldType.INTEGER).withDefaultValue(26))
|
||||
.withField(new QFieldMetaData("verifySlaDays", QFieldType.INTEGER).withDefaultValue(8))
|
||||
.withField(new QFieldMetaData("verifyFileSamplePercent", QFieldType.INTEGER).withDefaultValue(100))
|
||||
.withField(new QFieldMetaData("verifyCronExpression", QFieldType.STRING).withMaxLength(100))
|
||||
.withField(enumField("verifyLevel", "verify_level", "CONTENT"))
|
||||
.withField(new QFieldMetaData("retainDaily", QFieldType.INTEGER).withDefaultValue(7))
|
||||
|
||||
@@ -73,6 +73,7 @@ public final class BackupsProcesses
|
||||
List<QProcessMetaData> processes = new ArrayList<>();
|
||||
processes.add(permission(etl(RUN_PLAN, "Run backup now", BackupsTables.PLAN, RunPlanTransform.class, RunPlanLoad.class,
|
||||
"This is a preview of what the backup will include. Nothing is written until you continue."), BackupsPermissions.RUN));
|
||||
processes.addAll(VerifyProcesses.processes());
|
||||
return (processes);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.processes;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.backups.BackupsQBitConfig;
|
||||
import com.kingsrook.qbits.backups.BackupsRuntime;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.permissions.BackupsPermissions;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qbits.backups.sla.SlaChecker;
|
||||
import com.kingsrook.qbits.backups.verify.BackupVerifier;
|
||||
import com.kingsrook.qbits.backups.verify.VerificationOutcome;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
|
||||
import static com.kingsrook.qbits.backups.processes.BackupsProcesses.line;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The verification and SLA processes (design spec section 10):
|
||||
** <ul>
|
||||
** <li>backupVerify: the record action "Verify backup" on backups_run, a standard
|
||||
** streamed ETL. Review lists what will be checked (nothing is read); the run
|
||||
** does the STORAGE level and then, if that passes, the CONTENT level. The input
|
||||
** value verifyLevel (STORAGE, CONTENT or TEST_RESTORE) picks how far to go;
|
||||
** CONTENT is the default and TEST_RESTORE is for the API and schedules.</li>
|
||||
** <li>backupCheckSla: the scheduled SLA check, every slaCheckIntervalSeconds on
|
||||
** the configured scheduler, independent of the backup trigger. It is not in the
|
||||
** nav.</li>
|
||||
** </ul>
|
||||
*******************************************************************************/
|
||||
public final class VerifyProcesses
|
||||
{
|
||||
public static final String VERIFY = "backupVerify";
|
||||
public static final String CHECK_SLA = "backupCheckSla";
|
||||
|
||||
public static final String FIELD_VERIFY_LEVEL = "verifyLevel";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private VerifyProcesses()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The processes of this area.
|
||||
*******************************************************************************/
|
||||
public static List<QProcessMetaData> processes()
|
||||
{
|
||||
List<QProcessMetaData> processes = new ArrayList<>();
|
||||
processes.add(BackupsProcesses.permission(BackupsProcesses.etl(VERIFY, "Verify backup", BackupsTables.RUN, VerifyTransform.class, VerifyLoad.class,
|
||||
"This checks that the backup's objects are all in storage, then reads every byte back, decrypts and decodes it, and compares counts and hashes with the manifest. Nothing is changed in the backup."),
|
||||
BackupsPermissions.RUN));
|
||||
processes.add(checkSlaProcess(BackupsRuntime.getConfig()));
|
||||
return (processes);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The SLA check process. With a scheduler configured it is scheduled every
|
||||
** slaCheckIntervalSeconds.
|
||||
*******************************************************************************/
|
||||
public static QProcessMetaData checkSlaProcess(BackupsQBitConfig config)
|
||||
{
|
||||
QProcessMetaData process = new QProcessMetaData()
|
||||
.withName(CHECK_SLA)
|
||||
.withLabel("Check backup SLAs")
|
||||
.withStepList(List.<QStepMetaData>of(new QBackendStepMetaData().withName("check").withCode(new QCodeReference(CheckSlaStep.class))));
|
||||
process.setPermissionRules(BackupsTables.permissionRules());
|
||||
|
||||
if(config != null && config.getSchedulerName() != null && config.getSlaCheckIntervalSeconds() != null)
|
||||
{
|
||||
process.setSchedule(new QScheduleMetaData()
|
||||
.withSchedulerName(config.getSchedulerName())
|
||||
.withRepeatSeconds(config.getSlaCheckIntervalSeconds())
|
||||
.withInitialDelaySeconds(30)
|
||||
.withDescription("Backup SLA check (independent of the backup trigger)"));
|
||||
}
|
||||
return (BackupsProcesses.permission(process, BackupsPermissions.RUN));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The scheduled SLA check.
|
||||
*******************************************************************************/
|
||||
public static class CheckSlaStep implements BackendStep
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Check every active plan.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
List<SlaChecker.PlanResult> results = new SlaChecker().checkAll();
|
||||
long attention = results.stream().filter(r -> !SlaChecker.OK.equals(r.status()) && !SlaChecker.UNKNOWN.equals(r.status())).count();
|
||||
output.addValue("plansChecked", results.size());
|
||||
output.addValue("plansNeedingAttention", attention);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify backup, review: which runs can be verified and how much will be read.
|
||||
*******************************************************************************/
|
||||
public static class VerifyTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final ProcessSummaryLine runs = line(Status.OK, "backup will be verified", "backups will be verified", "backup was verified", "backups were verified");
|
||||
private final ProcessSummaryLine objects = line(Status.INFO, "stored object will be checked", "stored objects will be checked", "stored object was checked", "stored objects were checked");
|
||||
private final ProcessSummaryLine files = line(Status.INFO, "referenced file will be checked", "referenced files will be checked", "referenced file was checked", "referenced files were checked");
|
||||
private final ProcessSummaryLine noManifest = line(Status.ERROR, "backup has no manifest, so it is not usable and cannot be verified", "backups have no manifest, so they are not usable and cannot be verified",
|
||||
"backup had no manifest", "backups had no manifest");
|
||||
private final ProcessSummaryLine pruned = line(Status.ERROR, "backup was pruned, so its objects are gone", "backups were pruned, so their objects are gone", "backup was pruned", "backups were pruned");
|
||||
private final BackupCatalog catalog = new BackupCatalog();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Look at each run without reading any backup bytes.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
for(QRecord run : input.getRecords())
|
||||
{
|
||||
java.io.Serializable id = run.getValue("id");
|
||||
if(run.getValueInstant("prunedDate") != null)
|
||||
{
|
||||
pruned.incrementCountAndAddPrimaryKey(id);
|
||||
}
|
||||
else if(run.getValueString("manifestKey") == null)
|
||||
{
|
||||
noManifest.incrementCountAndAddPrimaryKey(id);
|
||||
}
|
||||
else
|
||||
{
|
||||
runs.incrementCountAndAddPrimaryKey(id);
|
||||
objects.incrementCount(catalog.where(BackupsTables.ARTIFACT, "backupRunId", run.getValueInteger("id")).size());
|
||||
files.incrementCount(run.getValueInteger("filesReferenced") == null ? 0 : run.getValueInteger("filesReferenced"));
|
||||
output.getRecords().add(run);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Review lines.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
runs.addSelfToListIfAnyCount(lines);
|
||||
objects.addSelfToListIfAnyCount(lines);
|
||||
files.addSelfToListIfAnyCount(lines);
|
||||
noManifest.addSelfToListIfAnyCount(lines);
|
||||
pruned.addSelfToListIfAnyCount(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify backup, execute: the storage level, then the content level.
|
||||
*******************************************************************************/
|
||||
public static class VerifyLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final ProcessSummaryLine passed = line(Status.OK, "backup will pass verification", "backups passed verification", "backup passed verification", "backups passed verification");
|
||||
private final ProcessSummaryLine warnings = line(Status.WARNING, "backup passed with warnings", "backups passed with warnings", "backup passed with warnings", "backups passed with warnings");
|
||||
private final ProcessSummaryLine failed = line(Status.ERROR, "backup failed verification", "backups failed verification", "backup failed verification", "backups failed verification");
|
||||
private final ProcessSummaryLine errored = line(Status.ERROR, "backup could not be verified", "backups could not be verified", "backup could not be verified", "backups could not be verified");
|
||||
private final ProcessSummaryLine rows = line(Status.INFO, "row was decoded and checked", "rows were decoded and checked", "row was decoded and checked", "rows were decoded and checked");
|
||||
private final ProcessSummaryLine files = line(Status.INFO, "file was checked", "files were checked", "file was checked", "files were checked");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify each run.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
String level = input.getValueString(FIELD_VERIFY_LEVEL) == null ? BackupVerifier.CONTENT : input.getValueString(FIELD_VERIFY_LEVEL);
|
||||
String user = QContext.getQSession() == null || QContext.getQSession().getUser() == null ? "system" : QContext.getQSession().getUser().getIdReference();
|
||||
for(QRecord run : input.getRecords())
|
||||
{
|
||||
List<VerificationOutcome> outcomes = new BackupVerifier().verifyThrough(run.getValueInteger("id"), level, "MANUAL", user);
|
||||
VerificationOutcome last = outcomes.get(outcomes.size() - 1);
|
||||
switch(last.status())
|
||||
{
|
||||
case "PASSED" -> passed.incrementCountAndAddPrimaryKey(run.getValue("id"));
|
||||
case "PASSED_WITH_WARNINGS" -> warnings.incrementCountAndAddPrimaryKey(run.getValue("id"));
|
||||
case "ERROR" ->
|
||||
{
|
||||
errored.incrementCountAndAddPrimaryKey(run.getValue("id"));
|
||||
errored.setMessageSuffix(last.messages().isEmpty() ? null : ": " + last.messages().get(0));
|
||||
}
|
||||
default ->
|
||||
{
|
||||
failed.incrementCountAndAddPrimaryKey(run.getValue("id"));
|
||||
failed.setMessageSuffix(last.messages().isEmpty() ? null : ": " + last.messages().get(0));
|
||||
}
|
||||
}
|
||||
for(VerificationOutcome outcome : outcomes)
|
||||
{
|
||||
rows.incrementCount((int) Math.min(Integer.MAX_VALUE, outcome.rowsChecked()));
|
||||
files.incrementCount(outcome.filesChecked());
|
||||
}
|
||||
output.getRecords().add(run);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Result lines.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
passed.addSelfToListIfAnyCount(lines);
|
||||
warnings.addSelfToListIfAnyCount(lines);
|
||||
failed.addSelfToListIfAnyCount(lines);
|
||||
errored.addSelfToListIfAnyCount(lines);
|
||||
rows.addSelfToListIfAnyCount(lines);
|
||||
files.addSelfToListIfAnyCount(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -35,6 +35,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.kingsrook.qbits.backups.BackupsRuntime;
|
||||
import com.kingsrook.qbits.backups.alerts.BackupAlertSink;
|
||||
import com.kingsrook.qbits.backups.codec.HashingOutputStream;
|
||||
import com.kingsrook.qbits.backups.events.BackupEvents;
|
||||
import com.kingsrook.qbits.backups.codec.TableArtifactWriter;
|
||||
import com.kingsrook.qbits.backups.files.FileCopier;
|
||||
import com.kingsrook.qbits.backups.files.FileReference;
|
||||
@@ -561,6 +562,7 @@ public class BackupEngine
|
||||
|
||||
BackupAlertSink sink = BackupsRuntime.getAlertSink();
|
||||
String planKey = request.getSettings().planKey();
|
||||
BackupEvents.runFinished(planKey, result.getRunKey(), status, result.getRows(), result.getBytesStored());
|
||||
if(ok)
|
||||
{
|
||||
sink.resolve("backups.run.failed", "backups.run.failed:" + planKey);
|
||||
|
||||
@@ -0,0 +1,543 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.sla;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import com.kingsrook.qbits.backups.BackupsQBitConfig;
|
||||
import com.kingsrook.qbits.backups.BackupsRuntime;
|
||||
import com.kingsrook.qbits.backups.alerts.BackupAlertSink;
|
||||
import com.kingsrook.qbits.backups.alerts.BackupAlerts;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qbits.backups.storage.BackupStorage;
|
||||
import com.kingsrook.qbits.backups.storage.StoredObjectInfo;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The SLA checker (design spec section 10.2): backupCheckSla. A scheduled process
|
||||
** that is independent of the backup trigger, so a broken trigger cannot also
|
||||
** silence the check. For each active plan it computes slaStatus:
|
||||
** <ul>
|
||||
** <li>MISSING_IN_STORAGE: the catalog says success but the manifest is gone
|
||||
** (a head on manifestKey, a positive answer cached for an hour);</li>
|
||||
** <li>FAILING: the last two finished runs failed;</li>
|
||||
** <li>ABANDONED: a RUNNING run has a heartbeat older than ten minutes (the run
|
||||
** is marked ABANDONED);</li>
|
||||
** <li>LATE: no success within slaHours;</li>
|
||||
** <li>OWNER_INVALID: the plan owner no longer validates (the configured
|
||||
** planOwnerValidator);</li>
|
||||
** <li>UNVERIFIED: no passed CONTENT or TEST_RESTORE verification within
|
||||
** verifySlaDays;</li>
|
||||
** <li>OK otherwise (UNKNOWN for a new plan that has not had time to run yet).</li>
|
||||
** </ul>
|
||||
** When several apply, the status shown is the first in that order; the alerts are
|
||||
** independent, one per condition. Once a day it also lists each plan's prefix in
|
||||
** the bucket and reports orphans (run prefixes that are not in the catalog) and
|
||||
** debris (prefixes with no manifest older than 24 hours): look at the bucket, not
|
||||
** at the job.
|
||||
*******************************************************************************/
|
||||
public class SlaChecker
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(SlaChecker.class);
|
||||
|
||||
public static final String OK = "OK";
|
||||
public static final String LATE = "LATE";
|
||||
public static final String FAILING = "FAILING";
|
||||
public static final String MISSING_IN_STORAGE = "MISSING_IN_STORAGE";
|
||||
public static final String UNVERIFIED = "UNVERIFIED";
|
||||
public static final String ABANDONED = "ABANDONED";
|
||||
public static final String OWNER_INVALID = "OWNER_INVALID";
|
||||
public static final String UNKNOWN = "UNKNOWN";
|
||||
|
||||
/** A RUNNING run whose heartbeat is older than this is abandoned. */
|
||||
public static final Duration STALE_HEARTBEAT = Duration.ofMinutes(10);
|
||||
|
||||
/** How long a successful manifest head is trusted. */
|
||||
public static final Duration MANIFEST_CACHE = Duration.ofHours(1);
|
||||
|
||||
/** How often the bucket is listed for orphans and debris. */
|
||||
public static final Duration BUCKET_AUDIT_INTERVAL = Duration.ofHours(24);
|
||||
|
||||
/** A prefix with no manifest older than this is debris. */
|
||||
public static final Duration DEBRIS_AGE = Duration.ofHours(24);
|
||||
|
||||
private static final Set<String> FAILED_STATUSES = Set.of("FAILED", "TIMED_OUT", "ABANDONED");
|
||||
private static final Set<String> FINISHED_STATUSES = Set.of("SUCCEEDED", "SUCCEEDED_WITH_WARNINGS", "FAILED", "TIMED_OUT", "ABANDONED");
|
||||
|
||||
private static final Map<String, Instant> MANIFEST_OK_AT = new ConcurrentHashMap<>();
|
||||
private static final Map<String, Instant> LAST_AUDIT_AT = new ConcurrentHashMap<>();
|
||||
|
||||
private final BackupCatalog catalog = new BackupCatalog();
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** The outcome of checking one plan.
|
||||
**
|
||||
** @param planId the plan id
|
||||
** @param planKey the plan key
|
||||
** @param previousStatus slaStatus before the check
|
||||
** @param status slaStatus now
|
||||
** @param abandonedRuns runs marked ABANDONED by this check
|
||||
** @param orphans orphan run prefixes found by the bucket audit (-1 when the audit did not run)
|
||||
** @param debris debris prefixes found by the bucket audit (-1 when the audit did not run)
|
||||
***************************************************************************/
|
||||
public record PlanResult(Integer planId, String planKey, String previousStatus, String status, int abandonedRuns, int orphans, int debris)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Forget the cached manifest heads and bucket audit times (tests).
|
||||
*******************************************************************************/
|
||||
public static void resetCaches()
|
||||
{
|
||||
MANIFEST_OK_AT.clear();
|
||||
LAST_AUDIT_AT.clear();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Check every active plan.
|
||||
*******************************************************************************/
|
||||
public List<PlanResult> checkAll() throws QException
|
||||
{
|
||||
List<PlanResult> results = new ArrayList<>();
|
||||
for(QRecord plan : catalog.query(BackupsTables.PLAN, new QQueryFilter(new QFilterCriteria("isActive", QCriteriaOperator.EQUALS, true))))
|
||||
{
|
||||
try
|
||||
{
|
||||
results.add(check(plan));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("The SLA check of a plan failed", e, logPair("plan", plan.getValueString("planKey")));
|
||||
}
|
||||
}
|
||||
return (results);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Check one plan: mark stale runs, compute the status, update the plan, raise and
|
||||
** resolve alerts, and (once a day) audit its prefix in the bucket.
|
||||
*******************************************************************************/
|
||||
public PlanResult check(QRecord plan) throws QException
|
||||
{
|
||||
Instant now = BackupsRuntime.getClock().instant();
|
||||
Integer planId = plan.getValueInteger("id");
|
||||
String planKey = plan.getValueString("planKey");
|
||||
boolean critical = Boolean.TRUE.equals(plan.getValueBoolean("critical"));
|
||||
|
||||
List<QRecord> runs = catalog.where(BackupsTables.RUN, "backupPlanId", planId);
|
||||
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
// a RUNNING run that stopped reporting is abandoned (pod died etc) //
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
int abandonedNow = 0;
|
||||
for(QRecord run : runs)
|
||||
{
|
||||
if("RUNNING".equals(run.getValueString("status")))
|
||||
{
|
||||
Instant lastSeen = run.getValueInstant("heartbeatDate") != null ? run.getValueInstant("heartbeatDate") : run.getValueInstant("startDate");
|
||||
if(lastSeen != null && lastSeen.isBefore(now.minus(STALE_HEARTBEAT)))
|
||||
{
|
||||
Map<String, Serializable> update = new LinkedHashMap<>();
|
||||
update.put("status", "ABANDONED");
|
||||
update.put("endDate", now);
|
||||
update.put("errorMessage", "The run stopped reporting a heartbeat (the process or pod probably died); resume it or run the plan again");
|
||||
catalog.update(BackupsTables.RUN, run.getValueInteger("id"), update);
|
||||
abandonedNow++;
|
||||
}
|
||||
}
|
||||
}
|
||||
if(abandonedNow > 0)
|
||||
{
|
||||
runs = catalog.where(BackupsTables.RUN, "backupPlanId", planId);
|
||||
}
|
||||
|
||||
List<QRecord> finished = runs.stream().filter(r -> FINISHED_STATUSES.contains(r.getValueString("status"))).toList();
|
||||
boolean latestFailed = !finished.isEmpty() && FAILED_STATUSES.contains(finished.get(0).getValueString("status"));
|
||||
boolean failing = finished.size() >= 2 && latestFailed && FAILED_STATUSES.contains(finished.get(1).getValueString("status"));
|
||||
boolean abandoned = abandonedNow > 0 || (!finished.isEmpty() && "ABANDONED".equals(finished.get(0).getValueString("status")));
|
||||
|
||||
boolean missing = isManifestMissing(plan, now);
|
||||
boolean late = isLate(plan, now);
|
||||
boolean ownerBad = isOwnerInvalid(plan);
|
||||
boolean unverified = isUnverified(plan, now);
|
||||
|
||||
String status;
|
||||
if(missing)
|
||||
{
|
||||
status = MISSING_IN_STORAGE;
|
||||
}
|
||||
else if(failing)
|
||||
{
|
||||
status = FAILING;
|
||||
}
|
||||
else if(abandoned)
|
||||
{
|
||||
status = ABANDONED;
|
||||
}
|
||||
else if(late)
|
||||
{
|
||||
status = LATE;
|
||||
}
|
||||
else if(ownerBad)
|
||||
{
|
||||
status = OWNER_INVALID;
|
||||
}
|
||||
else if(unverified)
|
||||
{
|
||||
status = UNVERIFIED;
|
||||
}
|
||||
else if(plan.getValueInstant("lastSuccessDate") == null)
|
||||
{
|
||||
status = UNKNOWN;
|
||||
}
|
||||
else
|
||||
{
|
||||
status = OK;
|
||||
}
|
||||
|
||||
String previous = plan.getValueString("slaStatus");
|
||||
if(!status.equals(previous))
|
||||
{
|
||||
catalog.update(BackupsTables.PLAN, planId, Map.of("slaStatus", status));
|
||||
}
|
||||
|
||||
signal(plan, planKey, critical, missing, late, ownerBad, unverified, latestFailed, failing, abandoned, finished.isEmpty() ? null : finished.get(0));
|
||||
|
||||
int[] audit = auditBucket(plan, planKey, runs, now);
|
||||
return (new PlanResult(planId, planKey, previous, status, abandonedNow, audit[0], audit[1]));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Raise and resolve the plan's alerts, one per condition.
|
||||
*******************************************************************************/
|
||||
private void signal(QRecord plan, String planKey, boolean critical, boolean missing, boolean late, boolean ownerBad, boolean unverified,
|
||||
boolean latestFailed, boolean failing, boolean abandoned, QRecord latestFinished)
|
||||
{
|
||||
String name = plan.getValueString("name") == null ? planKey : plan.getValueString("name");
|
||||
|
||||
if(missing)
|
||||
{
|
||||
BackupAlerts.raise(BackupAlerts.RUN_MISSING, planKey, BackupAlertSink.Severity.CRITICAL, "The latest backup is missing from storage",
|
||||
"plan", planKey, "planName", name);
|
||||
}
|
||||
else
|
||||
{
|
||||
BackupAlerts.resolve(BackupAlerts.RUN_MISSING, planKey);
|
||||
}
|
||||
|
||||
if(late)
|
||||
{
|
||||
BackupAlerts.raise(BackupAlerts.PLAN_MISSED, planKey, critical ? BackupAlertSink.Severity.CRITICAL : BackupAlertSink.Severity.ERROR,
|
||||
"A backup did not run in time", "plan", planKey, "planName", name, "slaHours", String.valueOf(slaHours(plan)));
|
||||
}
|
||||
else
|
||||
{
|
||||
BackupAlerts.resolve(BackupAlerts.PLAN_MISSED, planKey);
|
||||
}
|
||||
|
||||
if(latestFailed || abandoned)
|
||||
{
|
||||
BackupAlerts.raise(BackupAlerts.RUN_FAILED, planKey, failing ? BackupAlertSink.Severity.ERROR : BackupAlertSink.Severity.WARNING,
|
||||
failing ? "Backups have failed twice in a row" : "A backup run failed", "plan", planKey, "planName", name);
|
||||
}
|
||||
else if(latestFinished != null)
|
||||
{
|
||||
BackupAlerts.resolve(BackupAlerts.RUN_FAILED, planKey);
|
||||
}
|
||||
|
||||
if(ownerBad)
|
||||
{
|
||||
BackupAlerts.raise(BackupAlerts.PLAN_OWNER_INVALID, planKey, BackupAlertSink.Severity.WARNING, "The backup plan owner is not valid",
|
||||
"plan", planKey, "planName", name);
|
||||
}
|
||||
else
|
||||
{
|
||||
BackupAlerts.resolve(BackupAlerts.PLAN_OWNER_INVALID, planKey);
|
||||
}
|
||||
|
||||
if(unverified)
|
||||
{
|
||||
BackupAlerts.raise(BackupAlerts.VERIFY_OVERDUE, planKey, BackupAlertSink.Severity.WARNING, "No backup has been verified recently",
|
||||
"plan", planKey, "planName", name, "verifySlaDays", String.valueOf(plan.getValueInteger("verifySlaDays")));
|
||||
}
|
||||
else
|
||||
{
|
||||
BackupAlerts.resolve(BackupAlerts.VERIFY_OVERDUE, planKey);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The catalog says the last success exists; is its manifest still in the bucket?
|
||||
** A positive answer is trusted for an hour; a negative one is always re-checked.
|
||||
** A storage that cannot be reached does not mean the manifest is gone.
|
||||
*******************************************************************************/
|
||||
private boolean isManifestMissing(QRecord plan, Instant now) throws QException
|
||||
{
|
||||
Integer successId = plan.getValueInteger("lastSuccessRunId");
|
||||
if(successId == null)
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
QRecord run = catalog.get(BackupsTables.RUN, successId).orElse(null);
|
||||
if(run == null || run.getValueString("manifestKey") == null || run.getValueString("storageBucketAlias") == null || run.getValueInstant("prunedDate") != null)
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
|
||||
String cacheKey = run.getValueString("storageBucketAlias") + "|" + run.getValueString("manifestKey");
|
||||
Instant okAt = MANIFEST_OK_AT.get(cacheKey);
|
||||
if(okAt != null && okAt.plus(MANIFEST_CACHE).isAfter(now))
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
BackupStorage storage = BackupsRuntime.storageFor(run.getValueString("storageBucketAlias"));
|
||||
if(storage.head(run.getValueString("manifestKey")).isPresent())
|
||||
{
|
||||
MANIFEST_OK_AT.put(cacheKey, now);
|
||||
return (false);
|
||||
}
|
||||
MANIFEST_OK_AT.remove(cacheKey);
|
||||
return (true);
|
||||
}
|
||||
catch(RuntimeException e)
|
||||
{
|
||||
LOG.warn("Could not check the manifest of the last backup", e, logPair("plan", plan.getValueString("planKey")));
|
||||
return (false);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** No success within slaHours. A manual-only plan (no cron) with no slaHours has
|
||||
** no SLA. A new plan gets slaHours to run its first backup.
|
||||
*******************************************************************************/
|
||||
private boolean isLate(QRecord plan, Instant now)
|
||||
{
|
||||
Integer hours = slaHours(plan);
|
||||
if(hours == null)
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
Instant lastSuccess = plan.getValueInstant("lastSuccessDate");
|
||||
Instant reference = lastSuccess != null ? lastSuccess : plan.getValueInstant("createDate");
|
||||
return (reference != null && reference.plus(Duration.ofHours(hours)).isBefore(now));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** slaHours of a plan: its value, else 26 for a scheduled plan, else none.
|
||||
*******************************************************************************/
|
||||
private Integer slaHours(QRecord plan)
|
||||
{
|
||||
if(plan.getValueInteger("slaHours") != null)
|
||||
{
|
||||
return (plan.getValueInteger("slaHours"));
|
||||
}
|
||||
String cron = plan.getValueString("cronExpression");
|
||||
return (cron != null && !cron.isBlank() ? 26 : null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The plan has a successful backup but no passed CONTENT or TEST_RESTORE
|
||||
** verification within verifySlaDays.
|
||||
*******************************************************************************/
|
||||
private boolean isUnverified(QRecord plan, Instant now)
|
||||
{
|
||||
Integer days = plan.getValueInteger("verifySlaDays");
|
||||
if(days == null || plan.getValueInstant("lastSuccessDate") == null)
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
Instant reference = plan.getValueInstant("lastVerifiedDate") != null ? plan.getValueInstant("lastVerifiedDate") : plan.getValueInstant("createDate");
|
||||
return (reference != null && reference.plus(Duration.ofDays(days)).isBefore(now));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The plan owner no longer validates.
|
||||
*******************************************************************************/
|
||||
private boolean isOwnerInvalid(QRecord plan)
|
||||
{
|
||||
BackupsQBitConfig config = BackupsRuntime.getConfig();
|
||||
String owner = plan.getValueString("ownerUserId");
|
||||
if(config == null || config.getPlanOwnerValidator() == null || owner == null || owner.isBlank())
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
try
|
||||
{
|
||||
return (!config.getPlanOwnerValidator().test(owner));
|
||||
}
|
||||
catch(RuntimeException e)
|
||||
{
|
||||
LOG.warn("The plan owner validator failed", e, logPair("plan", plan.getValueString("planKey")));
|
||||
return (false);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Once a day, list the plan's prefix in the bucket: orphans (run prefixes not in
|
||||
** the catalog) and debris (no manifest, older than 24 hours). Returns
|
||||
* {orphans, debris}, or {-1, -1} when it did not run.
|
||||
*******************************************************************************/
|
||||
private int[] auditBucket(QRecord plan, String planKey, List<QRecord> runs, Instant now)
|
||||
{
|
||||
String alias = plan.getValueString("storageBucketAlias");
|
||||
if(alias == null)
|
||||
{
|
||||
return (new int[] { -1, -1 });
|
||||
}
|
||||
String auditKey = alias + "|" + planKey;
|
||||
Instant last = LAST_AUDIT_AT.get(auditKey);
|
||||
if(last != null && last.plus(BUCKET_AUDIT_INTERVAL).isAfter(now))
|
||||
{
|
||||
return (new int[] { -1, -1 });
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
BackupStorage storage = BackupsRuntime.storageFor(alias);
|
||||
String planPrefix = planPrefix(plan, planKey);
|
||||
|
||||
Set<String> known = new HashSet<>();
|
||||
Set<String> active = new HashSet<>();
|
||||
for(QRecord run : runs)
|
||||
{
|
||||
known.add(run.getValueString("runPrefix"));
|
||||
if("RUNNING".equals(run.getValueString("status")) || "QUEUED".equals(run.getValueString("status")))
|
||||
{
|
||||
active.add(run.getValueString("runPrefix"));
|
||||
}
|
||||
}
|
||||
|
||||
Map<String, Boolean> hasManifest = new LinkedHashMap<>();
|
||||
Map<String, Instant> newest = new LinkedHashMap<>();
|
||||
for(StoredObjectInfo object : storage.list(planPrefix))
|
||||
{
|
||||
String[] parts = object.key().substring(planPrefix.length()).split("/");
|
||||
if(parts.length < 5)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
String runPrefix = planPrefix + parts[0] + "/" + parts[1] + "/" + parts[2] + "/" + parts[3] + "/";
|
||||
hasManifest.merge(runPrefix, "manifest.json".equals(parts[4]), Boolean::logicalOr);
|
||||
if(object.lastModified() != null)
|
||||
{
|
||||
newest.merge(runPrefix, object.lastModified(), (a, b) -> a.isAfter(b) ? a : b);
|
||||
}
|
||||
}
|
||||
|
||||
int orphans = 0;
|
||||
int debris = 0;
|
||||
for(Map.Entry<String, Boolean> entry : hasManifest.entrySet())
|
||||
{
|
||||
String runPrefix = entry.getKey();
|
||||
if(active.contains(runPrefix))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if(!known.contains(runPrefix))
|
||||
{
|
||||
orphans++;
|
||||
}
|
||||
Instant modified = newest.get(runPrefix);
|
||||
if(!entry.getValue() && modified != null && modified.plus(DEBRIS_AGE).isBefore(now))
|
||||
{
|
||||
debris++;
|
||||
}
|
||||
}
|
||||
|
||||
LAST_AUDIT_AT.put(auditKey, now);
|
||||
if(orphans + debris > 0)
|
||||
{
|
||||
BackupAlerts.raise(BackupAlerts.STORAGE_DEBRIS, planKey, BackupAlertSink.Severity.INFO, "Backup storage has orphans or debris",
|
||||
"plan", planKey, "orphans", String.valueOf(orphans), "debris", String.valueOf(debris));
|
||||
}
|
||||
else
|
||||
{
|
||||
BackupAlerts.resolve(BackupAlerts.STORAGE_DEBRIS, planKey);
|
||||
}
|
||||
return (new int[] { orphans, debris });
|
||||
}
|
||||
catch(RuntimeException e)
|
||||
{
|
||||
LOG.warn("The bucket audit failed", e, logPair("plan", planKey));
|
||||
return (new int[] { -1, -1 });
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The prefix of a plan's runs in its bucket, the same way the backup engine
|
||||
** builds it: (plan prefix)/(instance name)/(planKey)/.
|
||||
*******************************************************************************/
|
||||
static String planPrefix(QRecord plan, String planKey)
|
||||
{
|
||||
BackupsQBitConfig config = BackupsRuntime.getConfig();
|
||||
String instance = config != null && config.getInstanceName() != null ? config.getInstanceName() : "qqq";
|
||||
String prefix = plan.getValueString("prefix") != null ? plan.getValueString("prefix") : "qqq-backups";
|
||||
while(prefix.startsWith("/"))
|
||||
{
|
||||
prefix = prefix.substring(1);
|
||||
}
|
||||
while(prefix.endsWith("/"))
|
||||
{
|
||||
prefix = prefix.substring(0, prefix.length() - 1);
|
||||
}
|
||||
return (prefix + "/" + instance + "/" + planKey + "/");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,630 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.verify;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.io.Serializable;
|
||||
import java.security.DigestInputStream;
|
||||
import java.security.MessageDigest;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.kingsrook.qbits.backups.BackupsQBitConfig;
|
||||
import com.kingsrook.qbits.backups.BackupsRuntime;
|
||||
import com.kingsrook.qbits.backups.alerts.BackupAlerts;
|
||||
import com.kingsrook.qbits.backups.alerts.BackupAlertSink;
|
||||
import com.kingsrook.qbits.backups.codec.ArtifactTransform;
|
||||
import com.kingsrook.qbits.backups.codec.HashingOutputStream;
|
||||
import com.kingsrook.qbits.backups.codec.JsonlRowCodec;
|
||||
import com.kingsrook.qbits.backups.manifest.BackupManifest;
|
||||
import com.kingsrook.qbits.backups.manifest.TableFingerprints;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qbits.backups.storage.BackupStorage;
|
||||
import com.kingsrook.qbits.backups.storage.BackupStorageException;
|
||||
import com.kingsrook.qbits.backups.storage.StoredObjectInfo;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verifies backup runs (design spec section 10.1). Three levels, each a row in
|
||||
** backups_verification:
|
||||
** <ul>
|
||||
** <li>STORAGE: head every object of the run, including every file-store object
|
||||
** the run's file index references; sizes (and a provider checksum, where the
|
||||
** provider stores one) must match the manifest.</li>
|
||||
** <li>CONTENT: read every byte. Each part is decrypted (the decrypt hook,
|
||||
** ArtifactTransform.unwrap), gunzipped and parsed row by row with the codec;
|
||||
** row counts, sha256Stored, sha256Raw and the table rowsHash are recomputed and
|
||||
** compared with the manifest. Every referenced file-store object is decrypted
|
||||
** and re-hashed against its SHA-256 key, except those verified within
|
||||
** fileReverifyDays (backups_file_object.lastVerifiedDate).</li>
|
||||
** <li>TEST_RESTORE: see {@link TestRestoreVerifier}.</li>
|
||||
** </ul>
|
||||
** The result updates backups_run.verificationStatus and backups_plan.lastVerified*,
|
||||
** and a failure raises backups.verify.failed. A mismatch is a FAILED verification;
|
||||
** a verification that could not run (storage unreachable, no key) is an ERROR.
|
||||
*******************************************************************************/
|
||||
public class BackupVerifier
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(BackupVerifier.class);
|
||||
|
||||
private static final ObjectMapper MAPPER = new ObjectMapper();
|
||||
|
||||
public static final String STORAGE = "STORAGE";
|
||||
public static final String CONTENT = "CONTENT";
|
||||
public static final String TEST_RESTORE = "TEST_RESTORE";
|
||||
|
||||
/** The levels in order; each includes the ones before it when run through. */
|
||||
public static final List<String> LEVELS = List.of(STORAGE, CONTENT, TEST_RESTORE);
|
||||
|
||||
private static final List<String> STATUS_RANK = List.of("UNVERIFIED", "STORAGE_OK", "CONTENT_OK", "TEST_RESTORE_OK");
|
||||
|
||||
private final BackupCatalog catalog = new BackupCatalog();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify a run at each level up to and including the given one, stopping at the
|
||||
** first level that does not pass. Returns one outcome per level that ran.
|
||||
*******************************************************************************/
|
||||
public List<VerificationOutcome> verifyThrough(Integer runId, String maxLevel, String trigger, String requestedBy) throws QException
|
||||
{
|
||||
List<VerificationOutcome> outcomes = new ArrayList<>();
|
||||
for(String level : LEVELS.subList(0, LEVELS.indexOf(maxLevel) + 1))
|
||||
{
|
||||
VerificationOutcome outcome = verify(runId, level, trigger, requestedBy);
|
||||
outcomes.add(outcome);
|
||||
if(!outcome.passed())
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
return (outcomes);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify a run at one level. Never throws for a backup problem: the outcome
|
||||
** carries the status and the first messages.
|
||||
*******************************************************************************/
|
||||
public VerificationOutcome verify(Integer runId, String level, String trigger, String requestedBy) throws QException
|
||||
{
|
||||
if(!LEVELS.contains(level))
|
||||
{
|
||||
throw (new QException("Unknown verification level: " + level));
|
||||
}
|
||||
QRecord run = catalog.get(BackupsTables.RUN, runId).orElseThrow(() -> new QException("No backup run with id " + runId));
|
||||
QRecord plan = run.getValueInteger("backupPlanId") == null ? null : catalog.get(BackupsTables.PLAN, run.getValueInteger("backupPlanId")).orElse(null);
|
||||
|
||||
Instant started = BackupsRuntime.getClock().instant();
|
||||
Map<String, Serializable> row = new LinkedHashMap<>();
|
||||
row.put("backupRunId", runId);
|
||||
row.put("level", level);
|
||||
row.put("trigger", trigger);
|
||||
row.put("requestedByUserId", requestedBy);
|
||||
row.put("startDate", started);
|
||||
row.put("status", "RUNNING");
|
||||
Integer verificationId = catalog.insert(BackupsTables.VERIFICATION, row);
|
||||
|
||||
VerifyFindings findings = new VerifyFindings();
|
||||
String status;
|
||||
String scratch = null;
|
||||
try
|
||||
{
|
||||
RunBundle bundle = RunBundle.load(run, plan, findings);
|
||||
if(bundle != null)
|
||||
{
|
||||
switch(level)
|
||||
{
|
||||
case STORAGE -> storageLevel(bundle, findings);
|
||||
case CONTENT -> contentLevel(bundle, findings);
|
||||
default -> scratch = new TestRestoreVerifier().verify(bundle, findings);
|
||||
}
|
||||
}
|
||||
status = findings.status();
|
||||
}
|
||||
catch(VerifyException | BackupStorageException | QException e)
|
||||
{
|
||||
status = "ERROR";
|
||||
findings.mismatch("The verification could not be carried out: " + e.getMessage());
|
||||
}
|
||||
catch(RuntimeException e)
|
||||
{
|
||||
LOG.warn("Verification crashed", e, logPair("runId", runId), logPair("level", level));
|
||||
status = "ERROR";
|
||||
findings.mismatch("The verification crashed: " + e.getClass().getSimpleName() + ": " + e.getMessage());
|
||||
}
|
||||
|
||||
Instant ended = BackupsRuntime.getClock().instant();
|
||||
VerificationOutcome outcome = new VerificationOutcome(verificationId, runId, level, status, findings.tablesChecked, findings.rowsChecked, findings.mismatches,
|
||||
findings.objectsChecked, findings.filesChecked, findings.filesSkipped, findings.messages(), scratch);
|
||||
|
||||
Map<String, Serializable> done = new LinkedHashMap<>();
|
||||
done.put("endDate", ended);
|
||||
done.put("status", status);
|
||||
done.put("tablesChecked", findings.tablesChecked);
|
||||
done.put("rowsChecked", findings.rowsChecked);
|
||||
done.put("mismatches", findings.mismatches);
|
||||
done.put("scratchTarget", scratch);
|
||||
done.put("detailJson", detailJson(findings));
|
||||
catalog.update(BackupsTables.VERIFICATION, verificationId, done);
|
||||
|
||||
applyOutcome(run, plan, outcome, ended);
|
||||
return (outcome);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** STORAGE level: head every object, including every file-store object.
|
||||
*******************************************************************************/
|
||||
void storageLevel(RunBundle bundle, VerifyFindings findings)
|
||||
{
|
||||
BackupManifest manifest = bundle.manifest();
|
||||
BackupStorage storage = bundle.storage();
|
||||
findings.tablesChecked = manifest.tables().size();
|
||||
findings.objectsChecked++;
|
||||
|
||||
headOrMismatch(storage, bundle.runPrefix() + "checksums.sha256", "checksums.sha256", findings);
|
||||
|
||||
for(BackupManifest.TableEntry table : manifest.tables())
|
||||
{
|
||||
for(BackupManifest.Part part : table.parts() == null ? List.<BackupManifest.Part>of() : table.parts())
|
||||
{
|
||||
findings.objectsChecked++;
|
||||
Optional<StoredObjectInfo> head = storage.head(bundle.runPrefix() + part.key());
|
||||
if(head.isEmpty())
|
||||
{
|
||||
findings.mismatch("Missing object: " + part.key() + " (table " + table.name() + ")");
|
||||
continue;
|
||||
}
|
||||
if(head.get().size() != part.bytesStored())
|
||||
{
|
||||
findings.mismatch("Size mismatch for " + part.key() + ": the manifest says " + part.bytesStored() + " bytes, the bucket has " + head.get().size());
|
||||
}
|
||||
String providerSha = head.get().metadata() == null ? null : head.get().metadata().get("sha256");
|
||||
if(providerSha != null && !providerSha.equalsIgnoreCase(part.sha256Stored()))
|
||||
{
|
||||
findings.mismatch("Checksum mismatch for " + part.key() + ": the provider's stored SHA-256 differs from the manifest");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(manifest.files() != null && manifest.files().indexKeys() != null)
|
||||
{
|
||||
for(String indexKey : manifest.files().indexKeys())
|
||||
{
|
||||
headOrMismatch(storage, bundle.runPrefix() + indexKey, indexKey, findings);
|
||||
}
|
||||
}
|
||||
|
||||
int missingFiles = 0;
|
||||
for(RunBundle.FileIndexEntry entry : bundle.distinctStoredFiles(findings))
|
||||
{
|
||||
findings.objectsChecked++;
|
||||
findings.filesChecked++;
|
||||
Optional<StoredObjectInfo> head = storage.head(entry.casKey());
|
||||
if(head.isEmpty())
|
||||
{
|
||||
missingFiles++;
|
||||
findings.mismatch("File store object missing: sha256 " + entry.sha256() + " (referenced by " + entry.table() + ")");
|
||||
}
|
||||
else if(!bundle.isEncrypted() && entry.size() >= 0 && head.get().size() != entry.size())
|
||||
{
|
||||
findings.mismatch("File store object has the wrong size: sha256 " + entry.sha256() + " should be " + entry.size() + " bytes, the bucket has " + head.get().size());
|
||||
}
|
||||
}
|
||||
findings.extra.put("filesMissingFromStore", missingFiles);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** CONTENT level: decrypt, gunzip, parse, recount and re-hash everything.
|
||||
*******************************************************************************/
|
||||
void contentLevel(RunBundle bundle, VerifyFindings findings) throws QException
|
||||
{
|
||||
BackupManifest manifest = bundle.manifest();
|
||||
QInstance qInstance = QContext.getQInstance();
|
||||
findings.tablesChecked = manifest.tables().size();
|
||||
|
||||
checkChecksumsFile(bundle, findings);
|
||||
|
||||
for(BackupManifest.TableEntry entry : manifest.tables())
|
||||
{
|
||||
QTableMetaData table = qInstance.getTable(entry.name());
|
||||
JsonlRowCodec codec = null;
|
||||
boolean roundTrip = false;
|
||||
if(table == null)
|
||||
{
|
||||
findings.warn("Table " + entry.name() + " no longer exists in this instance; its rows were checked as JSON, not with the codec");
|
||||
}
|
||||
else
|
||||
{
|
||||
List<String> excluded = entry.excludedFields() == null ? List.of() : entry.excludedFields().stream().map(BackupManifest.ExcludedField::name).toList();
|
||||
codec = new JsonlRowCodec(table, excluded);
|
||||
String backendType = qInstance.getBackend(table.getBackendName()) == null ? null : qInstance.getBackend(table.getBackendName()).getBackendType();
|
||||
roundTrip = TableFingerprints.of(table, backendType).equals(entry.fingerprint());
|
||||
if(!roundTrip)
|
||||
{
|
||||
findings.warn("Table " + entry.name() + " has changed since this backup; rows were parsed but not checked for an exact round trip");
|
||||
}
|
||||
}
|
||||
verifyTable(bundle, entry, codec, roundTrip, findings);
|
||||
}
|
||||
|
||||
verifyFiles(bundle, findings);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** All parts of one table.
|
||||
*******************************************************************************/
|
||||
private void verifyTable(RunBundle bundle, BackupManifest.TableEntry entry, JsonlRowCodec codec, boolean roundTrip, VerifyFindings findings)
|
||||
{
|
||||
MessageDigest rowsDigest = HashingOutputStream.newSha256();
|
||||
long rows = 0;
|
||||
boolean hasJsonl = false;
|
||||
int partCount = 0;
|
||||
int before = findings.mismatches;
|
||||
|
||||
for(BackupManifest.Part part : entry.parts() == null ? List.<BackupManifest.Part>of() : entry.parts())
|
||||
{
|
||||
partCount++;
|
||||
findings.objectsChecked++;
|
||||
boolean jsonl = !"CSV".equals(part.format());
|
||||
hasJsonl |= jsonl;
|
||||
long[] parsedRows = { 0 };
|
||||
try
|
||||
{
|
||||
PartScanner.Scan scan = PartScanner.scan(bundle, part, jsonl ? line ->
|
||||
{
|
||||
parsedRows[0]++;
|
||||
rowsDigest.update((line + "\n").getBytes(java.nio.charset.StandardCharsets.UTF_8));
|
||||
checkRow(entry.name(), part.key(), parsedRows[0], line, codec, roundTrip, findings);
|
||||
} : null);
|
||||
|
||||
if(!scan.sha256Stored().equals(part.sha256Stored()))
|
||||
{
|
||||
findings.mismatch("Stored checksum mismatch for " + part.key() + ": the bytes in the bucket are not the bytes that were written (corrupted or replaced)");
|
||||
}
|
||||
if(!scan.sha256Raw().equals(part.sha256Raw()))
|
||||
{
|
||||
findings.mismatch("Content checksum mismatch for " + part.key() + ": the decoded content differs from what was written");
|
||||
}
|
||||
if(jsonl && scan.lines() != part.rows())
|
||||
{
|
||||
findings.mismatch("Row count mismatch for " + part.key() + ": the manifest says " + part.rows() + " rows, the part has " + scan.lines());
|
||||
}
|
||||
if(jsonl)
|
||||
{
|
||||
rows += scan.lines();
|
||||
}
|
||||
}
|
||||
catch(IOException | RuntimeException e)
|
||||
{
|
||||
findings.mismatch("The part " + part.key() + " could not be read, decrypted or decompressed (" + e.getClass().getSimpleName() + (e.getMessage() == null ? "" : ": " + e.getMessage())
|
||||
+ "); it is corrupted, or the key is wrong");
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
findings.mismatch("The part " + part.key() + " could not be read: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
String actualHash = hasJsonl ? "sha256:" + HashingOutputStream.hex(rowsDigest.digest()) : null;
|
||||
if(hasJsonl)
|
||||
{
|
||||
findings.rowsChecked += rows;
|
||||
if(rows != entry.rowCount())
|
||||
{
|
||||
findings.mismatch("Row count mismatch for table " + entry.name() + ": the manifest says " + entry.rowCount() + ", the parts hold " + rows);
|
||||
}
|
||||
if(entry.rowsHash() != null && !entry.rowsHash().equals(actualHash))
|
||||
{
|
||||
findings.mismatch("Row content hash mismatch for table " + entry.name() + ": the rows are not the rows that were backed up");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
findings.warn("Table " + entry.name() + " has only CSV parts; their bytes were hashed but rows were not recounted");
|
||||
}
|
||||
|
||||
Map<String, Object> detail = new LinkedHashMap<>();
|
||||
detail.put("table", entry.name());
|
||||
detail.put("parts", partCount);
|
||||
detail.put("expectedRows", entry.rowCount());
|
||||
detail.put("actualRows", hasJsonl ? rows : null);
|
||||
detail.put("expectedRowsHash", entry.rowsHash());
|
||||
detail.put("actualRowsHash", actualHash);
|
||||
detail.put("ok", findings.mismatches == before);
|
||||
findings.tables.add(detail);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Parse one row with the codec (and check the exact round trip when the table
|
||||
** is unchanged). Without a table, check only that it is a JSON object.
|
||||
*******************************************************************************/
|
||||
private void checkRow(String table, String partKey, long rowNumber, String line, JsonlRowCodec codec, boolean roundTrip, VerifyFindings findings) throws QException
|
||||
{
|
||||
try
|
||||
{
|
||||
if(codec == null)
|
||||
{
|
||||
if(!MAPPER.readTree(line).isObject())
|
||||
{
|
||||
throw (new QException("not a JSON object"));
|
||||
}
|
||||
return;
|
||||
}
|
||||
QRecord record = codec.decode(line);
|
||||
if(roundTrip && !codec.encode(record).equals(line))
|
||||
{
|
||||
findings.mismatch("Row " + rowNumber + " of " + partKey + " does not survive a decode and encode round trip (table " + table + ")");
|
||||
}
|
||||
}
|
||||
catch(IOException | QException e)
|
||||
{
|
||||
findings.mismatch("Row " + rowNumber + " of " + partKey + " cannot be parsed (table " + table + ")");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The checksums.sha256 file must list every part with its stored SHA-256.
|
||||
*******************************************************************************/
|
||||
private void checkChecksumsFile(RunBundle bundle, VerifyFindings findings)
|
||||
{
|
||||
String key = bundle.runPrefix() + "checksums.sha256";
|
||||
findings.objectsChecked++;
|
||||
try(InputStream in = bundle.storage().openRead(key))
|
||||
{
|
||||
String text = new String(in.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8);
|
||||
for(BackupManifest.TableEntry table : bundle.manifest().tables())
|
||||
{
|
||||
for(BackupManifest.Part part : table.parts() == null ? List.<BackupManifest.Part>of() : table.parts())
|
||||
{
|
||||
if(!text.contains(part.sha256Stored() + " " + part.key()))
|
||||
{
|
||||
findings.mismatch("checksums.sha256 does not list " + part.key() + " with the manifest's checksum");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch(BackupStorageException e)
|
||||
{
|
||||
findings.mismatch("checksums.sha256 is missing from the bucket");
|
||||
}
|
||||
catch(IOException e)
|
||||
{
|
||||
findings.mismatch("checksums.sha256 cannot be read: " + e.getClass().getSimpleName());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Re-hash every referenced file-store object against its SHA-256 key, skipping
|
||||
** those verified within fileReverifyDays.
|
||||
*******************************************************************************/
|
||||
private void verifyFiles(RunBundle bundle, VerifyFindings findings) throws QException
|
||||
{
|
||||
BackupsQBitConfig config = BackupsRuntime.getConfig();
|
||||
int days = config == null || config.getFileReverifyDays() == null ? 30 : config.getFileReverifyDays();
|
||||
Instant now = BackupsRuntime.getClock().instant();
|
||||
Instant cutoff = now.minus(Duration.ofDays(days));
|
||||
int failed = 0;
|
||||
|
||||
for(RunBundle.FileIndexEntry entry : bundle.distinctStoredFiles(findings))
|
||||
{
|
||||
QRecord fileObject = catalog.where(BackupsTables.FILE_OBJECT, "sha256", entry.sha256()).stream().filter(r -> r.getValue("deletedDate") == null).findFirst().orElse(null);
|
||||
if(fileObject != null && "PASSED".equals(fileObject.getValueString("lastVerifyStatus"))
|
||||
&& fileObject.getValueInstant("lastVerifiedDate") != null && fileObject.getValueInstant("lastVerifiedDate").isAfter(cutoff))
|
||||
{
|
||||
findings.filesSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
findings.filesChecked++;
|
||||
findings.objectsChecked++;
|
||||
boolean encrypted = fileObject != null ? Boolean.TRUE.equals(fileObject.getValueBoolean("encrypted")) : bundle.isEncrypted();
|
||||
boolean ok = false;
|
||||
try
|
||||
{
|
||||
String actual = hashStoredFile(bundle, entry.casKey(), encrypted);
|
||||
ok = actual.equals(entry.sha256());
|
||||
if(!ok)
|
||||
{
|
||||
findings.mismatch("File store object sha256 " + entry.sha256() + " has the wrong content: re-hashing it gives " + actual);
|
||||
}
|
||||
}
|
||||
catch(BackupStorageException e)
|
||||
{
|
||||
findings.mismatch("File store object missing: sha256 " + entry.sha256() + " (referenced by " + entry.table() + ")");
|
||||
}
|
||||
catch(IOException | RuntimeException e)
|
||||
{
|
||||
findings.mismatch("File store object sha256 " + entry.sha256() + " could not be read or decrypted (" + e.getClass().getSimpleName() + ")");
|
||||
}
|
||||
|
||||
if(!ok)
|
||||
{
|
||||
failed++;
|
||||
}
|
||||
if(fileObject != null)
|
||||
{
|
||||
Map<String, Serializable> update = new LinkedHashMap<>();
|
||||
update.put("lastVerifiedDate", now);
|
||||
update.put("lastVerifyStatus", ok ? "PASSED" : "FAILED");
|
||||
catalog.update(BackupsTables.FILE_OBJECT, fileObject.getValueInteger("id"), update);
|
||||
}
|
||||
}
|
||||
findings.extra.put("filesFailed", failed);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** SHA-256 of a stored file's plaintext (decrypted when it was encrypted).
|
||||
*******************************************************************************/
|
||||
static String hashStoredFile(RunBundle bundle, String key, boolean encrypted) throws IOException
|
||||
{
|
||||
return (hashObject(bundle.storage(), key, encrypted ? bundle.transform() : ArtifactTransform.IDENTITY));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** SHA-256 of an object read through a transform.
|
||||
*******************************************************************************/
|
||||
static String hashObject(BackupStorage storage, String key, ArtifactTransform transform) throws IOException
|
||||
{
|
||||
MessageDigest digest = HashingOutputStream.newSha256();
|
||||
try(InputStream raw = storage.openRead(key);
|
||||
DigestInputStream plain = new DigestInputStream(transform.unwrap(raw), digest))
|
||||
{
|
||||
plain.transferTo(OutputStream.nullOutputStream());
|
||||
}
|
||||
return (HashingOutputStream.hex(digest.digest()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Head an object, recording a mismatch when it is missing.
|
||||
*******************************************************************************/
|
||||
private void headOrMismatch(BackupStorage storage, String key, String label, VerifyFindings findings)
|
||||
{
|
||||
findings.objectsChecked++;
|
||||
if(storage.head(key).isEmpty())
|
||||
{
|
||||
findings.mismatch("Missing object: " + label);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The per-table detail and the messages, as JSON text.
|
||||
*******************************************************************************/
|
||||
private String detailJson(VerifyFindings findings)
|
||||
{
|
||||
Map<String, Object> detail = new LinkedHashMap<>();
|
||||
detail.put("tables", findings.tables);
|
||||
detail.put("mismatches", findings.mismatchMessages);
|
||||
detail.put("warnings", findings.warnings);
|
||||
detail.put("objectsChecked", findings.objectsChecked);
|
||||
detail.put("filesChecked", findings.filesChecked);
|
||||
detail.put("filesSkipped", findings.filesSkipped);
|
||||
detail.putAll(findings.extra);
|
||||
try
|
||||
{
|
||||
return (MAPPER.writeValueAsString(detail));
|
||||
}
|
||||
catch(IOException e)
|
||||
{
|
||||
return ("{}");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Update the run, the plan and the alerts from an outcome.
|
||||
*******************************************************************************/
|
||||
private void applyOutcome(QRecord run, QRecord plan, VerificationOutcome outcome, Instant when) throws QException
|
||||
{
|
||||
Integer runId = run.getValueInteger("id");
|
||||
String current = run.getValueString("verificationStatus");
|
||||
String newState = null;
|
||||
if(outcome.passed())
|
||||
{
|
||||
String reached = switch(outcome.level())
|
||||
{
|
||||
case STORAGE -> "STORAGE_OK";
|
||||
case CONTENT -> "CONTENT_OK";
|
||||
default -> "TEST_RESTORE_OK";
|
||||
};
|
||||
boolean currentFailed = "FAILED".equals(current);
|
||||
// a cheap pass never clears a failure found by a deeper level
|
||||
if(!currentFailed || !STORAGE.equals(outcome.level()))
|
||||
{
|
||||
newState = !currentFailed && STATUS_RANK.indexOf(current) > STATUS_RANK.indexOf(reached) ? current : reached;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
newState = "FAILED";
|
||||
}
|
||||
if(newState != null && !newState.equals(current))
|
||||
{
|
||||
catalog.update(BackupsTables.RUN, runId, Map.of("verificationStatus", newState));
|
||||
}
|
||||
|
||||
if(plan == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
String planKey = plan.getValueString("planKey");
|
||||
int rank = LEVELS.indexOf(outcome.level());
|
||||
if(outcome.passed())
|
||||
{
|
||||
for(int i = 0; i <= rank; i++)
|
||||
{
|
||||
BackupAlerts.resolveKey(BackupAlerts.VERIFY_FAILED, BackupAlerts.verifyKey(planKey, LEVELS.get(i)));
|
||||
}
|
||||
if(rank >= LEVELS.indexOf(CONTENT))
|
||||
{
|
||||
Map<String, Serializable> planUpdate = new LinkedHashMap<>();
|
||||
planUpdate.put("lastVerifiedRunId", runId);
|
||||
planUpdate.put("lastVerifiedDate", when);
|
||||
catalog.update(BackupsTables.PLAN, plan.getValueInteger("id"), planUpdate);
|
||||
BackupAlerts.resolve(BackupAlerts.VERIFY_OVERDUE, planKey);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
Map<String, String> detail = new LinkedHashMap<>();
|
||||
detail.put("plan", planKey);
|
||||
detail.put("run", run.getValueString("runKey"));
|
||||
detail.put("level", outcome.level());
|
||||
detail.put("mismatches", String.valueOf(outcome.mismatches()));
|
||||
BackupAlerts.raiseWithKey(BackupAlerts.VERIFY_FAILED, BackupAlerts.verifyKey(planKey, outcome.level()), BackupAlertSink.Severity.ERROR,
|
||||
"Backup verification failed (" + outcome.level() + ")", detail);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.verify;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Consumer;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The MEMORY scratch target (design spec section 10.1): QQQ's memory backend. It
|
||||
** proves the files and the codecs, not the database. Each source table is cloned
|
||||
** into a temporary in-memory QTableMetaData (a different name, the memory backend,
|
||||
** no customizers, automation, behaviors, unique keys or associations) that is
|
||||
** registered in the instance for the length of the verification and removed by
|
||||
** drop().
|
||||
*******************************************************************************/
|
||||
public class MemoryScratchTarget implements ScratchTarget
|
||||
{
|
||||
public static final String BACKEND_NAME = "qqq_verify_memory";
|
||||
public static final String TYPE = "MEMORY";
|
||||
|
||||
private final Map<String, List<String>> tablesByRun = new LinkedHashMap<>();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String type()
|
||||
{
|
||||
return (TYPE);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String describe(String runKey)
|
||||
{
|
||||
return ("memory:" + prefix(runKey));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public synchronized ScratchTable create(QTableMetaData source, String runKey) throws QException
|
||||
{
|
||||
QInstance qInstance = QContext.getQInstance();
|
||||
if(qInstance.getBackend(BACKEND_NAME) == null)
|
||||
{
|
||||
qInstance.addBackend(new QBackendMetaData().withName(BACKEND_NAME).withBackendType(MemoryBackendModule.class));
|
||||
}
|
||||
|
||||
String scratchName = prefix(runKey) + "_" + source.getName();
|
||||
QTableMetaData clone = source.clone();
|
||||
clone.setName(scratchName);
|
||||
clone.setBackendName(BACKEND_NAME);
|
||||
clone.setBackendDetails(null);
|
||||
clone.setAutomationDetails(null);
|
||||
clone.setCustomizers(null);
|
||||
clone.setUniqueKeys(null);
|
||||
clone.setAssociations(new ArrayList<>());
|
||||
clone.setRecordSecurityLocks(new ArrayList<>());
|
||||
clone.setExposedJoins(new ArrayList<>());
|
||||
clone.setSupplementalMetaData(null);
|
||||
clone.setSourceQBitName(null);
|
||||
clone.getFields().values().forEach(field -> field.setBehaviors(null));
|
||||
qInstance.getTables().put(scratchName, clone);
|
||||
tablesByRun.computeIfAbsent(runKey, k -> new ArrayList<>()).add(scratchName);
|
||||
|
||||
return (new ScratchTable()
|
||||
{
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void insert(List<QRecord> records) throws QException
|
||||
{
|
||||
InsertInput input = new InsertInput(scratchName);
|
||||
input.setRecords(records);
|
||||
input.withPermissionBypass("backups verify scratch");
|
||||
List<QRecord> inserted = new InsertAction().execute(input).getRecords();
|
||||
for(QRecord record : inserted)
|
||||
{
|
||||
if(record.getErrors() != null && !record.getErrors().isEmpty())
|
||||
{
|
||||
throw (new QException("A row could not be restored into the scratch table for " + source.getName() + ": " + record.getErrors().get(0).getMessage()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void readInPrimaryKeyOrder(int pageSize, Consumer<QRecord> consumer) throws QException
|
||||
{
|
||||
int skip = 0;
|
||||
while(true)
|
||||
{
|
||||
QueryInput input = new QueryInput(scratchName);
|
||||
input.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderBy(source.getPrimaryKeyField(), true)).withSkip(skip).withLimit(pageSize));
|
||||
input.withPermissionBypass("backups verify scratch");
|
||||
List<QRecord> page = new QueryAction().execute(input).getRecords();
|
||||
page.forEach(consumer);
|
||||
if(page.size() < pageSize)
|
||||
{
|
||||
return;
|
||||
}
|
||||
skip += pageSize;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public synchronized void drop(String runKey) throws QException
|
||||
{
|
||||
List<String> names = tablesByRun.remove(runKey);
|
||||
if(names == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
for(String name : names)
|
||||
{
|
||||
DeleteInput input = new DeleteInput(name);
|
||||
input.setQueryFilter(new QQueryFilter());
|
||||
input.withPermissionBypass("backups verify scratch");
|
||||
new DeleteAction().execute(input);
|
||||
QContext.getQInstance().getTables().remove(name);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The scratch name prefix of a run: qqq_verify_ plus the run key made safe.
|
||||
*******************************************************************************/
|
||||
static String prefix(String runKey)
|
||||
{
|
||||
return ("qqq_verify_" + runKey.replaceAll("[^A-Za-z0-9]", "_").toLowerCase());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.verify;
|
||||
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.DigestInputStream;
|
||||
import java.security.MessageDigest;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
import com.kingsrook.qbits.backups.codec.HashingOutputStream;
|
||||
import com.kingsrook.qbits.backups.manifest.BackupManifest;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reads one stored part end to end the way a restore would: from storage,
|
||||
** through the decrypt hook, through gunzip, and hands each line to a consumer,
|
||||
** while hashing the stored bytes and the decoded bytes.
|
||||
*******************************************************************************/
|
||||
final class PartScanner
|
||||
{
|
||||
/***************************************************************************
|
||||
** What a scan saw.
|
||||
**
|
||||
** @param lines the number of lines handed to the consumer (0 when none was given)
|
||||
** @param sha256Stored SHA-256 of the bytes in the bucket
|
||||
** @param sha256Raw SHA-256 of the bytes after decrypting and decompressing
|
||||
***************************************************************************/
|
||||
record Scan(long lines, String sha256Stored, String sha256Raw)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Receives each line of a part (no trailing newline).
|
||||
***************************************************************************/
|
||||
@FunctionalInterface
|
||||
interface LineConsumer
|
||||
{
|
||||
/***************************************************************************
|
||||
** One line.
|
||||
***************************************************************************/
|
||||
void accept(String line) throws QException;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private PartScanner()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Scan a part. With no consumer the part is read to its end and hashed only (a
|
||||
** CSV part, where lines are not rows).
|
||||
*******************************************************************************/
|
||||
static Scan scan(RunBundle bundle, BackupManifest.Part part, LineConsumer consumer) throws IOException, QException
|
||||
{
|
||||
MessageDigest storedDigest = HashingOutputStream.newSha256();
|
||||
MessageDigest rawDigest = HashingOutputStream.newSha256();
|
||||
long lines = 0;
|
||||
|
||||
try(InputStream fromStorage = bundle.storage().openRead(bundle.runPrefix() + part.key());
|
||||
DigestInputStream stored = new DigestInputStream(fromStorage, storedDigest))
|
||||
{
|
||||
InputStream plain = part.encrypted() ? bundle.transform().unwrap(stored) : stored;
|
||||
InputStream decompressed = "GZIP".equals(part.compression()) ? new GZIPInputStream(plain) : plain;
|
||||
|
||||
try(DigestInputStream raw = new DigestInputStream(decompressed, rawDigest))
|
||||
{
|
||||
if(consumer == null)
|
||||
{
|
||||
raw.transferTo(OutputStream.nullOutputStream());
|
||||
}
|
||||
else
|
||||
{
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(raw, StandardCharsets.UTF_8));
|
||||
String line;
|
||||
while((line = reader.readLine()) != null)
|
||||
{
|
||||
lines++;
|
||||
consumer.accept(line);
|
||||
}
|
||||
}
|
||||
// anything after the decoded stream (a gzip trailer, padding) still counts toward the stored hash
|
||||
stored.transferTo(OutputStream.nullOutputStream());
|
||||
}
|
||||
}
|
||||
|
||||
return (new Scan(lines, HashingOutputStream.hex(storedDigest.digest()), HashingOutputStream.hex(rawDigest.digest())));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.verify;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The RDBMS_SCHEMA scratch target (design spec section 10.1): a configured scratch
|
||||
** backend whose dedicated database user may CREATE SCHEMA and DROP SCHEMA only for
|
||||
** qqq_verify_*. The qbit creates qqq_verify_(runKey), builds each table there
|
||||
** from a DDL-only copy generated from QQQ metadata (columns by type, primary key,
|
||||
** unique keys), restores through a temporary in-memory QTableMetaData clone that
|
||||
** points at the scratch schema, and drops the schema.
|
||||
**
|
||||
** STUB / TODO: this needs the restore agent's writer (the restore-mode insert,
|
||||
** QRun-IO/qqq PR #1111 RestoreWriteFlag, and the table writer built on it) and the
|
||||
** schema DDL generator. Until those land, create() refuses with a clear message,
|
||||
** so a plan that selects this target fails its TEST_RESTORE verification with an
|
||||
** ERROR instead of silently passing. Replace the body with the real
|
||||
** implementation when the restore writer is available.
|
||||
*******************************************************************************/
|
||||
public class RdbmsSchemaScratchTarget implements ScratchTarget
|
||||
{
|
||||
public static final String TYPE = "RDBMS_SCHEMA";
|
||||
|
||||
private final String backendName;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor.
|
||||
**
|
||||
** @param backendName the configured scratch RDBMS backend
|
||||
*******************************************************************************/
|
||||
public RdbmsSchemaScratchTarget(String backendName)
|
||||
{
|
||||
this.backendName = backendName;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String type()
|
||||
{
|
||||
return (TYPE);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String describe(String runKey)
|
||||
{
|
||||
return (backendName + ":schema=qqq_verify_" + runKey.replaceAll("[^A-Za-z0-9]", "_").toLowerCase());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ScratchTable create(QTableMetaData source, String runKey) throws QException
|
||||
{
|
||||
// TODO: CREATE SCHEMA, generate the DDL-only table copy, and return a ScratchTable that restores
|
||||
// through the restore writer. Needs the restore agent's writer.
|
||||
throw (new QException("The RDBMS_SCHEMA scratch target is not implemented yet (it needs the restore writer); use the MEMORY scratch target."));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void drop(String runKey) throws QException
|
||||
{
|
||||
// TODO: DROP SCHEMA qqq_verify_(runKey) once create() is implemented.
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,367 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.verify;
|
||||
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.BufferedReader;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.kingsrook.qbits.backups.BackupsRuntime;
|
||||
import com.kingsrook.qbits.backups.codec.ArtifactTransform;
|
||||
import com.kingsrook.qbits.backups.codec.HashingOutputStream;
|
||||
import com.kingsrook.qbits.backups.manifest.BackupManifest;
|
||||
import com.kingsrook.qbits.backups.storage.BackupStorage;
|
||||
import com.kingsrook.qbits.backups.storage.StoredObjectInfo;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Everything a verification needs to know about one run, loaded once: the run
|
||||
** row, its storage, its manifest (read and hashed), the transform that undoes
|
||||
** the plan's encryption, and (lazily) the file index.
|
||||
*******************************************************************************/
|
||||
class RunBundle
|
||||
{
|
||||
private static final ObjectMapper MAPPER = new ObjectMapper();
|
||||
|
||||
private final QRecord run;
|
||||
private final QRecord plan;
|
||||
private final BackupStorage storage;
|
||||
private final String runPrefix;
|
||||
private final BackupManifest manifest;
|
||||
private final ArtifactTransform transform;
|
||||
private final boolean encrypted;
|
||||
|
||||
private List<FileIndexEntry> fileIndex;
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** One line of the per-run file index.
|
||||
**
|
||||
** @param table the referencing table
|
||||
** @param recordKey the referencing record's primary key
|
||||
** @param field the referencing field
|
||||
** @param sha256 the content hash, null when not known
|
||||
** @param size the size in bytes, -1 when not known
|
||||
** @param status COPIED, DEDUPLICATED, MISSING or REFERENCE_ONLY
|
||||
** @param casKey the content-addressed key in the bucket, null when not stored
|
||||
***************************************************************************/
|
||||
record FileIndexEntry(String table, String recordKey, String field, String sha256, long size, String status, String casKey)
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Whether the file is in the file store (so it can be checked).
|
||||
*******************************************************************************/
|
||||
boolean isStored()
|
||||
{
|
||||
return (casKey != null && sha256 != null && ("COPIED".equals(status) || "DEDUPLICATED".equals(status)));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor.
|
||||
*******************************************************************************/
|
||||
private RunBundle(QRecord run, QRecord plan, BackupStorage storage, String runPrefix, BackupManifest manifest, ArtifactTransform transform, boolean encrypted)
|
||||
{
|
||||
this.run = run;
|
||||
this.plan = plan;
|
||||
this.storage = storage;
|
||||
this.runPrefix = runPrefix;
|
||||
this.manifest = manifest;
|
||||
this.transform = transform;
|
||||
this.encrypted = encrypted;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Load a run. Returns null, after recording a mismatch, when the run has no
|
||||
** readable manifest (so it is not usable and there is nothing more to check).
|
||||
** Throws VerifyException when the run cannot be examined at all.
|
||||
*******************************************************************************/
|
||||
static RunBundle load(QRecord run, QRecord plan, VerifyFindings findings)
|
||||
{
|
||||
String alias = run.getValueString("storageBucketAlias");
|
||||
String runPrefix = run.getValueString("runPrefix");
|
||||
if(alias == null || runPrefix == null)
|
||||
{
|
||||
throw (new VerifyException("The run has no storage bucket or run prefix recorded"));
|
||||
}
|
||||
BackupStorage storage = BackupsRuntime.storageFor(alias);
|
||||
|
||||
String manifestKey = run.getValueString("manifestKey") != null ? run.getValueString("manifestKey") : runPrefix + "manifest.json";
|
||||
Optional<StoredObjectInfo> head = storage.head(manifestKey);
|
||||
if(head.isEmpty())
|
||||
{
|
||||
findings.mismatch("The manifest is missing from the bucket, so this backup is not usable");
|
||||
return (null);
|
||||
}
|
||||
|
||||
byte[] bytes;
|
||||
try(InputStream in = storage.openRead(manifestKey))
|
||||
{
|
||||
bytes = in.readAllBytes();
|
||||
}
|
||||
catch(IOException e)
|
||||
{
|
||||
throw (new VerifyException("The manifest could not be read: " + e.getMessage(), e));
|
||||
}
|
||||
|
||||
String sha = HashingOutputStream.sha256Hex(bytes);
|
||||
if(run.getValueString("manifestSha256") != null && !run.getValueString("manifestSha256").equals(sha))
|
||||
{
|
||||
findings.mismatch("The manifest changed after it was written (its SHA-256 no longer matches the catalog)");
|
||||
}
|
||||
|
||||
BackupManifest manifest;
|
||||
try
|
||||
{
|
||||
manifest = BackupManifest.read(bytes);
|
||||
}
|
||||
catch(IOException e)
|
||||
{
|
||||
findings.mismatch("The manifest is not valid JSON: " + e.getMessage());
|
||||
return (null);
|
||||
}
|
||||
if(manifest.tables() == null)
|
||||
{
|
||||
findings.mismatch("The manifest lists no tables section");
|
||||
return (null);
|
||||
}
|
||||
|
||||
boolean encrypted = manifest.encryption() != null && manifest.encryption().mode() != null && !"NONE".equals(manifest.encryption().mode());
|
||||
ArtifactTransform transform = ArtifactTransform.IDENTITY;
|
||||
if(encrypted)
|
||||
{
|
||||
String keyRef = manifest.encryption().keyRef();
|
||||
var config = BackupsRuntime.getConfig();
|
||||
if(config == null || config.getArtifactTransformResolver() == null)
|
||||
{
|
||||
throw (new VerifyException("This backup is encrypted but no decrypt hook is configured (BackupsQBitConfig.withArtifactTransformResolver), so it cannot be verified"));
|
||||
}
|
||||
transform = config.getArtifactTransformResolver().apply(keyRef);
|
||||
if(transform == null)
|
||||
{
|
||||
throw (new VerifyException("No key is available for the key reference " + keyRef + ", so this backup cannot be decrypted"));
|
||||
}
|
||||
}
|
||||
return (new RunBundle(run, plan, storage, runPrefix, manifest, transform, encrypted));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The distinct stored files the run's file index references, in first-seen order
|
||||
** (one entry per hash). Reads the index once.
|
||||
*******************************************************************************/
|
||||
synchronized List<FileIndexEntry> distinctStoredFiles(VerifyFindings findings)
|
||||
{
|
||||
if(fileIndex == null)
|
||||
{
|
||||
fileIndex = readIndex(findings);
|
||||
}
|
||||
Map<String, FileIndexEntry> bySha = new LinkedHashMap<>();
|
||||
for(FileIndexEntry entry : fileIndex)
|
||||
{
|
||||
if(entry.isStored())
|
||||
{
|
||||
bySha.putIfAbsent(entry.sha256(), entry);
|
||||
}
|
||||
}
|
||||
return (new ArrayList<>(bySha.values()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every index entry, stored or not.
|
||||
*******************************************************************************/
|
||||
synchronized List<FileIndexEntry> allFileEntries(VerifyFindings findings)
|
||||
{
|
||||
if(fileIndex == null)
|
||||
{
|
||||
fileIndex = readIndex(findings);
|
||||
}
|
||||
return (fileIndex);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the file index objects. The index is gzip JSON Lines; it is read as plain
|
||||
** gzip first and, if that is not readable and the plan encrypts, through the
|
||||
** decrypt hook.
|
||||
*******************************************************************************/
|
||||
private List<FileIndexEntry> readIndex(VerifyFindings findings)
|
||||
{
|
||||
List<FileIndexEntry> entries = new ArrayList<>();
|
||||
if(manifest.files() == null || manifest.files().indexKeys() == null)
|
||||
{
|
||||
return (entries);
|
||||
}
|
||||
for(String relative : manifest.files().indexKeys())
|
||||
{
|
||||
String key = runPrefix + relative;
|
||||
if(storage.head(key).isEmpty())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
byte[] bytes;
|
||||
try(InputStream in = storage.openRead(key))
|
||||
{
|
||||
bytes = in.readAllBytes();
|
||||
}
|
||||
try
|
||||
{
|
||||
parseIndex(new GZIPInputStream(new ByteArrayInputStream(bytes)), entries);
|
||||
}
|
||||
catch(IOException plainFailed)
|
||||
{
|
||||
if(!encrypted)
|
||||
{
|
||||
throw (plainFailed);
|
||||
}
|
||||
entries.clear();
|
||||
parseIndex(new GZIPInputStream(transform.unwrap(new ByteArrayInputStream(bytes))), entries);
|
||||
}
|
||||
}
|
||||
catch(IOException e)
|
||||
{
|
||||
findings.mismatch("The file index " + relative + " cannot be read: " + e.getClass().getSimpleName());
|
||||
}
|
||||
}
|
||||
return (entries);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private void parseIndex(InputStream in, List<FileIndexEntry> entries) throws IOException
|
||||
{
|
||||
try(BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8)))
|
||||
{
|
||||
String line;
|
||||
while((line = reader.readLine()) != null)
|
||||
{
|
||||
if(line.isBlank())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
JsonNode node = MAPPER.readTree(line);
|
||||
entries.add(new FileIndexEntry(text(node, "table"), text(node, "recordKey"), text(node, "field"), text(node, "sha256"),
|
||||
node.hasNonNull("size") ? node.get("size").asLong() : -1L, text(node, "status"), text(node, "casKey")));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static String text(JsonNode node, String name)
|
||||
{
|
||||
return (node.hasNonNull(name) ? node.get(name).asText() : null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for run
|
||||
*******************************************************************************/
|
||||
QRecord run()
|
||||
{
|
||||
return (run);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for plan (null for an ad-hoc run)
|
||||
*******************************************************************************/
|
||||
QRecord plan()
|
||||
{
|
||||
return (plan);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for storage
|
||||
*******************************************************************************/
|
||||
BackupStorage storage()
|
||||
{
|
||||
return (storage);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for runPrefix
|
||||
*******************************************************************************/
|
||||
String runPrefix()
|
||||
{
|
||||
return (runPrefix);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for manifest
|
||||
*******************************************************************************/
|
||||
BackupManifest manifest()
|
||||
{
|
||||
return (manifest);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for transform (the decrypt hook, or the identity)
|
||||
*******************************************************************************/
|
||||
ArtifactTransform transform()
|
||||
{
|
||||
return (transform);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether the run's artifacts are encrypted.
|
||||
*******************************************************************************/
|
||||
boolean isEncrypted()
|
||||
{
|
||||
return (encrypted);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.verify;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import java.util.function.Consumer;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One restored table inside a scratch target.
|
||||
*******************************************************************************/
|
||||
public interface ScratchTable
|
||||
{
|
||||
/***************************************************************************
|
||||
** Restore rows into the scratch table, preserving their primary keys.
|
||||
***************************************************************************/
|
||||
void insert(List<QRecord> records) throws QException;
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Read every row back in primary key order, a page at a time.
|
||||
***************************************************************************/
|
||||
void readInPrimaryKeyOrder(int pageSize, Consumer<QRecord> consumer) throws QException;
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.verify;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Where a TEST_RESTORE verification restores to (design spec section 10.1). The
|
||||
** target builds a scratch copy of each table from the source table's metadata,
|
||||
** takes the restored rows, reads them back in primary key order, and is dropped
|
||||
** afterwards. Nothing a scratch target does may touch the real tables.
|
||||
**
|
||||
** Implementations: MEMORY (MemoryScratchTarget, small apps and CI) and RDBMS_SCHEMA
|
||||
** (RdbmsSchemaScratchTarget, a stub until the restore agent's writer ships).
|
||||
*******************************************************************************/
|
||||
public interface ScratchTarget
|
||||
{
|
||||
/***************************************************************************
|
||||
** The target type: MEMORY, RDBMS_SCHEMA or SEPARATE_DATABASE.
|
||||
***************************************************************************/
|
||||
String type();
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** A description for backups_verification.scratchTarget, for example
|
||||
** "memory:qqq_verify_20261012t020000z_7f3a".
|
||||
***************************************************************************/
|
||||
String describe(String runKey);
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Create the scratch copy of a table for a run.
|
||||
***************************************************************************/
|
||||
ScratchTable create(QTableMetaData source, String runKey) throws QException;
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Drop everything created for the run. Safe to call more than once.
|
||||
***************************************************************************/
|
||||
void drop(String runKey) throws QException;
|
||||
}
|
||||
@@ -0,0 +1,282 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.verify;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.backups.BackupsRuntime;
|
||||
import com.kingsrook.qbits.backups.codec.ArtifactTransform;
|
||||
import com.kingsrook.qbits.backups.codec.HashingOutputStream;
|
||||
import com.kingsrook.qbits.backups.codec.JsonlRowCodec;
|
||||
import com.kingsrook.qbits.backups.manifest.BackupManifest;
|
||||
import com.kingsrook.qbits.backups.storage.BackupStorage;
|
||||
import com.kingsrook.qbits.backups.storage.WriteOptions;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** TEST_RESTORE verification (design spec section 10.1): restore every table of the
|
||||
** run into a scratch target (MEMORY by default), read it back in primary key order,
|
||||
** and compare each table's row count and rowsHash with the manifest; restore the
|
||||
** run's files into the scratch prefix qqq-verify/(runKey)/ in the backup bucket,
|
||||
** re-hash each one, and check that every file reference of the index resolves to a
|
||||
** restored file. Then drop the scratch tables and delete the scratch files, even
|
||||
** when something fails.
|
||||
**
|
||||
** The scratch target is BackupsQBitConfig.getScratchTarget(); the MEMORY target
|
||||
** proves the files and the codecs, not the database. RDBMS_SCHEMA is a stub until
|
||||
** the restore agent's writer ships (see RdbmsSchemaScratchTarget). Sequence resync
|
||||
** is part of the real restore (spec 5.4) and not exercised by the MEMORY target.
|
||||
*******************************************************************************/
|
||||
public class TestRestoreVerifier
|
||||
{
|
||||
private static final int PAGE_SIZE = 1000;
|
||||
|
||||
public static final String SCRATCH_FILE_PREFIX = "qqq-verify/";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Run the test restore, recording what it finds. Returns the scratch target
|
||||
** description for backups_verification.scratchTarget.
|
||||
*******************************************************************************/
|
||||
String verify(RunBundle bundle, VerifyFindings findings) throws QException
|
||||
{
|
||||
ScratchTarget target = BackupsRuntime.getConfig() != null && BackupsRuntime.getConfig().getScratchTarget() != null
|
||||
? BackupsRuntime.getConfig().getScratchTarget() : new MemoryScratchTarget();
|
||||
String runKey = bundle.manifest().runKey();
|
||||
String scratch = target.describe(runKey);
|
||||
|
||||
List<String> scratchFileKeys = new ArrayList<>();
|
||||
try
|
||||
{
|
||||
findings.tablesChecked = bundle.manifest().tables().size();
|
||||
restoreTables(bundle, target, runKey, findings);
|
||||
restoreFiles(bundle, runKey, scratchFileKeys, findings);
|
||||
}
|
||||
finally
|
||||
{
|
||||
try
|
||||
{
|
||||
target.drop(runKey);
|
||||
}
|
||||
finally
|
||||
{
|
||||
for(String key : scratchFileKeys)
|
||||
{
|
||||
bundle.storage().deletePermanent(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
return (scratch);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Restore each table into the scratch target and compare it with the manifest.
|
||||
*******************************************************************************/
|
||||
private void restoreTables(RunBundle bundle, ScratchTarget target, String runKey, VerifyFindings findings) throws QException
|
||||
{
|
||||
BackupManifest manifest = bundle.manifest();
|
||||
List<String> order = new ArrayList<>(manifest.restoreOrder() == null ? List.of() : manifest.restoreOrder());
|
||||
for(BackupManifest.TableEntry entry : manifest.tables())
|
||||
{
|
||||
if(!order.contains(entry.name()))
|
||||
{
|
||||
order.add(entry.name());
|
||||
}
|
||||
}
|
||||
|
||||
for(String name : order)
|
||||
{
|
||||
BackupManifest.TableEntry entry = manifest.table(name);
|
||||
if(entry == null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
QTableMetaData table = QContext.getQInstance().getTable(name);
|
||||
if(table == null)
|
||||
{
|
||||
findings.mismatch("Table " + name + " no longer exists in this instance, so it cannot be test-restored");
|
||||
continue;
|
||||
}
|
||||
boolean hasJsonl = entry.parts() != null && entry.parts().stream().anyMatch(p -> !"CSV".equals(p.format()));
|
||||
if(!hasJsonl)
|
||||
{
|
||||
findings.warn("Table " + name + " has only CSV parts and was not test-restored");
|
||||
continue;
|
||||
}
|
||||
|
||||
List<String> excluded = entry.excludedFields() == null ? List.of() : entry.excludedFields().stream().map(BackupManifest.ExcludedField::name).toList();
|
||||
JsonlRowCodec codec = new JsonlRowCodec(table, excluded);
|
||||
ScratchTable scratchTable = target.create(table, runKey);
|
||||
|
||||
int before = findings.mismatches;
|
||||
restoreRows(bundle, entry, codec, scratchTable, findings);
|
||||
|
||||
MessageDigest digest = HashingOutputStream.newSha256();
|
||||
long[] count = { 0 };
|
||||
scratchTable.readInPrimaryKeyOrder(PAGE_SIZE, record ->
|
||||
{
|
||||
count[0]++;
|
||||
digest.update((codec.encode(record) + "\n").getBytes(StandardCharsets.UTF_8));
|
||||
});
|
||||
String actualHash = "sha256:" + HashingOutputStream.hex(digest.digest());
|
||||
|
||||
if(count[0] != entry.rowCount())
|
||||
{
|
||||
findings.mismatch("Test restore of table " + name + " has " + count[0] + " rows, the manifest says " + entry.rowCount());
|
||||
}
|
||||
if(entry.rowsHash() != null && !entry.rowsHash().equals(actualHash))
|
||||
{
|
||||
findings.mismatch("Test restore of table " + name + " does not reproduce the backed up rows (row content hash differs)");
|
||||
}
|
||||
findings.rowsChecked += count[0];
|
||||
|
||||
Map<String, Object> detail = new LinkedHashMap<>();
|
||||
detail.put("table", name);
|
||||
detail.put("expectedRows", entry.rowCount());
|
||||
detail.put("actualRows", count[0]);
|
||||
detail.put("expectedRowsHash", entry.rowsHash());
|
||||
detail.put("actualRowsHash", actualHash);
|
||||
detail.put("ok", findings.mismatches == before);
|
||||
findings.tables.add(detail);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Decode the JSONL parts of one table and insert them into the scratch table.
|
||||
*******************************************************************************/
|
||||
private void restoreRows(RunBundle bundle, BackupManifest.TableEntry entry, JsonlRowCodec codec, ScratchTable scratchTable, VerifyFindings findings) throws QException
|
||||
{
|
||||
for(BackupManifest.Part part : entry.parts())
|
||||
{
|
||||
if("CSV".equals(part.format()))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
List<QRecord> batch = new ArrayList<>();
|
||||
try
|
||||
{
|
||||
PartScanner.scan(bundle, part, line ->
|
||||
{
|
||||
batch.add(codec.decode(line));
|
||||
if(batch.size() >= PAGE_SIZE)
|
||||
{
|
||||
scratchTable.insert(new ArrayList<>(batch));
|
||||
batch.clear();
|
||||
}
|
||||
});
|
||||
if(!batch.isEmpty())
|
||||
{
|
||||
scratchTable.insert(batch);
|
||||
}
|
||||
}
|
||||
catch(IOException | RuntimeException e)
|
||||
{
|
||||
findings.mismatch("The part " + part.key() + " could not be restored (" + e.getClass().getSimpleName() + "); it is corrupted, or the key is wrong");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Restore the run's files into the scratch prefix, re-hash each, and check that
|
||||
** every index reference resolves to a restored file. verifyFileSamplePercent
|
||||
** (on the plan, default 100) limits how many files are restored.
|
||||
*******************************************************************************/
|
||||
private void restoreFiles(RunBundle bundle, String runKey, List<String> scratchFileKeys, VerifyFindings findings)
|
||||
{
|
||||
int percent = 100;
|
||||
if(bundle.plan() != null && bundle.plan().getValueInteger("verifyFileSamplePercent") != null)
|
||||
{
|
||||
percent = Math.max(0, Math.min(100, bundle.plan().getValueInteger("verifyFileSamplePercent")));
|
||||
}
|
||||
|
||||
BackupStorage storage = bundle.storage();
|
||||
String prefix = SCRATCH_FILE_PREFIX + runKey + "/";
|
||||
Set<String> restored = new LinkedHashSet<>();
|
||||
int sampled = 0;
|
||||
for(RunBundle.FileIndexEntry entry : bundle.distinctStoredFiles(findings))
|
||||
{
|
||||
if(Math.floorMod(entry.sha256().hashCode(), 100) >= percent)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
sampled++;
|
||||
String scratchKey = prefix + entry.sha256();
|
||||
scratchFileKeys.add(scratchKey);
|
||||
try
|
||||
{
|
||||
ArtifactTransform transform = bundle.isEncrypted() ? bundle.transform() : ArtifactTransform.IDENTITY;
|
||||
String written;
|
||||
try(InputStream raw = storage.openRead(entry.casKey());
|
||||
InputStream plain = transform.unwrap(raw);
|
||||
HashingOutputStream out = new HashingOutputStream(storage.openWrite(scratchKey, WriteOptions.create())))
|
||||
{
|
||||
plain.transferTo(out);
|
||||
out.close();
|
||||
written = out.getSha256();
|
||||
}
|
||||
String reread = BackupVerifier.hashObject(storage, scratchKey, ArtifactTransform.IDENTITY);
|
||||
if(written.equals(entry.sha256()) && reread.equals(entry.sha256()))
|
||||
{
|
||||
restored.add(entry.sha256());
|
||||
}
|
||||
else
|
||||
{
|
||||
findings.mismatch("Test restore of file sha256 " + entry.sha256() + " does not reproduce its content");
|
||||
}
|
||||
}
|
||||
catch(IOException | RuntimeException e)
|
||||
{
|
||||
findings.mismatch("Test restore of file sha256 " + entry.sha256() + " failed (" + e.getClass().getSimpleName() + ")");
|
||||
}
|
||||
}
|
||||
|
||||
int unresolved = 0;
|
||||
for(RunBundle.FileIndexEntry entry : bundle.allFileEntries(findings))
|
||||
{
|
||||
if(entry.isStored() && Math.floorMod(entry.sha256().hashCode(), 100) < percent && !restored.contains(entry.sha256()))
|
||||
{
|
||||
unresolved++;
|
||||
}
|
||||
}
|
||||
if(unresolved > 0)
|
||||
{
|
||||
findings.mismatch(unresolved + " file references in the index do not resolve to a restored file");
|
||||
}
|
||||
findings.filesChecked += sampled;
|
||||
findings.extra.put("filesRestored", restored.size());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.verify;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The result of verifying one run at one level; also what was written to the
|
||||
** backups_verification row.
|
||||
**
|
||||
** @param verificationId the backups_verification id
|
||||
** @param runId the backups_run id
|
||||
** @param level STORAGE, CONTENT or TEST_RESTORE
|
||||
** @param status PASSED, PASSED_WITH_WARNINGS, FAILED or ERROR
|
||||
** @param tablesChecked tables checked
|
||||
** @param rowsChecked rows decoded and counted
|
||||
** @param mismatches how many things did not match (the messages list is capped)
|
||||
** @param objectsChecked storage objects examined
|
||||
** @param filesChecked file-store objects examined (head, or read and re-hashed)
|
||||
** @param filesSkipped file-store objects skipped because they were verified recently
|
||||
** @param messages the first mismatches and warnings, in plain words
|
||||
** @param scratchTarget for TEST_RESTORE, the scratch target used
|
||||
*******************************************************************************/
|
||||
public record VerificationOutcome(Integer verificationId, Integer runId, String level, String status, int tablesChecked, long rowsChecked, int mismatches,
|
||||
int objectsChecked, int filesChecked, int filesSkipped, List<String> messages, String scratchTarget)
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Whether the verification passed (with or without warnings).
|
||||
*******************************************************************************/
|
||||
public boolean passed()
|
||||
{
|
||||
return ("PASSED".equals(status) || "PASSED_WITH_WARNINGS".equals(status));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.verify;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A verification could not be carried out (as opposed to carried out and found
|
||||
** a mismatch): the storage is unreachable, there is no key to decrypt with. The
|
||||
** verification ends with status ERROR.
|
||||
*******************************************************************************/
|
||||
public class VerifyException extends RuntimeException
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Constructor.
|
||||
*******************************************************************************/
|
||||
public VerifyException(String message)
|
||||
{
|
||||
super(message);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor.
|
||||
*******************************************************************************/
|
||||
public VerifyException(String message, Throwable cause)
|
||||
{
|
||||
super(message, cause);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.verify;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** What one verification found: counts, the first mismatches and warnings (capped,
|
||||
** so a ruined bucket does not write megabytes of messages), and per-table detail
|
||||
** for backups_verification.detailJson. Messages never contain row values.
|
||||
*******************************************************************************/
|
||||
class VerifyFindings
|
||||
{
|
||||
static final int MAX_MESSAGES = 100;
|
||||
|
||||
final List<String> mismatchMessages = new ArrayList<>();
|
||||
final List<String> warnings = new ArrayList<>();
|
||||
final List<Map<String, Object>> tables = new ArrayList<>();
|
||||
final Map<String, Object> extra = new LinkedHashMap<>();
|
||||
|
||||
int mismatches;
|
||||
int tablesChecked;
|
||||
long rowsChecked;
|
||||
int objectsChecked;
|
||||
int filesChecked;
|
||||
int filesSkipped;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Something did not match.
|
||||
*******************************************************************************/
|
||||
void mismatch(String message)
|
||||
{
|
||||
mismatches++;
|
||||
if(mismatchMessages.size() < MAX_MESSAGES)
|
||||
{
|
||||
mismatchMessages.add(message);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Something worth knowing that is not a failure.
|
||||
*******************************************************************************/
|
||||
void warn(String message)
|
||||
{
|
||||
if(warnings.size() < MAX_MESSAGES)
|
||||
{
|
||||
warnings.add(message);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The mismatches, then the warnings.
|
||||
*******************************************************************************/
|
||||
List<String> messages()
|
||||
{
|
||||
List<String> all = new ArrayList<>(mismatchMessages);
|
||||
all.addAll(warnings);
|
||||
return (all);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** PASSED, PASSED_WITH_WARNINGS or FAILED.
|
||||
*******************************************************************************/
|
||||
String status()
|
||||
{
|
||||
if(mismatches > 0)
|
||||
{
|
||||
return ("FAILED");
|
||||
}
|
||||
return (warnings.isEmpty() ? "PASSED" : "PASSED_WITH_WARNINGS");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups;
|
||||
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneId;
|
||||
import java.time.ZoneOffset;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A synthetic clock for tests: it stands still until it is advanced, so SLA
|
||||
** windows can be crossed without waiting.
|
||||
*******************************************************************************/
|
||||
public class MutableClock extends Clock
|
||||
{
|
||||
private Instant now;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor.
|
||||
*******************************************************************************/
|
||||
public MutableClock(Instant start)
|
||||
{
|
||||
this.now = start;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Move the clock forward.
|
||||
*******************************************************************************/
|
||||
public synchronized MutableClock advance(Duration duration)
|
||||
{
|
||||
now = now.plus(duration);
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ZoneId getZone()
|
||||
{
|
||||
return (ZoneOffset.UTC);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public Clock withZone(ZoneId zone)
|
||||
{
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public synchronized Instant instant()
|
||||
{
|
||||
return (now);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.backups.alerts.BackupAlertSink;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A test double for the alert sink: remembers every call, and which alerts are
|
||||
** currently open (raised and not resolved), by dedupe key.
|
||||
*******************************************************************************/
|
||||
public class RecordingAlertSink implements BackupAlertSink
|
||||
{
|
||||
/** One raise call. */
|
||||
public record Raised(String type, String dedupeKey, Severity severity, String title, Map<String, String> detail)
|
||||
{
|
||||
}
|
||||
|
||||
public final List<Raised> raises = new ArrayList<>();
|
||||
public final List<String> resolves = new ArrayList<>();
|
||||
public final List<String> heartbeats = new ArrayList<>();
|
||||
public final Map<String, Raised> open = new LinkedHashMap<>();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void raise(String type, String dedupeKey, Severity severity, String title, Map<String, String> detail)
|
||||
{
|
||||
Raised raised = new Raised(type, dedupeKey, severity, title, detail);
|
||||
raises.add(raised);
|
||||
open.put(dedupeKey, raised);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void resolve(String type, String dedupeKey)
|
||||
{
|
||||
resolves.add(dedupeKey);
|
||||
open.remove(dedupeKey);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void heartbeat(String monitorKey, long expectedEverySeconds, long graceSeconds)
|
||||
{
|
||||
heartbeats.add(monitorKey);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether an alert is open.
|
||||
*******************************************************************************/
|
||||
public boolean isOpen(String dedupeKey)
|
||||
{
|
||||
return (open.containsKey(dedupeKey));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The open alert, or null.
|
||||
*******************************************************************************/
|
||||
public Raised openAlert(String dedupeKey)
|
||||
{
|
||||
return (open.get(dedupeKey));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups;
|
||||
|
||||
|
||||
import java.io.FilterInputStream;
|
||||
import java.io.FilterOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.backups.codec.ArtifactTransform;
|
||||
import com.kingsrook.qbits.backups.events.BackupEvents;
|
||||
import com.kingsrook.qbits.backups.metrics.BackupsMetrics;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qbits.backups.run.BackupEngine;
|
||||
import com.kingsrook.qbits.backups.run.BackupRunFactory;
|
||||
import com.kingsrook.qbits.backups.run.BackupRunResult;
|
||||
import com.kingsrook.qbits.backups.sla.SlaChecker;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Shared setup for the verification, SLA, alert, health and event tests: the
|
||||
** blog instance, a synthetic clock, a recording alert sink, plans and runs.
|
||||
*******************************************************************************/
|
||||
public final class VerifyTestSupport
|
||||
{
|
||||
public static MutableClock clock;
|
||||
public static RecordingAlertSink sink;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private VerifyTestSupport()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fresh instance with posts, a synthetic clock starting now, and a recording sink.
|
||||
*******************************************************************************/
|
||||
public static void setUp(int posts) throws Exception
|
||||
{
|
||||
BackupsTestUtils.init();
|
||||
BackupsTestUtils.loadBlog(posts);
|
||||
SlaChecker.resetCaches();
|
||||
BackupsMetrics.resetCache();
|
||||
BackupEvents.setPublisher(null);
|
||||
clock = new MutableClock(Instant.now());
|
||||
sink = new RecordingAlertSink();
|
||||
BackupsRuntime.setClock(clock);
|
||||
BackupsRuntime.setAlertSink(sink);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Insert a nightly plan; overrides replace the defaults.
|
||||
*******************************************************************************/
|
||||
public static QRecord plan(Map<String, Serializable> overrides) throws Exception
|
||||
{
|
||||
Map<String, Serializable> values = new LinkedHashMap<>();
|
||||
values.put("name", "Nightly");
|
||||
values.put("planKey", "nightly");
|
||||
values.put("isActive", true);
|
||||
values.put("scopeType", "INSTANCE");
|
||||
values.put("storageBucketAlias", "backups");
|
||||
values.put("slaHours", 26);
|
||||
values.put("verifySlaDays", 8);
|
||||
values.putAll(overrides);
|
||||
Integer id = new BackupCatalog().insert(BackupsTables.PLAN, values);
|
||||
return (new BackupCatalog().get(BackupsTables.PLAN, id).orElseThrow());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The plan as it is now.
|
||||
*******************************************************************************/
|
||||
public static QRecord reload(QRecord plan) throws Exception
|
||||
{
|
||||
return (new BackupCatalog().get(BackupsTables.PLAN, plan.getValueInteger("id")).orElseThrow());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The run as it is now.
|
||||
*******************************************************************************/
|
||||
public static QRecord run(Integer runId) throws Exception
|
||||
{
|
||||
return (new BackupCatalog().get(BackupsTables.RUN, runId).orElseThrow());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Run the plan's backup.
|
||||
*******************************************************************************/
|
||||
public static BackupRunResult backup(QRecord plan) throws Exception
|
||||
{
|
||||
return (new BackupEngine().execute(BackupRunFactory.forPlan(plan, "MANUAL")));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Run the plan's backup encrypted with a transform.
|
||||
*******************************************************************************/
|
||||
public static BackupRunResult backupEncrypted(QRecord plan, ArtifactTransform transform, String keyRef) throws Exception
|
||||
{
|
||||
return (new BackupEngine().execute(BackupRunFactory.forPlan(plan, "MANUAL").withTransform(transform, keyRef, "fingerprint-" + keyRef)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The part keys (relative to the run prefix) of a table in a run's manifest.
|
||||
*******************************************************************************/
|
||||
public static String firstPartKey(BackupRunResult result, String table) throws Exception
|
||||
{
|
||||
var manifest = com.kingsrook.qbits.backups.manifest.BackupManifest.read(BackupsTestUtils.BACKUPS.getBytes(result.getManifestKey()));
|
||||
return (result.getRunPrefix() + manifest.table(table).parts().get(0).key());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** A toy "cipher" for tests: every byte is XORed with a key byte. Different
|
||||
** key bytes stand in for "the wrong key".
|
||||
***************************************************************************/
|
||||
public static class XorTransform implements ArtifactTransform
|
||||
{
|
||||
private final int key;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor.
|
||||
*******************************************************************************/
|
||||
public XorTransform(int key)
|
||||
{
|
||||
this.key = key;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public OutputStream wrap(OutputStream out)
|
||||
{
|
||||
return (new FilterOutputStream(out)
|
||||
{
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void write(int b) throws IOException
|
||||
{
|
||||
out.write((b ^ key) & 0xFF);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void write(byte[] b, int off, int len) throws IOException
|
||||
{
|
||||
for(int i = 0; i < len; i++)
|
||||
{
|
||||
write(b[off + i]);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public InputStream unwrap(InputStream in)
|
||||
{
|
||||
return (new FilterInputStream(in)
|
||||
{
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public int read() throws IOException
|
||||
{
|
||||
int b = super.read();
|
||||
return (b < 0 ? b : (b ^ key) & 0xFF);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public int read(byte[] b, int off, int len) throws IOException
|
||||
{
|
||||
int n = super.read(b, off, len);
|
||||
for(int i = 0; i < n; i++)
|
||||
{
|
||||
b[off + i] = (byte) ((b[off + i] ^ key) & 0xFF);
|
||||
}
|
||||
return (n);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean encrypts()
|
||||
{
|
||||
return (true);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String keySuffix()
|
||||
{
|
||||
return (".xor");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.alerts;
|
||||
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.backups.BackupsRuntime;
|
||||
import com.kingsrook.qbits.backups.VerifyTestSupport;
|
||||
import com.kingsrook.qbits.backups.alerts.fakeapi.AlertsInstanceMetaData;
|
||||
import com.kingsrook.qbits.backups.alerts.fakeapi.QAlertTypeMetaData;
|
||||
import com.kingsrook.qbits.backups.alerts.fakeapi.QAlerts;
|
||||
import com.kingsrook.qbits.backups.sla.SlaChecker;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The Alerts qbit adapter, bound reflectively. The SPI is not in QQQ 4.1.0-RC.4,
|
||||
** so these tests run it against a test double with the same shape as the real
|
||||
** com.kingsrook.qqq.backend.core.alerts package (QRun-IO/qqq PR #1108).
|
||||
*******************************************************************************/
|
||||
class AlertsQBitAlertSinkTest
|
||||
{
|
||||
private static final String FAKE_API = "com.kingsrook.qbits.backups.alerts.fakeapi";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp() throws Exception
|
||||
{
|
||||
VerifyTestSupport.setUp(3);
|
||||
QAlerts.reset();
|
||||
QAlertTypeMetaData.ADDED.clear();
|
||||
AlertsInstanceMetaData.installed = true;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@AfterEach
|
||||
void tearDown()
|
||||
{
|
||||
AlertsInstanceMetaData.installed = false;
|
||||
QAlerts.reset();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Raise, resolve and heartbeat reach the SPI with the stable dedupe key, the
|
||||
** severity mapped by name, and the detail as the summary and attributes.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRaiseResolveHeartbeat()
|
||||
{
|
||||
AlertsQBitAlertSink sink = new AlertsQBitAlertSink(FAKE_API);
|
||||
Map<String, String> detail = new LinkedHashMap<>();
|
||||
detail.put("plan", "nightly");
|
||||
detail.put("slaHours", "26");
|
||||
|
||||
sink.raise("backups.plan.missed", "backups.plan.missed:nightly", BackupAlertSink.Severity.CRITICAL, "A backup did not run in time", detail);
|
||||
sink.resolve("backups.plan.missed", "backups.plan.missed:nightly");
|
||||
sink.heartbeat("backups.plan:nightly", 93600, 0);
|
||||
|
||||
assertThat(QAlerts.RAISED).hasSize(1);
|
||||
var signal = QAlerts.RAISED.get(0);
|
||||
assertThat(signal.alertTypeName).isEqualTo("backups.plan.missed");
|
||||
assertThat(signal.dedupKey).isEqualTo("backups.plan.missed:nightly");
|
||||
assertThat(signal.severity.name()).isEqualTo("CRITICAL");
|
||||
assertThat(signal.title).isEqualTo("A backup did not run in time");
|
||||
assertThat(signal.summary).isEqualTo("plan=nightly, slaHours=26");
|
||||
assertThat(signal.attributes).containsEntry("plan", "nightly").containsEntry("slaHours", "26");
|
||||
assertThat(QAlerts.RESOLVED).containsExactly("backups.plan.missed|backups.plan.missed:nightly|Cleared by the Backups qbit");
|
||||
assertThat(QAlerts.HEARTBEATS).containsExactly("backups.plan:nightly");
|
||||
|
||||
sink.raise("backups.storage.debris", "k", BackupAlertSink.Severity.INFO, "no detail", null);
|
||||
assertThat(QAlerts.RAISED.get(1).summary).isNull();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Without the Alerts qbit installed (no alerts meta-data on the instance) every
|
||||
** call is a no-op.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testNoOpWithoutTheAlertsQBit()
|
||||
{
|
||||
AlertsInstanceMetaData.installed = false;
|
||||
AlertsQBitAlertSink sink = new AlertsQBitAlertSink(FAKE_API);
|
||||
sink.raise("t", "k", BackupAlertSink.Severity.INFO, "x", Map.of());
|
||||
sink.resolve("t", "k");
|
||||
sink.heartbeat("m", 1, 1);
|
||||
assertThat(QAlerts.RAISED).isEmpty();
|
||||
assertThat(QAlerts.RESOLVED).isEmpty();
|
||||
assertThat(QAlerts.HEARTBEATS).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A broken or missing SPI never throws into the backup.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testFailuresAreSwallowed()
|
||||
{
|
||||
AlertsQBitAlertSink missing = new AlertsQBitAlertSink("no.such.package");
|
||||
missing.raise("t", "k", BackupAlertSink.Severity.INFO, "x", Map.of());
|
||||
missing.resolve("t", "k");
|
||||
missing.heartbeat("m", 1, 1);
|
||||
assertThat(missing.registerAlertTypes(QContext.getQInstance())).isZero();
|
||||
|
||||
QAlerts.failWith = new IllegalStateException("alerts down");
|
||||
AlertsQBitAlertSink broken = new AlertsQBitAlertSink(FAKE_API);
|
||||
broken.raise("t", "k", BackupAlertSink.Severity.INFO, "x", Map.of());
|
||||
broken.resolve("t", "k");
|
||||
broken.heartbeat("m", 1, 1);
|
||||
assertThat(QAlerts.RAISED).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The backup alert types are declared with the Alerts qbit.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRegisterAlertTypes()
|
||||
{
|
||||
int added = new AlertsQBitAlertSink(FAKE_API).registerAlertTypes(QContext.getQInstance());
|
||||
assertThat(added).isEqualTo(BackupAlerts.alertTypes().size()).isEqualTo(9);
|
||||
assertThat(QAlertTypeMetaData.ADDED).extracting(t -> t.name).containsExactly(
|
||||
"backups.plan.missed", "backups.run.failed", "backups.run.missing_in_storage", "backups.files.missing", "backups.verify.failed",
|
||||
"backups.verify.overdue", "backups.plan.owner_invalid", "backups.storage.debris", "backups.restore.completed");
|
||||
assertThat(QAlertTypeMetaData.ADDED).allSatisfy(t ->
|
||||
{
|
||||
assertThat(t.groupName).isEqualTo("Backups");
|
||||
assertThat(t.howToFix).isNotBlank();
|
||||
assertThat(t.label).isNotBlank();
|
||||
});
|
||||
assertThat(QAlertTypeMetaData.ADDED.get(2).defaultSeverity.name()).isEqualTo("CRITICAL");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Detection: the SPI is present or not by class name; the real SPI is not in
|
||||
** RC.4, so nothing is bound by default there.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testDetectionAndBinding()
|
||||
{
|
||||
assertThat(AlertsQBitAlertSink.isPresent(FAKE_API)).isTrue();
|
||||
assertThat(AlertsQBitAlertSink.isPresent("no.such.package")).isFalse();
|
||||
|
||||
boolean realSpiPresent = AlertsQBitAlertSink.isPresent(AlertsQBitAlertSink.DEFAULT_API_PACKAGE);
|
||||
assertThat(AlertsQBitAlertSink.bindIfPresent()).isEqualTo(realSpiPresent);
|
||||
if(realSpiPresent)
|
||||
{
|
||||
assertThat(BackupsRuntime.getAlertSink()).isInstanceOf(AlertsQBitAlertSink.class);
|
||||
assertThat(AlertsQBitAlertSink.bindIfPresent()).as("an already bound sink is not replaced").isFalse();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** End to end: the SLA checker's alerts reach the Alerts qbit SPI and resolve.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testSlaAlertsThroughTheAdapter() throws Exception
|
||||
{
|
||||
BackupsRuntime.setAlertSink(new AlertsQBitAlertSink(FAKE_API));
|
||||
QRecord plan = VerifyTestSupport.plan(Map.of());
|
||||
VerifyTestSupport.backup(plan);
|
||||
assertThat(QAlerts.HEARTBEATS).contains("backups.plan:nightly");
|
||||
|
||||
VerifyTestSupport.clock.advance(Duration.ofHours(30));
|
||||
new SlaChecker().check(VerifyTestSupport.reload(plan));
|
||||
assertThat(QAlerts.RAISED).anySatisfy(s ->
|
||||
{
|
||||
assertThat(s.alertTypeName).isEqualTo("backups.plan.missed");
|
||||
assertThat(s.dedupKey).isEqualTo("backups.plan.missed:nightly");
|
||||
assertThat(s.severity.name()).isEqualTo("ERROR");
|
||||
});
|
||||
|
||||
VerifyTestSupport.backup(plan);
|
||||
new SlaChecker().check(VerifyTestSupport.reload(plan));
|
||||
assertThat(QAlerts.RESOLVED).anyMatch(r -> r.startsWith("backups.plan.missed|backups.plan.missed:nightly|"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.alerts;
|
||||
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import com.kingsrook.qbits.backups.BackupsRuntime;
|
||||
import com.kingsrook.qbits.backups.RecordingAlertSink;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The alert catalog and its helpers: the spec table, stable dedupe keys, and
|
||||
** helpers that never throw into the caller.
|
||||
*******************************************************************************/
|
||||
class BackupAlertsTest
|
||||
{
|
||||
private RecordingAlertSink sink;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp()
|
||||
{
|
||||
BackupsRuntime.reset();
|
||||
sink = new RecordingAlertSink();
|
||||
BackupsRuntime.setAlertSink(sink);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@AfterEach
|
||||
void tearDown()
|
||||
{
|
||||
BackupsRuntime.reset();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every alert of the spec table (section 10.4) is declared, once, with a default
|
||||
** severity and a way to fix it.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testTheSpecTable()
|
||||
{
|
||||
Set<String> names = BackupAlerts.alertTypes().stream().map(BackupAlerts.AlertType::name).collect(Collectors.toSet());
|
||||
assertThat(names).containsExactlyInAnyOrder("backups.plan.missed", "backups.run.failed", "backups.run.missing_in_storage", "backups.files.missing",
|
||||
"backups.verify.failed", "backups.verify.overdue", "backups.plan.owner_invalid", "backups.storage.debris", "backups.restore.completed");
|
||||
assertThat(BackupAlerts.alertTypes()).allSatisfy(t -> assertThat(t.howToFix()).isNotBlank());
|
||||
assertThat(BackupAlerts.alertTypes().stream().filter(t -> t.name().equals(BackupAlerts.RUN_MISSING)).findFirst().orElseThrow().defaultSeverity())
|
||||
.isEqualTo(BackupAlertSink.Severity.CRITICAL);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Dedupe keys are the type plus the plan (or plan and level), never a value.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testStableKeys()
|
||||
{
|
||||
assertThat(BackupAlerts.key(BackupAlerts.PLAN_MISSED, "nightly")).isEqualTo("backups.plan.missed:nightly");
|
||||
assertThat(BackupAlerts.verifyKey("nightly", "CONTENT")).isEqualTo("backups.verify.failed:nightly:CONTENT");
|
||||
|
||||
BackupAlerts.raise(BackupAlerts.FILES_MISSING, "nightly", BackupAlertSink.Severity.WARNING, "t", "missing", "3", "odd");
|
||||
BackupAlerts.raise(BackupAlerts.FILES_MISSING, "nightly", BackupAlertSink.Severity.WARNING, "t", "missing", "4");
|
||||
assertThat(sink.raises).hasSize(2).allSatisfy(r -> assertThat(r.dedupeKey()).isEqualTo("backups.files.missing:nightly"));
|
||||
assertThat(sink.raises.get(0).detail()).containsExactly(Map.entry("missing", "3"));
|
||||
assertThat(sink.open).hasSize(1);
|
||||
|
||||
BackupAlerts.resolve(BackupAlerts.FILES_MISSING, "nightly");
|
||||
assertThat(sink.open).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A restore is never silent: info, or error when it failed.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRestoreCompleted()
|
||||
{
|
||||
BackupAlerts.restoreCompleted("20261012T030000Z-r1", "SUCCEEDED", "nightly", 42);
|
||||
BackupAlerts.restoreCompleted("20261012T030000Z-r2", "FAILED", null, 0);
|
||||
assertThat(sink.raises).extracting(RecordingAlertSink.Raised::severity).containsExactly(BackupAlertSink.Severity.INFO, BackupAlertSink.Severity.ERROR);
|
||||
assertThat(sink.raises.get(0).detail()).containsEntry("rows", "42").containsEntry("plan", "nightly");
|
||||
assertThat(sink.raises.get(0).type()).isEqualTo("backups.restore.completed");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A sink that throws never fails the caller.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testASinkThatThrowsIsSwallowed()
|
||||
{
|
||||
BackupsRuntime.setAlertSink(new NoOpBackupAlertSink()
|
||||
{
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void raise(String type, String dedupeKey, Severity severity, String title, Map<String, String> detail)
|
||||
{
|
||||
throw (new IllegalStateException("down"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void resolve(String type, String dedupeKey)
|
||||
{
|
||||
throw (new IllegalStateException("down"));
|
||||
}
|
||||
});
|
||||
BackupAlerts.raise("t", "p", BackupAlertSink.Severity.INFO, "x");
|
||||
BackupAlerts.raiseWithKey("t", "k", BackupAlertSink.Severity.INFO, "x", Map.of());
|
||||
BackupAlerts.resolve("t", "p");
|
||||
BackupAlerts.resolveKey("t", "k");
|
||||
|
||||
new NoOpBackupAlertSink().raise("t", "k", BackupAlertSink.Severity.INFO, "x", Map.of());
|
||||
new NoOpBackupAlertSink().resolve("t", "k");
|
||||
new NoOpBackupAlertSink().heartbeat("m", 1, 1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.alerts.fakeapi;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test double of the Alerts producer SPI (QRun-IO/qqq PR #1108): the same shape as
|
||||
** com.kingsrook.qqq.backend.core.alerts.AlertSeverity.
|
||||
*******************************************************************************/
|
||||
public enum AlertSeverity
|
||||
{
|
||||
INFO,
|
||||
WARNING,
|
||||
ERROR,
|
||||
CRITICAL
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.alerts.fakeapi;
|
||||
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test double of the Alerts producer SPI AlertSignal (only what the adapter uses).
|
||||
*******************************************************************************/
|
||||
public class AlertSignal
|
||||
{
|
||||
public final String alertTypeName;
|
||||
public String dedupKey;
|
||||
public AlertSeverity severity;
|
||||
public String title;
|
||||
public String summary;
|
||||
public final Map<String, String> attributes = new LinkedHashMap<>();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor.
|
||||
*******************************************************************************/
|
||||
public AlertSignal(String alertTypeName)
|
||||
{
|
||||
this.alertTypeName = alertTypeName;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter.
|
||||
*******************************************************************************/
|
||||
public void setDedupKey(String dedupKey)
|
||||
{
|
||||
this.dedupKey = dedupKey;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter.
|
||||
*******************************************************************************/
|
||||
public void setSeverity(AlertSeverity severity)
|
||||
{
|
||||
this.severity = severity;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter.
|
||||
*******************************************************************************/
|
||||
public void setTitle(String title)
|
||||
{
|
||||
this.title = title;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter.
|
||||
*******************************************************************************/
|
||||
public void setSummary(String summary)
|
||||
{
|
||||
this.summary = summary;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent attribute.
|
||||
*******************************************************************************/
|
||||
public AlertSignal withAttribute(String key, String value)
|
||||
{
|
||||
attributes.put(key, value);
|
||||
return (this);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.alerts.fakeapi;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test double of the Alerts producer SPI AlertsInstanceMetaData: of(instance) is
|
||||
** non-null only when a test says the Alerts qbit is installed.
|
||||
*******************************************************************************/
|
||||
public class AlertsInstanceMetaData
|
||||
{
|
||||
public static boolean installed = false;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The alerts meta-data of an instance, or null.
|
||||
*******************************************************************************/
|
||||
public static AlertsInstanceMetaData of(QInstance qInstance)
|
||||
{
|
||||
return (installed ? new AlertsInstanceMetaData() : null);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.alerts.fakeapi;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test double of the Alerts producer SPI QAlertTypeMetaData: records the types
|
||||
** added to an instance.
|
||||
*******************************************************************************/
|
||||
public class QAlertTypeMetaData
|
||||
{
|
||||
public static final List<QAlertTypeMetaData> ADDED = new ArrayList<>();
|
||||
|
||||
public String name;
|
||||
public String label;
|
||||
public String groupName;
|
||||
public String howToFix;
|
||||
public AlertSeverity defaultSeverity;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent.
|
||||
*******************************************************************************/
|
||||
public QAlertTypeMetaData withName(String name)
|
||||
{
|
||||
this.name = name;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent.
|
||||
*******************************************************************************/
|
||||
public QAlertTypeMetaData withLabel(String label)
|
||||
{
|
||||
this.label = label;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent.
|
||||
*******************************************************************************/
|
||||
public QAlertTypeMetaData withGroupName(String groupName)
|
||||
{
|
||||
this.groupName = groupName;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent.
|
||||
*******************************************************************************/
|
||||
public QAlertTypeMetaData withHowToFix(String howToFix)
|
||||
{
|
||||
this.howToFix = howToFix;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent.
|
||||
*******************************************************************************/
|
||||
public QAlertTypeMetaData withDefaultSeverity(AlertSeverity defaultSeverity)
|
||||
{
|
||||
this.defaultSeverity = defaultSeverity;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Add to the instance (recorded).
|
||||
*******************************************************************************/
|
||||
public void addSelfToInstance(QInstance qInstance)
|
||||
{
|
||||
ADDED.add(this);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.alerts.fakeapi;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test double of the Alerts producer SPI QAlerts: static calls that record what
|
||||
** they were given.
|
||||
*******************************************************************************/
|
||||
public final class QAlerts
|
||||
{
|
||||
public static final List<AlertSignal> RAISED = new ArrayList<>();
|
||||
public static final List<String> RESOLVED = new ArrayList<>();
|
||||
public static final List<String> HEARTBEATS = new ArrayList<>();
|
||||
|
||||
/** When set, every call throws it (the SPI is meant not to, but the adapter must survive). */
|
||||
public static RuntimeException failWith;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private QAlerts()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Forget everything.
|
||||
*******************************************************************************/
|
||||
public static void reset()
|
||||
{
|
||||
RAISED.clear();
|
||||
RESOLVED.clear();
|
||||
HEARTBEATS.clear();
|
||||
failWith = null;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Raise.
|
||||
*******************************************************************************/
|
||||
public static void raise(AlertSignal signal)
|
||||
{
|
||||
maybeFail();
|
||||
RAISED.add(signal);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve.
|
||||
*******************************************************************************/
|
||||
public static void resolve(String alertTypeName, String dedupKey, String note)
|
||||
{
|
||||
maybeFail();
|
||||
RESOLVED.add(alertTypeName + "|" + dedupKey + "|" + note);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Heartbeat.
|
||||
*******************************************************************************/
|
||||
public static void heartbeat(String heartbeatKey)
|
||||
{
|
||||
maybeFail();
|
||||
HEARTBEATS.add(heartbeatKey);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static void maybeFail()
|
||||
{
|
||||
if(failWith != null)
|
||||
{
|
||||
throw (failWith);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.events;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.backups.BackupsRuntime;
|
||||
import com.kingsrook.qbits.backups.BackupsTestUtils;
|
||||
import com.kingsrook.qbits.backups.VerifyTestSupport;
|
||||
import com.kingsrook.qbits.backups.run.BackupRunResult;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** ESB events are fire and forget: published for run outcomes, never allowed to
|
||||
** fail a backup, and silent when no destination is configured.
|
||||
*******************************************************************************/
|
||||
class BackupEventsTest
|
||||
{
|
||||
private final List<String> types = new ArrayList<>();
|
||||
private final List<Map<String, Serializable>> datas = new ArrayList<>();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp() throws Exception
|
||||
{
|
||||
VerifyTestSupport.setUp(3);
|
||||
BackupEvents.setPublisher((type, data) ->
|
||||
{
|
||||
types.add(type);
|
||||
datas.add(data);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@AfterEach
|
||||
void tearDown()
|
||||
{
|
||||
BackupEvents.setPublisher(null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A successful run publishes run.succeeded; a failed one run.failed.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRunEvents() throws Exception
|
||||
{
|
||||
QRecord plan = VerifyTestSupport.plan(Map.of());
|
||||
BackupRunResult result = VerifyTestSupport.backup(plan);
|
||||
assertThat(types).containsExactly("qqq.backups.run.succeeded");
|
||||
assertThat(datas.get(0)).containsEntry("plan", "nightly").containsEntry("run", result.getRunKey()).containsEntry("rows", result.getRows());
|
||||
|
||||
BackupsTestUtils.BACKUPS.withFailWritesFor(key -> key.contains("/tables/"));
|
||||
VerifyTestSupport.backup(plan);
|
||||
assertThat(types).containsExactly("qqq.backups.run.succeeded", "qqq.backups.run.failed");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Cancelled runs publish nothing; restores publish restore.completed.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testCancelledAndRestore()
|
||||
{
|
||||
BackupEvents.runFinished("nightly", "r1", "CANCELLED", 0, 0);
|
||||
BackupEvents.runFinished("nightly", "r2", "TIMED_OUT", 0, 0);
|
||||
BackupEvents.restoreCompleted("rr1", "SUCCEEDED", 12);
|
||||
assertThat(types).containsExactly("qqq.backups.run.failed", "qqq.backups.restore.completed");
|
||||
assertThat(datas.get(1)).containsEntry("restoreRun", "rr1").containsEntry("rows", 12L);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A publisher that throws never fails the run.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testPublishFailureNeverFailsTheBackup() throws Exception
|
||||
{
|
||||
BackupEvents.setPublisher((type, data) ->
|
||||
{
|
||||
throw (new IllegalStateException("bus down"));
|
||||
});
|
||||
BackupRunResult result = VerifyTestSupport.backup(VerifyTestSupport.plan(Map.of()));
|
||||
assertThat(result.getStatus()).startsWith("SUCCEEDED");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** With no destination configured nothing is published; with one, the ESB is
|
||||
** tried and a missing or unconfigured ESB is swallowed.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testEsbDefault() throws Exception
|
||||
{
|
||||
BackupEvents.setPublisher(null);
|
||||
BackupEvents.publish("qqq.backups.run.succeeded", Map.of());
|
||||
|
||||
BackupsRuntime.getConfig().withEsbDestinationName("backups-events");
|
||||
BackupEvents.publish("qqq.backups.run.succeeded", Map.of("plan", "nightly"));
|
||||
BackupEvents.publish("qqq.backups.run.succeeded", Map.of("plan", "nightly"));
|
||||
assertThat(VerifyTestSupport.backup(VerifyTestSupport.plan(Map.of())).getStatus()).startsWith("SUCCEEDED");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.health;
|
||||
|
||||
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.backups.VerifyTestSupport;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.middleware.health.model.metadata.HealthCheckResult;
|
||||
import com.kingsrook.qqq.middleware.health.model.metadata.HealthStatus;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The health indicator: DOWN only when a critical, active plan is LATE, FAILING or
|
||||
** MISSING_IN_STORAGE.
|
||||
*******************************************************************************/
|
||||
class BackupsHealthIndicatorTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp() throws Exception
|
||||
{
|
||||
VerifyTestSupport.setUp(3);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private void setStatus(QRecord plan, String status) throws Exception
|
||||
{
|
||||
new BackupCatalog().update(BackupsTables.PLAN, plan.getValueInteger("id"), Map.of("slaStatus", status));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** UP with details when nothing critical is wrong, including a non-critical plan
|
||||
** that is LATE; DOWN when a critical plan is LATE, FAILING or MISSING_IN_STORAGE.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testUpAndDown() throws Exception
|
||||
{
|
||||
BackupsHealthIndicator indicator = new BackupsHealthIndicator();
|
||||
assertThat(indicator.getName()).isEqualTo("backups");
|
||||
|
||||
HealthCheckResult empty = indicator.check(QContext.getQInstance());
|
||||
assertThat(empty.getStatus()).isEqualTo(HealthStatus.UP);
|
||||
assertThat(empty.getDetails()).containsEntry("activePlans", 0);
|
||||
|
||||
QRecord critical = VerifyTestSupport.plan(Map.of("critical", true));
|
||||
QRecord ordinary = VerifyTestSupport.plan(Map.of("name", "Other", "planKey", "other"));
|
||||
VerifyTestSupport.plan(Map.of("name", "Off", "planKey", "off", "isActive", false, "critical", true));
|
||||
|
||||
setStatus(critical, "OK");
|
||||
setStatus(ordinary, "LATE");
|
||||
HealthCheckResult up = indicator.check(QContext.getQInstance());
|
||||
assertThat(up.getStatus()).isEqualTo(HealthStatus.UP);
|
||||
assertThat(up.getDetails()).containsEntry("activePlans", 2).doesNotContainKey("criticalPlansWithProblems");
|
||||
assertThat(up.getDurationMs()).isNotNull();
|
||||
|
||||
for(String status : new String[] { "LATE", "FAILING", "MISSING_IN_STORAGE" })
|
||||
{
|
||||
setStatus(critical, status);
|
||||
HealthCheckResult down = indicator.check(QContext.getQInstance());
|
||||
assertThat(down.getStatus()).as(status).isEqualTo(HealthStatus.DOWN);
|
||||
assertThat(down.getDetails().get("criticalPlansWithProblems").toString()).contains("nightly: " + status);
|
||||
}
|
||||
|
||||
setStatus(critical, "UNVERIFIED");
|
||||
assertThat(indicator.check(QContext.getQInstance()).getStatus()).as("not verified is not down").isEqualTo(HealthStatus.UP);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** On a thread with no QQQ context (a health request) the indicator makes its own
|
||||
** and cleans up.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testWorksWithoutAContext() throws Exception
|
||||
{
|
||||
QInstance qInstance = QContext.getQInstance();
|
||||
QContext.clear();
|
||||
try
|
||||
{
|
||||
assertThat(new BackupsHealthIndicator().check(qInstance).getStatus()).isEqualTo(HealthStatus.UP);
|
||||
assertThat(QContext.getQInstance()).isNull();
|
||||
}
|
||||
finally
|
||||
{
|
||||
QContext.init(qInstance, new com.kingsrook.qqq.backend.core.model.session.QSession());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.metrics;
|
||||
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.backups.BackupsRuntime;
|
||||
import com.kingsrook.qbits.backups.BackupsTestUtils;
|
||||
import com.kingsrook.qbits.backups.VerifyTestSupport;
|
||||
import com.kingsrook.qbits.backups.run.BackupRunResult;
|
||||
import com.kingsrook.qbits.backups.verify.BackupVerifier;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The Prometheus text, computed from the tables.
|
||||
*******************************************************************************/
|
||||
class BackupsMetricsTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp() throws Exception
|
||||
{
|
||||
VerifyTestSupport.setUp(3);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The five gauges, labeled with the instance and the plan, from real runs.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRender() throws Exception
|
||||
{
|
||||
QRecord plan = VerifyTestSupport.plan(Map.of());
|
||||
BackupRunResult result = VerifyTestSupport.backup(plan);
|
||||
new BackupVerifier().verify(result.getRunId(), BackupVerifier.CONTENT, "MANUAL", "t");
|
||||
|
||||
String text = BackupsMetrics.render();
|
||||
String labels = "{instance_name=\"testapp\",plan=\"nightly\"}";
|
||||
long now = VerifyTestSupport.clock.instant().getEpochSecond();
|
||||
assertThat(text).contains("# TYPE qqq_backup_last_success_timestamp_seconds gauge")
|
||||
.contains("qqq_backup_last_success_timestamp_seconds" + labels + " " + now + "\n")
|
||||
.contains("qqq_backup_last_run_failed" + labels + " 0\n")
|
||||
.contains("qqq_backup_sla_seconds" + labels + " 93600\n")
|
||||
.contains("qqq_backup_last_verify_success_timestamp_seconds" + labels + " " + now + "\n")
|
||||
.contains("qqq_backup_last_success_bytes" + labels + " ");
|
||||
|
||||
BackupsTestUtils.BACKUPS.withFailWritesFor(key -> key.contains("/tables/"));
|
||||
VerifyTestSupport.backup(plan);
|
||||
assertThat(BackupsMetrics.render()).contains("qqq_backup_last_run_failed" + labels + " 1\n");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Cached for 60 seconds; label values are escaped; the config flag is read.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testCacheEscapeAndFlag() throws Exception
|
||||
{
|
||||
assertThat(BackupsMetrics.isEnabled()).isFalse();
|
||||
BackupsRuntime.getConfig().withPrometheusEndpoint(true);
|
||||
assertThat(BackupsMetrics.isEnabled()).isTrue();
|
||||
assertThat(BackupsMetrics.ENDPOINT_PATH).isEqualTo("/metrics/backups");
|
||||
|
||||
String empty = BackupsMetrics.renderCached();
|
||||
assertThat(empty).doesNotContain("{instance_name");
|
||||
|
||||
VerifyTestSupport.backup(VerifyTestSupport.plan(Map.of()));
|
||||
VerifyTestSupport.clock.advance(Duration.ofSeconds(30));
|
||||
assertThat(BackupsMetrics.renderCached()).as("cached").isEqualTo(empty);
|
||||
VerifyTestSupport.clock.advance(Duration.ofSeconds(31));
|
||||
assertThat(BackupsMetrics.renderCached()).contains("plan=\"nightly\"");
|
||||
|
||||
assertThat(BackupsMetrics.escape("a\"b\\c\nd")).isEqualTo("a\\\"b\\\\c\\nd");
|
||||
assertThat(BackupsMetrics.escape(null)).isEmpty();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.processes;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.backups.BackupsQBitConfig;
|
||||
import com.kingsrook.qbits.backups.BackupsRuntime;
|
||||
import com.kingsrook.qbits.backups.BackupsTestUtils;
|
||||
import com.kingsrook.qbits.backups.VerifyTestSupport;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.permissions.BackupsPermissions;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qbits.backups.run.BackupRunResult;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The Verify backup record action (a streamed ETL with Review and Result lines)
|
||||
** and the scheduled SLA check.
|
||||
*******************************************************************************/
|
||||
class VerifyProcessesTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp() throws Exception
|
||||
{
|
||||
VerifyTestSupport.setUp(7);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Both processes are defined on the instance with the right permissions; neither
|
||||
** is in the nav, and the label is plain.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testDefinitions()
|
||||
{
|
||||
QProcessMetaData verify = QContext.getQInstance().getProcess(VerifyProcesses.VERIFY);
|
||||
assertThat(verify).isNotNull();
|
||||
assertThat(verify.getLabel()).isEqualTo("Verify backup");
|
||||
assertThat(verify.getTableName()).isEqualTo(BackupsTables.RUN);
|
||||
assertThat(QContext.getQInstance().getProcess(VerifyProcesses.CHECK_SLA)).isNotNull();
|
||||
assertThat(BackupsRuntime.processPermissions()).containsEntry("backupVerify", BackupsPermissions.RUN).containsEntry("backupCheckSla", BackupsPermissions.RUN);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The SLA check is scheduled every slaCheckIntervalSeconds when a scheduler is
|
||||
** configured, independent of any backup trigger; otherwise it is unscheduled.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testSlaCheckSchedule()
|
||||
{
|
||||
assertThat(VerifyProcesses.checkSlaProcess(BackupsRuntime.getConfig()).getSchedule()).isNull();
|
||||
assertThat(VerifyProcesses.checkSlaProcess(null).getSchedule()).isNull();
|
||||
|
||||
BackupsQBitConfig scheduled = new BackupsQBitConfig().withBackendName("memory").withSchedulerName("quartz").withSlaCheckIntervalSeconds(120);
|
||||
QProcessMetaData process = VerifyProcesses.checkSlaProcess(scheduled);
|
||||
assertThat(process.getSchedule().getSchedulerName()).isEqualTo("quartz");
|
||||
assertThat(process.getSchedule().getRepeatSeconds()).isEqualTo(120);
|
||||
assertThat(process.getSchedule().getCronExpression()).isNull();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The scheduled step checks every active plan.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testCheckSlaStep() throws Exception
|
||||
{
|
||||
VerifyTestSupport.backup(VerifyTestSupport.plan(Map.of()));
|
||||
VerifyTestSupport.plan(Map.of("name", "Late", "planKey", "late", "storageBucketAlias", "backups"));
|
||||
VerifyTestSupport.clock.advance(java.time.Duration.ofHours(30));
|
||||
|
||||
RunProcessInput input = new RunProcessInput();
|
||||
input.setProcessName(VerifyProcesses.CHECK_SLA);
|
||||
input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
|
||||
RunProcessOutput output = new RunProcessAction().execute(input);
|
||||
assertThat(output.getException()).isEmpty();
|
||||
assertThat(output.getValues()).containsEntry("plansChecked", 2).containsEntry("plansNeedingAttention", 2L);
|
||||
assertThat(new BackupCatalog().where(BackupsTables.PLAN, "planKey", "late").get(0).getValueString("slaStatus")).isEqualTo("LATE");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify backup on a good run: Review lists what will be checked without reading
|
||||
** anything; the run passes, and the Result lines link back to the run.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testVerifyGoodRun() throws Exception
|
||||
{
|
||||
BackupRunResult result = VerifyTestSupport.backup(VerifyTestSupport.plan(Map.of()));
|
||||
assertThat(new BackupCatalog().where(BackupsTables.VERIFICATION, "backupRunId", result.getRunId())).isEmpty();
|
||||
|
||||
RunProcessInput input = input(result.getRunId());
|
||||
RunProcessOutput output = new RunProcessAction().execute(input);
|
||||
List<ProcessSummaryLineInterface> review = lines(output, StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY);
|
||||
assertThat(review).extracting(l -> ((ProcessSummaryLine) l).getStatus()).contains(Status.OK, Status.INFO);
|
||||
assertThat(new BackupCatalog().where(BackupsTables.VERIFICATION, "backupRunId", result.getRunId())).as("Review reads nothing and writes nothing").isEmpty();
|
||||
|
||||
input.setStartAfterStep("review");
|
||||
output = new RunProcessAction().execute(input);
|
||||
assertThat(output.getException()).isEmpty();
|
||||
List<ProcessSummaryLineInterface> summary = lines(output, StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY);
|
||||
ProcessSummaryLine ok = (ProcessSummaryLine) summary.stream().filter(l -> ((ProcessSummaryLine) l).getStatus() == Status.OK).findFirst().orElseThrow();
|
||||
assertThat(ok.getCount()).isEqualTo(1);
|
||||
assertThat(ok.getPrimaryKeys()).containsExactly(result.getRunId());
|
||||
assertThat(new BackupCatalog().where(BackupsTables.VERIFICATION, "backupRunId", result.getRunId())).hasSize(2);
|
||||
assertThat(VerifyTestSupport.run(result.getRunId()).getValueString("verificationStatus")).isEqualTo("CONTENT_OK");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A corrupted run fails with the first mismatch in the Result line; a run with no
|
||||
** manifest is refused at Review.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testVerifyBadRunsAndRefusals() throws Exception
|
||||
{
|
||||
BackupRunResult result = VerifyTestSupport.backup(VerifyTestSupport.plan(Map.of()));
|
||||
String key = VerifyTestSupport.firstPartKey(result, "blogPost");
|
||||
byte[] bytes = BackupsTestUtils.BACKUPS.getBytes(key).clone();
|
||||
bytes[bytes.length / 2] ^= 0x55;
|
||||
BackupsTestUtils.BACKUPS.corrupt(key, bytes);
|
||||
|
||||
Integer noManifest = new BackupCatalog().insert(BackupsTables.RUN, new java.util.LinkedHashMap<>(Map.<String, Serializable>of("runKey", "x1", "status", "FAILED")));
|
||||
Integer pruned = new BackupCatalog().insert(BackupsTables.RUN, new java.util.LinkedHashMap<>(Map.<String, Serializable>of("runKey", "x2", "status", "SUCCEEDED", "manifestKey", "m", "prunedDate", java.time.Instant.now())));
|
||||
|
||||
RunProcessInput input = input(result.getRunId(), noManifest, pruned);
|
||||
input.addValue(VerifyProcesses.FIELD_VERIFY_LEVEL, "STORAGE");
|
||||
RunProcessOutput output = new RunProcessAction().execute(input);
|
||||
List<ProcessSummaryLineInterface> review = lines(output, StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY);
|
||||
assertThat(review.stream().filter(l -> ((ProcessSummaryLine) l).getStatus() == Status.ERROR)).hasSize(2);
|
||||
|
||||
input.addValue(VerifyProcesses.FIELD_VERIFY_LEVEL, "CONTENT");
|
||||
input.setStartAfterStep("review");
|
||||
output = new RunProcessAction().execute(input);
|
||||
List<ProcessSummaryLineInterface> summary = lines(output, StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY);
|
||||
ProcessSummaryLine failed = (ProcessSummaryLine) summary.stream().filter(l -> ((ProcessSummaryLine) l).getStatus() == Status.ERROR).findFirst().orElseThrow();
|
||||
assertThat(failed.getCount()).isEqualTo(1);
|
||||
assertThat(failed.getMessageSuffix()).contains("corrupted");
|
||||
assertThat(VerifyTestSupport.run(result.getRunId()).getValueString("verificationStatus")).isEqualTo("FAILED");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private RunProcessInput input(Integer... runIds)
|
||||
{
|
||||
RunProcessInput input = new RunProcessInput();
|
||||
input.setProcessName(VerifyProcesses.VERIFY);
|
||||
input.addValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.IN, List.of((Serializable[]) runIds))));
|
||||
return (input);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@SuppressWarnings("unchecked")
|
||||
private List<ProcessSummaryLineInterface> lines(RunProcessOutput output, String field)
|
||||
{
|
||||
return ((List<ProcessSummaryLineInterface>) output.getValues().get(field));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,430 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.sla;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.backups.BackupsRuntime;
|
||||
import com.kingsrook.qbits.backups.BackupsTestUtils;
|
||||
import com.kingsrook.qbits.backups.VerifyTestSupport;
|
||||
import com.kingsrook.qbits.backups.alerts.BackupAlertSink;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qbits.backups.run.BackupRunResult;
|
||||
import com.kingsrook.qbits.backups.verify.BackupVerifier;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static com.kingsrook.qbits.backups.VerifyTestSupport.backup;
|
||||
import static com.kingsrook.qbits.backups.VerifyTestSupport.clock;
|
||||
import static com.kingsrook.qbits.backups.VerifyTestSupport.plan;
|
||||
import static com.kingsrook.qbits.backups.VerifyTestSupport.reload;
|
||||
import static com.kingsrook.qbits.backups.VerifyTestSupport.sink;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every SLA state of the checker, driven by a synthetic clock, with the alerts it
|
||||
** raises and resolves recorded through a recording sink.
|
||||
*******************************************************************************/
|
||||
class SlaCheckerTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp() throws Exception
|
||||
{
|
||||
VerifyTestSupport.setUp(7);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private SlaChecker.PlanResult check(QRecord plan) throws Exception
|
||||
{
|
||||
return (new SlaChecker().check(reload(plan)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A new plan that has not run yet is UNKNOWN, then LATE once slaHours pass; a
|
||||
** manual-only plan with no slaHours never is.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testNewPlanThenLate() throws Exception
|
||||
{
|
||||
QRecord plan = plan(Map.of("critical", true));
|
||||
assertThat(check(plan).status()).isEqualTo(SlaChecker.UNKNOWN);
|
||||
assertThat(sink.open).isEmpty();
|
||||
|
||||
clock.advance(Duration.ofHours(27));
|
||||
SlaChecker.PlanResult late = check(plan);
|
||||
assertThat(late.status()).isEqualTo(SlaChecker.LATE);
|
||||
assertThat(late.previousStatus()).isEqualTo(SlaChecker.UNKNOWN);
|
||||
assertThat(reload(plan).getValueString("slaStatus")).isEqualTo("LATE");
|
||||
BackupAlertsAssert.assertOpen("backups.plan.missed:nightly", BackupAlertSink.Severity.CRITICAL);
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The plan.missed alert is error for an ordinary plan, critical for a critical
|
||||
** one, and resolves when the next run succeeds.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testLateRaisesAndResolves() throws Exception
|
||||
{
|
||||
QRecord plan = plan(Map.of());
|
||||
backup(plan);
|
||||
assertThat(check(plan).status()).isEqualTo(SlaChecker.OK);
|
||||
assertThat(sink.open).isEmpty();
|
||||
|
||||
clock.advance(Duration.ofHours(27));
|
||||
assertThat(check(plan).status()).isEqualTo(SlaChecker.LATE);
|
||||
BackupAlertsAssert.assertOpen("backups.plan.missed:nightly", BackupAlertSink.Severity.ERROR);
|
||||
assertThat(sink.openAlert("backups.plan.missed:nightly").detail()).containsEntry("plan", "nightly").containsEntry("slaHours", "26");
|
||||
|
||||
backup(plan);
|
||||
assertThat(check(plan).status()).isEqualTo(SlaChecker.OK);
|
||||
assertThat(sink.isOpen("backups.plan.missed:nightly")).isFalse();
|
||||
assertThat(sink.heartbeats).contains("backups.plan:nightly");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A manual-only plan with no slaHours has no SLA; a scheduled one defaults to 26.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testManualPlanHasNoSlaUnlessScheduled() throws Exception
|
||||
{
|
||||
QRecord manual = plan(Map.of("name", "Manual", "planKey", "manual"));
|
||||
new BackupCatalog().update(BackupsTables.PLAN, manual.getValueInteger("id"), java.util.Collections.singletonMap("slaHours", null));
|
||||
clock.advance(Duration.ofDays(3));
|
||||
assertThat(check(manual).status()).isEqualTo(SlaChecker.UNKNOWN);
|
||||
|
||||
QRecord scheduled = plan(Map.of("name", "Sched", "planKey", "sched", "cronExpression", "0 0 2 * * ?"));
|
||||
new BackupCatalog().update(BackupsTables.PLAN, scheduled.getValueInteger("id"), java.util.Collections.singletonMap("slaHours", null));
|
||||
assertThat(check(scheduled).status()).isEqualTo(SlaChecker.LATE);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Two failed runs in a row are FAILING (error); one is only a warning; a later
|
||||
** success clears it.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testFailingAfterTwoFailures() throws Exception
|
||||
{
|
||||
QRecord plan = plan(Map.of());
|
||||
BackupsTestUtils.BACKUPS.withFailWritesFor(key -> key.contains("/tables/"));
|
||||
|
||||
assertThat(backup(plan).getStatus()).isEqualTo("FAILED");
|
||||
assertThat(check(plan).status()).isEqualTo(SlaChecker.UNKNOWN);
|
||||
BackupAlertsAssert.assertOpen("backups.run.failed:nightly", BackupAlertSink.Severity.WARNING);
|
||||
|
||||
assertThat(backup(plan).getStatus()).isEqualTo("FAILED");
|
||||
assertThat(check(plan).status()).isEqualTo(SlaChecker.FAILING);
|
||||
BackupAlertsAssert.assertOpen("backups.run.failed:nightly", BackupAlertSink.Severity.ERROR);
|
||||
|
||||
BackupsTestUtils.BACKUPS.withFailWritesFor(null);
|
||||
assertThat(backup(plan).getStatus()).startsWith("SUCCEEDED");
|
||||
assertThat(check(plan).status()).isEqualTo(SlaChecker.OK);
|
||||
assertThat(sink.isOpen("backups.run.failed:nightly")).isFalse();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A deleted manifest is MISSING_IN_STORAGE with a critical alert (a head, cached
|
||||
** for an hour); putting it back clears it.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testDeletedManifestIsMissingInStorage() throws Exception
|
||||
{
|
||||
QRecord plan = plan(Map.of());
|
||||
BackupRunResult result = backup(plan);
|
||||
byte[] bytes = BackupsTestUtils.BACKUPS.getBytes(result.getManifestKey());
|
||||
|
||||
assertThat(check(plan).status()).isEqualTo(SlaChecker.OK);
|
||||
BackupsTestUtils.BACKUPS.removeIgnoringLock(result.getManifestKey());
|
||||
|
||||
clock.advance(Duration.ofMinutes(10));
|
||||
assertThat(check(plan).status()).as("the earlier positive head is trusted for an hour").isEqualTo(SlaChecker.OK);
|
||||
|
||||
clock.advance(Duration.ofMinutes(55));
|
||||
assertThat(check(plan).status()).isEqualTo(SlaChecker.MISSING_IN_STORAGE);
|
||||
BackupAlertsAssert.assertOpen("backups.run.missing_in_storage:nightly", BackupAlertSink.Severity.CRITICAL);
|
||||
assertThat(check(plan).status()).as("a negative answer is never cached").isEqualTo(SlaChecker.MISSING_IN_STORAGE);
|
||||
|
||||
BackupsTestUtils.BACKUPS.putBytes(result.getManifestKey(), bytes);
|
||||
assertThat(check(plan).status()).isEqualTo(SlaChecker.OK);
|
||||
assertThat(sink.isOpen("backups.run.missing_in_storage:nightly")).isFalse();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A RUNNING run whose heartbeat is older than ten minutes is marked ABANDONED
|
||||
** and the plan is ABANDONED; a fresh heartbeat is left alone.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testStaleHeartbeatIsAbandoned() throws Exception
|
||||
{
|
||||
QRecord plan = plan(Map.of());
|
||||
backup(plan);
|
||||
Integer stale = runRow(plan, "20261012T000000Z-aaaa", "RUNNING", clock.instant());
|
||||
clock.advance(Duration.ofMinutes(5));
|
||||
Integer fresh = runRow(plan, "20261012T000500Z-bbbb", "RUNNING", clock.instant());
|
||||
|
||||
clock.advance(Duration.ofMinutes(6));
|
||||
SlaChecker.PlanResult result = check(plan);
|
||||
assertThat(result.abandonedRuns()).isEqualTo(1);
|
||||
assertThat(new BackupCatalog().get(BackupsTables.RUN, stale).orElseThrow().getValueString("status")).isEqualTo("ABANDONED");
|
||||
assertThat(new BackupCatalog().get(BackupsTables.RUN, stale).orElseThrow().getValueString("errorMessage")).contains("heartbeat");
|
||||
assertThat(new BackupCatalog().get(BackupsTables.RUN, fresh).orElseThrow().getValueString("status")).isEqualTo("RUNNING");
|
||||
|
||||
assertThat(result.status()).isEqualTo(SlaChecker.ABANDONED);
|
||||
BackupAlertsAssert.assertOpen("backups.run.failed:nightly", BackupAlertSink.Severity.WARNING);
|
||||
|
||||
new BackupCatalog().update(BackupsTables.RUN, fresh, java.util.Collections.singletonMap("status", "SUCCEEDED"));
|
||||
backup(plan);
|
||||
assertThat(check(plan).status()).isEqualTo(SlaChecker.OK);
|
||||
assertThat(sink.isOpen("backups.run.failed:nightly")).isFalse();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** UNVERIFIED when no content verification has passed within verifySlaDays; a
|
||||
** passed verification clears it.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testUnverifiedThenVerified() throws Exception
|
||||
{
|
||||
QRecord plan = plan(Map.of("slaHours", 24 * 30));
|
||||
backup(plan);
|
||||
assertThat(check(plan).status()).isEqualTo(SlaChecker.OK);
|
||||
|
||||
clock.advance(Duration.ofDays(9));
|
||||
BackupRunResult second = backup(plan);
|
||||
assertThat(check(plan).status()).isEqualTo(SlaChecker.UNVERIFIED);
|
||||
BackupAlertsAssert.assertOpen("backups.verify.overdue:nightly", BackupAlertSink.Severity.WARNING);
|
||||
|
||||
new BackupVerifier().verifyThrough(second.getRunId(), BackupVerifier.CONTENT, "MANUAL", "t");
|
||||
assertThat(sink.isOpen("backups.verify.overdue:nightly")).isFalse();
|
||||
assertThat(check(plan).status()).isEqualTo(SlaChecker.OK);
|
||||
|
||||
clock.advance(Duration.ofDays(9));
|
||||
backup(plan);
|
||||
assertThat(check(plan).status()).isEqualTo(SlaChecker.UNVERIFIED);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** OWNER_INVALID when the configured validator rejects the plan owner.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testOwnerInvalid() throws Exception
|
||||
{
|
||||
QRecord plan = plan(Map.of("ownerUserId", "bob"));
|
||||
backup(plan);
|
||||
assertThat(check(plan).status()).isEqualTo(SlaChecker.OK);
|
||||
|
||||
BackupsRuntime.getConfig().withPlanOwnerValidator(owner -> false);
|
||||
assertThat(check(plan).status()).isEqualTo(SlaChecker.OWNER_INVALID);
|
||||
BackupAlertsAssert.assertOpen("backups.plan.owner_invalid:nightly", BackupAlertSink.Severity.WARNING);
|
||||
|
||||
BackupsRuntime.getConfig().withPlanOwnerValidator(owner -> "bob".equals(owner));
|
||||
assertThat(check(plan).status()).isEqualTo(SlaChecker.OK);
|
||||
assertThat(sink.isOpen("backups.plan.owner_invalid:nightly")).isFalse();
|
||||
|
||||
BackupsRuntime.getConfig().withPlanOwnerValidator(owner ->
|
||||
{
|
||||
throw (new IllegalStateException("directory down"));
|
||||
});
|
||||
assertThat(check(plan).status()).as("a validator that fails is not a verdict").isEqualTo(SlaChecker.OK);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Worst state first: MISSING_IN_STORAGE beats LATE.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testPrecedence() throws Exception
|
||||
{
|
||||
QRecord plan = plan(Map.of());
|
||||
BackupRunResult result = backup(plan);
|
||||
BackupsTestUtils.BACKUPS.removeIgnoringLock(result.getManifestKey());
|
||||
clock.advance(Duration.ofDays(2));
|
||||
assertThat(check(plan).status()).isEqualTo(SlaChecker.MISSING_IN_STORAGE);
|
||||
assertThat(sink.isOpen("backups.plan.missed:nightly")).as("the alerts are independent of the status shown").isTrue();
|
||||
assertThat(sink.isOpen("backups.run.missing_in_storage:nightly")).isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The daily bucket listing reports orphans and debris (no manifest older than 24h)
|
||||
** once a day, raises an info alert, and resolves it when the bucket is clean.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testBucketAuditFindsOrphansAndDebris() throws Exception
|
||||
{
|
||||
QRecord plan = plan(Map.of());
|
||||
backup(plan);
|
||||
|
||||
String debris = "qqq-backups/testapp/nightly/2026/09/01/20260901T020000Z-dead/tables/author/part-00001.jsonl.gz";
|
||||
String orphan = "qqq-backups/testapp/nightly/2026/09/02/20260902T020000Z-beef/manifest.json";
|
||||
BackupsTestUtils.BACKUPS.withClock(Clock.fixed(clock.instant().minus(Duration.ofDays(3)), java.time.ZoneOffset.UTC));
|
||||
BackupsTestUtils.BACKUPS.putBytes(debris, new byte[] { 1 });
|
||||
BackupsTestUtils.BACKUPS.putBytes(orphan, new byte[] { 2 });
|
||||
BackupsTestUtils.BACKUPS.withClock(Clock.systemUTC());
|
||||
|
||||
SlaChecker.PlanResult first = check(plan);
|
||||
assertThat(first.orphans()).isEqualTo(2);
|
||||
assertThat(first.debris()).isEqualTo(1);
|
||||
BackupAlertsAssert.assertOpen("backups.storage.debris:nightly", BackupAlertSink.Severity.INFO);
|
||||
assertThat(sink.openAlert("backups.storage.debris:nightly").detail()).containsEntry("orphans", "2").containsEntry("debris", "1");
|
||||
|
||||
assertThat(check(plan).orphans()).as("listed once a day").isEqualTo(-1);
|
||||
|
||||
BackupsTestUtils.BACKUPS.removeIgnoringLock(debris);
|
||||
BackupsTestUtils.BACKUPS.removeIgnoringLock(orphan);
|
||||
clock.advance(Duration.ofHours(25));
|
||||
SlaChecker.PlanResult clean = check(plan);
|
||||
assertThat(clean.orphans()).isZero();
|
||||
assertThat(clean.debris()).isZero();
|
||||
assertThat(sink.isOpen("backups.storage.debris:nightly")).isFalse();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A run that is still going (RUNNING with a fresh heartbeat) is not debris, even
|
||||
** though its prefix has no manifest yet.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRunningPrefixIsNotDebris() throws Exception
|
||||
{
|
||||
QRecord plan = plan(Map.of());
|
||||
backup(plan);
|
||||
String prefix = "qqq-backups/testapp/nightly/2026/10/12/20261012T020000Z-cafe/";
|
||||
BackupsTestUtils.BACKUPS.withClock(Clock.fixed(clock.instant().minus(Duration.ofDays(2)), java.time.ZoneOffset.UTC));
|
||||
BackupsTestUtils.BACKUPS.putBytes(prefix + "tables/author/part-00001.jsonl.gz", new byte[] { 1 });
|
||||
BackupsTestUtils.BACKUPS.withClock(Clock.systemUTC());
|
||||
Map<String, Serializable> values = new LinkedHashMap<>();
|
||||
values.put("backupPlanId", plan.getValueInteger("id"));
|
||||
values.put("runKey", "20261012T020000Z-cafe");
|
||||
values.put("status", "RUNNING");
|
||||
values.put("startDate", clock.instant());
|
||||
values.put("heartbeatDate", clock.instant());
|
||||
values.put("runPrefix", prefix);
|
||||
new BackupCatalog().insert(BackupsTables.RUN, values);
|
||||
|
||||
SlaChecker.PlanResult result = check(plan);
|
||||
assertThat(result.debris()).isZero();
|
||||
assertThat(result.orphans()).isZero();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** checkAll covers only active plans, and survives a plan whose bucket is unknown.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testCheckAllSkipsInactivePlansAndSurvivesBadBucket() throws Exception
|
||||
{
|
||||
plan(Map.of());
|
||||
plan(Map.of("name", "Off", "planKey", "off", "isActive", false));
|
||||
plan(Map.of("name", "Bad", "planKey", "bad", "storageBucketAlias", "no-such-bucket"));
|
||||
clock.advance(Duration.ofHours(30));
|
||||
|
||||
List<SlaChecker.PlanResult> results = new SlaChecker().checkAll();
|
||||
assertThat(results).extracting(SlaChecker.PlanResult::planKey).containsExactlyInAnyOrder("nightly", "bad");
|
||||
assertThat(results).extracting(SlaChecker.PlanResult::status).containsOnly(SlaChecker.LATE);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Without a recorded success or storage the checks degrade quietly: a success
|
||||
** run with no manifest key, and a pruned run, are not "missing".
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testPrunedOrKeylessSuccessIsNotMissing() throws Exception
|
||||
{
|
||||
QRecord plan = plan(Map.of());
|
||||
BackupRunResult result = backup(plan);
|
||||
new BackupCatalog().update(BackupsTables.RUN, result.getRunId(), java.util.Collections.singletonMap("prunedDate", Instant.now()));
|
||||
BackupsTestUtils.BACKUPS.removeIgnoringLock(result.getManifestKey());
|
||||
assertThat(check(plan).status()).isEqualTo(SlaChecker.OK);
|
||||
|
||||
new BackupCatalog().update(BackupsTables.RUN, result.getRunId(), java.util.Collections.singletonMap("manifestKey", null));
|
||||
assertThat(check(plan).status()).isEqualTo(SlaChecker.OK);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private Integer runRow(QRecord plan, String runKey, String status, Instant heartbeat) throws Exception
|
||||
{
|
||||
Map<String, Serializable> values = new LinkedHashMap<>();
|
||||
values.put("backupPlanId", plan.getValueInteger("id"));
|
||||
values.put("runKey", runKey);
|
||||
values.put("status", status);
|
||||
values.put("startDate", heartbeat);
|
||||
values.put("heartbeatDate", heartbeat);
|
||||
values.put("runPrefix", "qqq-backups/testapp/nightly/2026/10/12/" + runKey + "/");
|
||||
return (new BackupCatalog().insert(BackupsTables.RUN, values));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** Alert assertions over the recording sink.
|
||||
***************************************************************************/
|
||||
private static final class BackupAlertsAssert
|
||||
{
|
||||
/*******************************************************************************
|
||||
** The alert is open at that severity.
|
||||
*******************************************************************************/
|
||||
static void assertOpen(String dedupeKey, BackupAlertSink.Severity severity)
|
||||
{
|
||||
assertThat(sink.isOpen(dedupeKey)).as("alert " + dedupeKey + " is open").isTrue();
|
||||
assertThat(sink.openAlert(dedupeKey).severity()).isEqualTo(severity);
|
||||
assertThat(sink.openAlert(dedupeKey).dedupeKey()).isEqualTo(dedupeKey);
|
||||
assertThat(sink.openAlert(dedupeKey).type()).isEqualTo(dedupeKey.substring(0, dedupeKey.indexOf(':')));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,594 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.verify;
|
||||
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
import com.kingsrook.qbits.backups.BackupsRuntime;
|
||||
import com.kingsrook.qbits.backups.BackupsTestUtils;
|
||||
import com.kingsrook.qbits.backups.VerifyTestSupport;
|
||||
import com.kingsrook.qbits.backups.alerts.BackupAlerts;
|
||||
import com.kingsrook.qbits.backups.manifest.BackupManifest;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qbits.backups.run.BackupRunResult;
|
||||
import com.kingsrook.qbits.backups.storage.WriteOptions;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static com.kingsrook.qbits.backups.VerifyTestSupport.backup;
|
||||
import static com.kingsrook.qbits.backups.VerifyTestSupport.plan;
|
||||
import static com.kingsrook.qbits.backups.VerifyTestSupport.reload;
|
||||
import static com.kingsrook.qbits.backups.VerifyTestSupport.run;
|
||||
import static com.kingsrook.qbits.backups.VerifyTestSupport.sink;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verification at every level against the memory backend, including the failure
|
||||
** injection cases: a corrupted part, a deleted file-store object, a wrong
|
||||
** checksum, a missing manifest, and a wrong key.
|
||||
*******************************************************************************/
|
||||
class BackupVerifierTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp() throws Exception
|
||||
{
|
||||
VerifyTestSupport.setUp(7);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A good run passes STORAGE then CONTENT; every table, row and file is checked,
|
||||
** the verification rows, the run status and the plan are updated.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testStorageAndContentPass() throws Exception
|
||||
{
|
||||
QRecord plan = plan(Map.of());
|
||||
BackupRunResult result = backup(plan);
|
||||
|
||||
List<VerificationOutcome> outcomes = new BackupVerifier().verifyThrough(result.getRunId(), BackupVerifier.CONTENT, "MANUAL", "tester");
|
||||
assertThat(outcomes).extracting(VerificationOutcome::level).containsExactly("STORAGE", "CONTENT");
|
||||
assertThat(outcomes).extracting(VerificationOutcome::status).containsExactly("PASSED", "PASSED");
|
||||
|
||||
VerificationOutcome content = outcomes.get(1);
|
||||
assertThat(content.tablesChecked()).isEqualTo(2);
|
||||
assertThat(content.rowsChecked()).isEqualTo(10);
|
||||
assertThat(content.filesChecked()).isEqualTo(3);
|
||||
assertThat(content.mismatches()).isZero();
|
||||
|
||||
List<QRecord> rows = new BackupCatalog().where(BackupsTables.VERIFICATION, "backupRunId", result.getRunId());
|
||||
assertThat(rows).hasSize(2);
|
||||
QRecord contentRow = rows.stream().filter(r -> "CONTENT".equals(r.getValueString("level"))).findFirst().orElseThrow();
|
||||
assertThat(contentRow.getValueString("status")).isEqualTo("PASSED");
|
||||
assertThat(contentRow.getValueLong("rowsChecked")).isEqualTo(10L);
|
||||
assertThat(contentRow.getValueString("detailJson")).contains("\"table\":\"blogPost\"").contains("\"expectedRows\":7").contains("\"actualRows\":7").contains("\"ok\":true");
|
||||
|
||||
assertThat(run(result.getRunId()).getValueString("verificationStatus")).isEqualTo("CONTENT_OK");
|
||||
QRecord reloaded = reload(plan);
|
||||
assertThat(reloaded.getValueInteger("lastVerifiedRunId")).isEqualTo(result.getRunId());
|
||||
assertThat(reloaded.getValueInstant("lastVerifiedDate")).isEqualTo(VerifyTestSupport.clock.instant());
|
||||
|
||||
assertThat(new BackupCatalog().query(BackupsTables.FILE_OBJECT, new com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter()))
|
||||
.allSatisfy(f -> assertThat(f.getValueString("lastVerifyStatus")).isEqualTo("PASSED"));
|
||||
assertThat(sink.open).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** An unknown level is refused; an unknown run is refused.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testBadArguments() throws Exception
|
||||
{
|
||||
BackupRunResult result = backup(plan(Map.of()));
|
||||
assertThatThrownBy(() -> new BackupVerifier().verify(result.getRunId(), "WHATEVER", "MANUAL", "t")).hasMessageContaining("Unknown verification level");
|
||||
assertThatThrownBy(() -> new BackupVerifier().verify(99999, "STORAGE", "MANUAL", "t")).hasMessageContaining("No backup run");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A flipped byte in a part is caught by CONTENT with a clear message, the run is
|
||||
** marked FAILED, and an alert is raised; STORAGE (sizes only) still passes.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testCorruptedPartFailsContent() throws Exception
|
||||
{
|
||||
QRecord plan = plan(Map.of());
|
||||
BackupRunResult result = backup(plan);
|
||||
String key = VerifyTestSupport.firstPartKey(result, "blogPost");
|
||||
byte[] bytes = BackupsTestUtils.BACKUPS.getBytes(key).clone();
|
||||
bytes[bytes.length / 2] ^= 0x55;
|
||||
BackupsTestUtils.BACKUPS.corrupt(key, bytes);
|
||||
|
||||
List<VerificationOutcome> outcomes = new BackupVerifier().verifyThrough(result.getRunId(), BackupVerifier.CONTENT, "MANUAL", "tester");
|
||||
assertThat(outcomes).extracting(VerificationOutcome::status).containsExactly("PASSED", "FAILED");
|
||||
assertThat(outcomes.get(1).messages()).anyMatch(m -> m.contains("tables/blogPost/part-00001.jsonl.gz") && (m.contains("corrupted") || m.contains("checksum mismatch")));
|
||||
assertThat(run(result.getRunId()).getValueString("verificationStatus")).isEqualTo("FAILED");
|
||||
assertThat(reload(plan).getValueInstant("lastVerifiedDate")).isNull();
|
||||
|
||||
String key2 = BackupAlerts.verifyKey("nightly", "CONTENT");
|
||||
assertThat(sink.isOpen(key2)).isTrue();
|
||||
assertThat(sink.openAlert(key2).severity()).isEqualTo(com.kingsrook.qbits.backups.alerts.BackupAlertSink.Severity.ERROR);
|
||||
assertThat(sink.openAlert(key2).type()).isEqualTo("backups.verify.failed");
|
||||
assertThat(sink.openAlert(key2).detail()).containsEntry("plan", "nightly").containsEntry("level", "CONTENT").doesNotContainKey("key");
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
// a later verification of a good run of the same plan resolves the alert //
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
BackupRunResult good = backup(plan);
|
||||
assertThat(new BackupVerifier().verifyThrough(good.getRunId(), BackupVerifier.CONTENT, "MANUAL", "tester")).extracting(VerificationOutcome::status).containsExactly("PASSED", "PASSED");
|
||||
assertThat(sink.isOpen(key2)).isFalse();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A part that is valid gzip but whose bytes differ from what was written fails
|
||||
** the stored and content checksums, and the row content hash.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testReplacedPartContentFails() throws Exception
|
||||
{
|
||||
BackupRunResult result = backup(plan(Map.of()));
|
||||
String key = VerifyTestSupport.firstPartKey(result, "blogPost");
|
||||
String text = gunzip(BackupsTestUtils.BACKUPS.getBytes(key)).replace("Post 1", "Post X");
|
||||
BackupsTestUtils.BACKUPS.putBytes(key, gzip(text));
|
||||
|
||||
VerificationOutcome outcome = new BackupVerifier().verify(result.getRunId(), BackupVerifier.CONTENT, "MANUAL", "t");
|
||||
assertThat(outcome.status()).isEqualTo("FAILED");
|
||||
assertThat(outcome.messages()).anyMatch(m -> m.startsWith("Stored checksum mismatch")).anyMatch(m -> m.startsWith("Content checksum mismatch")).anyMatch(m -> m.startsWith("Row content hash mismatch"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A truncated part fails the STORAGE size check.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testTruncatedPartFailsStorage() throws Exception
|
||||
{
|
||||
BackupRunResult result = backup(plan(Map.of()));
|
||||
String key = VerifyTestSupport.firstPartKey(result, "blogPost");
|
||||
byte[] bytes = BackupsTestUtils.BACKUPS.getBytes(key);
|
||||
BackupsTestUtils.BACKUPS.corrupt(key, java.util.Arrays.copyOf(bytes, bytes.length - 5));
|
||||
|
||||
List<VerificationOutcome> outcomes = new BackupVerifier().verifyThrough(result.getRunId(), BackupVerifier.CONTENT, "MANUAL", "t");
|
||||
assertThat(outcomes).hasSize(1);
|
||||
assertThat(outcomes.get(0).status()).isEqualTo("FAILED");
|
||||
assertThat(outcomes.get(0).messages()).anyMatch(m -> m.startsWith("Size mismatch for") && m.contains("blogPost"));
|
||||
assertThat(sink.isOpen(BackupAlerts.verifyKey("nightly", "STORAGE"))).isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A deleted part is a missing object at STORAGE level.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testDeletedPartFailsStorage() throws Exception
|
||||
{
|
||||
BackupRunResult result = backup(plan(Map.of()));
|
||||
BackupsTestUtils.BACKUPS.removeIgnoringLock(VerifyTestSupport.firstPartKey(result, "author"));
|
||||
|
||||
VerificationOutcome outcome = new BackupVerifier().verify(result.getRunId(), BackupVerifier.STORAGE, "MANUAL", "t");
|
||||
assertThat(outcome.status()).isEqualTo("FAILED");
|
||||
assertThat(outcome.messages()).anyMatch(m -> m.startsWith("Missing object:") && m.contains("table author"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A provider that stores SHA-256 checksums: a stored value that differs from the
|
||||
** manifest is a STORAGE mismatch.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testProviderChecksumMismatchFailsStorage() throws Exception
|
||||
{
|
||||
BackupRunResult result = backup(plan(Map.of()));
|
||||
String key = VerifyTestSupport.firstPartKey(result, "author");
|
||||
byte[] bytes = BackupsTestUtils.BACKUPS.getBytes(key);
|
||||
try(var out = BackupsTestUtils.BACKUPS.openWrite(key, WriteOptions.create().withMetadata("sha256", "0000")))
|
||||
{
|
||||
out.write(bytes);
|
||||
}
|
||||
VerificationOutcome outcome = new BackupVerifier().verify(result.getRunId(), BackupVerifier.STORAGE, "MANUAL", "t");
|
||||
assertThat(outcome.status()).isEqualTo("FAILED");
|
||||
assertThat(outcome.messages()).anyMatch(m -> m.startsWith("Checksum mismatch for"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A file-store object that was deleted fails STORAGE (head) and CONTENT.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testDeletedFileObjectFails() throws Exception
|
||||
{
|
||||
BackupRunResult result = backup(plan(Map.of()));
|
||||
QRecord file = new BackupCatalog().query(BackupsTables.FILE_OBJECT, new com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter()).get(0);
|
||||
BackupsTestUtils.BACKUPS.removeIgnoringLock(file.getValueString("objectKey"));
|
||||
|
||||
VerificationOutcome storage = new BackupVerifier().verify(result.getRunId(), BackupVerifier.STORAGE, "MANUAL", "t");
|
||||
assertThat(storage.status()).isEqualTo("FAILED");
|
||||
assertThat(storage.messages()).anyMatch(m -> m.startsWith("File store object missing: sha256 " + file.getValueString("sha256")));
|
||||
|
||||
VerificationOutcome content = new BackupVerifier().verify(result.getRunId(), BackupVerifier.CONTENT, "MANUAL", "t");
|
||||
assertThat(content.status()).isEqualTo("FAILED");
|
||||
assertThat(content.messages()).anyMatch(m -> m.startsWith("File store object missing"));
|
||||
assertThat(new BackupCatalog().get(BackupsTables.FILE_OBJECT, file.getValueInteger("id")).orElseThrow().getValueString("lastVerifyStatus")).isEqualTo("FAILED");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A file-store object with the right size but the wrong bytes (a wrong checksum)
|
||||
** fails CONTENT only.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testWrongFileChecksumFailsContent() throws Exception
|
||||
{
|
||||
BackupRunResult result = backup(plan(Map.of()));
|
||||
QRecord file = new BackupCatalog().query(BackupsTables.FILE_OBJECT, new com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter()).get(0);
|
||||
byte[] bytes = BackupsTestUtils.BACKUPS.getBytes(file.getValueString("objectKey")).clone();
|
||||
bytes[0] ^= 0x01;
|
||||
BackupsTestUtils.BACKUPS.corrupt(file.getValueString("objectKey"), bytes);
|
||||
|
||||
assertThat(new BackupVerifier().verify(result.getRunId(), BackupVerifier.STORAGE, "MANUAL", "t").status()).isEqualTo("PASSED");
|
||||
VerificationOutcome content = new BackupVerifier().verify(result.getRunId(), BackupVerifier.CONTENT, "MANUAL", "t");
|
||||
assertThat(content.status()).isEqualTo("FAILED");
|
||||
assertThat(content.messages()).anyMatch(m -> m.contains("has the wrong content") && m.contains(file.getValueString("sha256")));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A manifest that is gone: the run is not usable.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testMissingManifest() throws Exception
|
||||
{
|
||||
BackupRunResult result = backup(plan(Map.of()));
|
||||
BackupsTestUtils.BACKUPS.removeIgnoringLock(result.getManifestKey());
|
||||
|
||||
VerificationOutcome outcome = new BackupVerifier().verify(result.getRunId(), BackupVerifier.STORAGE, "MANUAL", "t");
|
||||
assertThat(outcome.status()).isEqualTo("FAILED");
|
||||
assertThat(outcome.messages()).containsExactly("The manifest is missing from the bucket, so this backup is not usable");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A manifest that was changed after it was written (its hash no longer matches
|
||||
** the catalog) and one that is not JSON.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testTamperedAndUnreadableManifest() throws Exception
|
||||
{
|
||||
BackupRunResult result = backup(plan(Map.of()));
|
||||
byte[] bytes = BackupsTestUtils.BACKUPS.getBytes(result.getManifestKey());
|
||||
BackupsTestUtils.BACKUPS.putBytes(result.getManifestKey(), (new String(bytes, StandardCharsets.UTF_8) + " ").getBytes(StandardCharsets.UTF_8));
|
||||
VerificationOutcome tampered = new BackupVerifier().verify(result.getRunId(), BackupVerifier.STORAGE, "MANUAL", "t");
|
||||
assertThat(tampered.status()).isEqualTo("FAILED");
|
||||
assertThat(tampered.messages()).anyMatch(m -> m.contains("manifest changed after it was written"));
|
||||
|
||||
BackupsTestUtils.BACKUPS.putBytes(result.getManifestKey(), "not json".getBytes(StandardCharsets.UTF_8));
|
||||
VerificationOutcome garbage = new BackupVerifier().verify(result.getRunId(), BackupVerifier.STORAGE, "MANUAL", "t");
|
||||
assertThat(garbage.status()).isEqualTo("FAILED");
|
||||
assertThat(garbage.messages()).anyMatch(m -> m.contains("not valid JSON"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A run with no recorded bucket cannot be examined at all: ERROR, not FAILED.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testUnexaminableRunIsError() throws Exception
|
||||
{
|
||||
BackupRunResult result = backup(plan(Map.of()));
|
||||
new BackupCatalog().update(BackupsTables.RUN, result.getRunId(), java.util.Collections.singletonMap("storageBucketAlias", null));
|
||||
VerificationOutcome outcome = new BackupVerifier().verify(result.getRunId(), BackupVerifier.STORAGE, "MANUAL", "t");
|
||||
assertThat(outcome.status()).isEqualTo("ERROR");
|
||||
assertThat(outcome.messages()).anyMatch(m -> m.contains("could not be carried out"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Encrypted runs verify through the decrypt hook; a wrong key fails clearly; no
|
||||
** hook at all is an ERROR that says what to configure.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testEncryptedRun() throws Exception
|
||||
{
|
||||
QRecord plan = plan(Map.of());
|
||||
BackupRunResult result = VerifyTestSupport.backupEncrypted(plan, new VerifyTestSupport.XorTransform(0x5A), "kek-1");
|
||||
|
||||
assertThat(new BackupVerifier().verify(result.getRunId(), BackupVerifier.CONTENT, "MANUAL", "t").status()).isEqualTo("ERROR");
|
||||
assertThat(new BackupVerifier().verify(result.getRunId(), BackupVerifier.CONTENT, "MANUAL", "t").messages()).anyMatch(m -> m.contains("no decrypt hook is configured"));
|
||||
|
||||
BackupsRuntime.getConfig().withArtifactTransformResolver(ref -> null);
|
||||
assertThat(new BackupVerifier().verify(result.getRunId(), BackupVerifier.CONTENT, "MANUAL", "t").messages()).anyMatch(m -> m.contains("No key is available for the key reference kek-1"));
|
||||
|
||||
BackupsRuntime.getConfig().withArtifactTransformResolver(ref -> new VerifyTestSupport.XorTransform(0x11));
|
||||
VerificationOutcome wrongKey = new BackupVerifier().verify(result.getRunId(), BackupVerifier.CONTENT, "MANUAL", "t");
|
||||
assertThat(wrongKey.status()).isEqualTo("FAILED");
|
||||
assertThat(wrongKey.messages()).anyMatch(m -> m.contains("corrupted, or the key is wrong"));
|
||||
|
||||
BackupsRuntime.getConfig().withArtifactTransformResolver(ref -> new VerifyTestSupport.XorTransform(0x5A));
|
||||
List<VerificationOutcome> good = new BackupVerifier().verifyThrough(result.getRunId(), BackupVerifier.TEST_RESTORE, "MANUAL", "t");
|
||||
assertThat(good).extracting(VerificationOutcome::status).containsExactly("PASSED", "PASSED", "PASSED");
|
||||
assertThat(good.get(1).filesChecked()).isEqualTo(3);
|
||||
assertThat(run(result.getRunId()).getValueString("verificationStatus")).isEqualTo("TEST_RESTORE_OK");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Files verified within fileReverifyDays are skipped; later they are checked again.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testFileReverifyDaysBoundsTheWork() throws Exception
|
||||
{
|
||||
BackupRunResult result = backup(plan(Map.of()));
|
||||
assertThat(new BackupVerifier().verify(result.getRunId(), BackupVerifier.CONTENT, "MANUAL", "t").filesChecked()).isEqualTo(3);
|
||||
|
||||
VerificationOutcome again = new BackupVerifier().verify(result.getRunId(), BackupVerifier.CONTENT, "MANUAL", "t");
|
||||
assertThat(again.filesChecked()).isZero();
|
||||
assertThat(again.filesSkipped()).isEqualTo(3);
|
||||
|
||||
VerifyTestSupport.clock.advance(Duration.ofDays(31));
|
||||
VerificationOutcome later = new BackupVerifier().verify(result.getRunId(), BackupVerifier.CONTENT, "MANUAL", "t");
|
||||
assertThat(later.filesChecked()).isEqualTo(3);
|
||||
assertThat(later.filesSkipped()).isZero();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A cheap STORAGE pass does not clear a FAILED found by CONTENT, and never
|
||||
** downgrades a CONTENT_OK; a later CONTENT pass does clear the failure.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testVerificationStatusRules() throws Exception
|
||||
{
|
||||
BackupRunResult result = backup(plan(Map.of()));
|
||||
new BackupVerifier().verify(result.getRunId(), BackupVerifier.CONTENT, "MANUAL", "t");
|
||||
assertThat(run(result.getRunId()).getValueString("verificationStatus")).isEqualTo("CONTENT_OK");
|
||||
new BackupVerifier().verify(result.getRunId(), BackupVerifier.STORAGE, "MANUAL", "t");
|
||||
assertThat(run(result.getRunId()).getValueString("verificationStatus")).isEqualTo("CONTENT_OK");
|
||||
|
||||
String key = VerifyTestSupport.firstPartKey(result, "blogPost");
|
||||
byte[] bytes = BackupsTestUtils.BACKUPS.getBytes(key).clone();
|
||||
bytes[bytes.length / 2] ^= 0x55;
|
||||
BackupsTestUtils.BACKUPS.corrupt(key, bytes);
|
||||
new BackupVerifier().verify(result.getRunId(), BackupVerifier.CONTENT, "MANUAL", "t");
|
||||
assertThat(run(result.getRunId()).getValueString("verificationStatus")).isEqualTo("FAILED");
|
||||
new BackupVerifier().verify(result.getRunId(), BackupVerifier.STORAGE, "MANUAL", "t");
|
||||
assertThat(run(result.getRunId()).getValueString("verificationStatus")).isEqualTo("FAILED");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** An ad-hoc run (no plan) verifies without touching a plan or raising plan alerts.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testAdHocRunHasNoPlan() throws Exception
|
||||
{
|
||||
BackupRunResult result = backup(plan(Map.of()));
|
||||
new BackupCatalog().update(BackupsTables.RUN, result.getRunId(), java.util.Collections.singletonMap("backupPlanId", null));
|
||||
assertThat(new BackupVerifier().verify(result.getRunId(), BackupVerifier.CONTENT, "MANUAL", "t").status()).isEqualTo("PASSED");
|
||||
assertThat(sink.raises).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A table that is no longer in the instance is checked as JSON with a warning; a
|
||||
** table whose schema changed is parsed without the exact round trip.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testSchemaDriftIsAWarning() throws Exception
|
||||
{
|
||||
BackupRunResult result = backup(plan(Map.of()));
|
||||
QContext.getQInstance().getTable("blogPost").addField(new com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData("added", com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType.STRING));
|
||||
QContext.getQInstance().getTables().remove("author");
|
||||
|
||||
VerificationOutcome outcome = new BackupVerifier().verify(result.getRunId(), BackupVerifier.CONTENT, "MANUAL", "t");
|
||||
assertThat(outcome.status()).isEqualTo("PASSED_WITH_WARNINGS");
|
||||
assertThat(outcome.messages()).anyMatch(m -> m.contains("Table author no longer exists")).anyMatch(m -> m.contains("Table blogPost has changed since this backup"));
|
||||
assertThat(outcome.passed()).isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** CSV parts are hashed (stored and decoded bytes) but rows are not recounted.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testCsvOnlyPlanIsHashedWithAWarning() throws Exception
|
||||
{
|
||||
BackupRunResult result = backup(plan(Map.of("format", "CSV")));
|
||||
assertThat(result.getStatus()).startsWith("SUCCEEDED");
|
||||
VerificationOutcome outcome = new BackupVerifier().verify(result.getRunId(), BackupVerifier.CONTENT, "MANUAL", "t");
|
||||
assertThat(outcome.status()).isEqualTo("PASSED_WITH_WARNINGS");
|
||||
assertThat(outcome.messages()).anyMatch(m -> m.contains("only CSV parts"));
|
||||
|
||||
BackupsTestUtils.BACKUPS.corrupt(VerifyTestSupport.firstPartKey(result, "author"), new byte[] { 1, 2, 3 });
|
||||
assertThat(new BackupVerifier().verify(result.getRunId(), BackupVerifier.CONTENT, "MANUAL", "t").status()).isEqualTo("FAILED");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A good run test-restores into the memory scratch target: every table matches,
|
||||
** every file is restored and re-hashed, and the scratch tables and files are gone.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testTestRestoreMemory() throws Exception
|
||||
{
|
||||
QRecord plan = plan(Map.of());
|
||||
BackupRunResult result = backup(plan);
|
||||
|
||||
VerificationOutcome outcome = new BackupVerifier().verify(result.getRunId(), BackupVerifier.TEST_RESTORE, "MANUAL", "t");
|
||||
assertThat(outcome.messages()).isEmpty();
|
||||
assertThat(outcome.status()).isEqualTo("PASSED");
|
||||
assertThat(outcome.rowsChecked()).isEqualTo(10);
|
||||
assertThat(outcome.filesChecked()).isEqualTo(3);
|
||||
assertThat(outcome.scratchTarget()).startsWith("memory:qqq_verify_");
|
||||
|
||||
assertThat(QContext.getQInstance().getTables().keySet()).noneMatch(n -> n.startsWith("qqq_verify_"));
|
||||
assertThat(BackupsTestUtils.BACKUPS.list(TestRestoreVerifier.SCRATCH_FILE_PREFIX)).isEmpty();
|
||||
QRecord row = new BackupCatalog().where(BackupsTables.VERIFICATION, "level", "TEST_RESTORE").get(0);
|
||||
assertThat(row.getValueString("scratchTarget")).startsWith("memory:");
|
||||
assertThat(row.getValueString("detailJson")).contains("\"filesRestored\":3");
|
||||
assertThat(run(result.getRunId()).getValueString("verificationStatus")).isEqualTo("TEST_RESTORE_OK");
|
||||
assertThat(reload(plan).getValueInteger("lastVerifiedRunId")).isEqualTo(result.getRunId());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Test restore catches rows that differ from the backed up rows, a missing
|
||||
** table, an unreadable part and a stored file that is not its hash.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testTestRestoreCatchesProblems() throws Exception
|
||||
{
|
||||
BackupRunResult result = backup(plan(Map.of()));
|
||||
String key = VerifyTestSupport.firstPartKey(result, "blogPost");
|
||||
byte[] good = BackupsTestUtils.BACKUPS.getBytes(key);
|
||||
|
||||
BackupsTestUtils.BACKUPS.putBytes(key, gzip(gunzip(good).replace("Post 1", "Post X")));
|
||||
VerificationOutcome changed = new BackupVerifier().verify(result.getRunId(), BackupVerifier.TEST_RESTORE, "MANUAL", "t");
|
||||
assertThat(changed.status()).isEqualTo("FAILED");
|
||||
assertThat(changed.messages()).anyMatch(m -> m.contains("does not reproduce the backed up rows"));
|
||||
assertThat(QContext.getQInstance().getTables().keySet()).noneMatch(n -> n.startsWith("qqq_verify_"));
|
||||
|
||||
BackupsTestUtils.BACKUPS.putBytes(key, new byte[] { 1, 2, 3 });
|
||||
VerificationOutcome unreadable = new BackupVerifier().verify(result.getRunId(), BackupVerifier.TEST_RESTORE, "MANUAL", "t");
|
||||
assertThat(unreadable.status()).isEqualTo("FAILED");
|
||||
assertThat(unreadable.messages()).anyMatch(m -> m.contains("could not be restored"));
|
||||
BackupsTestUtils.BACKUPS.putBytes(key, good);
|
||||
|
||||
QRecord file = new BackupCatalog().query(BackupsTables.FILE_OBJECT, new com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter()).get(0);
|
||||
BackupsTestUtils.BACKUPS.corrupt(file.getValueString("objectKey"), "different bytes".getBytes(StandardCharsets.UTF_8));
|
||||
VerificationOutcome badFile = new BackupVerifier().verify(result.getRunId(), BackupVerifier.TEST_RESTORE, "MANUAL", "t");
|
||||
assertThat(badFile.status()).isEqualTo("FAILED");
|
||||
assertThat(badFile.messages()).anyMatch(m -> m.startsWith("Test restore of file sha256")).anyMatch(m -> m.contains("do not resolve to a restored file"));
|
||||
assertThat(BackupsTestUtils.BACKUPS.list(TestRestoreVerifier.SCRATCH_FILE_PREFIX)).isEmpty();
|
||||
|
||||
QContext.getQInstance().getTables().remove("author");
|
||||
assertThat(new BackupVerifier().verify(result.getRunId(), BackupVerifier.TEST_RESTORE, "MANUAL", "t").messages()).anyMatch(m -> m.contains("Table author no longer exists"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** verifyFileSamplePercent 0 restores no files; CSV-only tables are skipped.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testTestRestoreFileSampleAndCsv() throws Exception
|
||||
{
|
||||
BackupRunResult result = backup(plan(Map.of("verifyFileSamplePercent", 0)));
|
||||
VerificationOutcome outcome = new BackupVerifier().verify(result.getRunId(), BackupVerifier.TEST_RESTORE, "MANUAL", "t");
|
||||
assertThat(outcome.status()).isEqualTo("PASSED");
|
||||
assertThat(outcome.filesChecked()).isZero();
|
||||
|
||||
BackupRunResult csv = backup(VerifyTestSupport.plan(Map.of("name", "Csv", "planKey", "csv", "format", "CSV")));
|
||||
VerificationOutcome csvOutcome = new BackupVerifier().verify(csv.getRunId(), BackupVerifier.TEST_RESTORE, "MANUAL", "t");
|
||||
assertThat(csvOutcome.status()).isEqualTo("PASSED_WITH_WARNINGS");
|
||||
assertThat(csvOutcome.messages()).anyMatch(m -> m.contains("only CSV parts and was not test-restored"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The RDBMS_SCHEMA scratch target is a stub: it ends the verification with an
|
||||
** ERROR that says so, rather than passing.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRdbmsSchemaScratchTargetIsAStub() throws Exception
|
||||
{
|
||||
BackupsRuntime.getConfig().withScratchTarget(new RdbmsSchemaScratchTarget("scratchdb"));
|
||||
BackupRunResult result = backup(plan(Map.of()));
|
||||
|
||||
VerificationOutcome outcome = new BackupVerifier().verify(result.getRunId(), BackupVerifier.TEST_RESTORE, "MANUAL", "t");
|
||||
assertThat(outcome.status()).isEqualTo("ERROR");
|
||||
assertThat(outcome.messages()).anyMatch(m -> m.contains("RDBMS_SCHEMA scratch target is not implemented yet"));
|
||||
|
||||
ScratchTarget target = BackupsRuntime.getConfig().getScratchTarget();
|
||||
assertThat(target.type()).isEqualTo("RDBMS_SCHEMA");
|
||||
assertThat(target.describe("2026-A")).isEqualTo("scratchdb:schema=qqq_verify_2026_a");
|
||||
target.drop("2026-A");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The outcome of a verification that has not passed or failed reads sensibly.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testManifestReadableByTheVerifier() throws Exception
|
||||
{
|
||||
BackupRunResult result = backup(plan(Map.of()));
|
||||
BackupManifest manifest = BackupManifest.read(BackupsTestUtils.BACKUPS.getBytes(result.getManifestKey()));
|
||||
assertThat(manifest.tables()).hasSize(2);
|
||||
assertThat(new VerificationOutcome(1, 2, "STORAGE", "ERROR", 0, 0, 1, 0, 0, 0, List.of(), null).passed()).isFalse();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static String gunzip(byte[] bytes) throws Exception
|
||||
{
|
||||
return (new String(new java.util.zip.GZIPInputStream(new java.io.ByteArrayInputStream(bytes)).readAllBytes(), StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static byte[] gzip(String text) throws Exception
|
||||
{
|
||||
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
|
||||
try(GZIPOutputStream out = new GZIPOutputStream(bytes))
|
||||
{
|
||||
out.write(text.getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
return (bytes.toByteArray());
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user