feat(restore): restore, catalog import and download (spec 9.3-9.7) #6
No files matched your search
@@ -74,6 +74,8 @@ public final class BackupsProcesses
|
||||
processes.add(permission(etl(RUN_PLAN, "Run backup now", BackupsTables.PLAN, RunPlanTransform.class, RunPlanLoad.class,
|
||||
"This is a preview of what the backup will include. Nothing is written until you continue."), BackupsPermissions.RUN));
|
||||
processes.addAll(VerifyProcesses.processes());
|
||||
processes.addAll(RestoreProcesses.processes());
|
||||
processes.addAll(CatalogProcesses.processes());
|
||||
return (processes);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,443 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.processes;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.backups.BackupsRuntime;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.permissions.BackupsPermissions;
|
||||
import com.kingsrook.qbits.backups.restore.CatalogImporter;
|
||||
import com.kingsrook.qbits.backups.restore.RestoreSummaries;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qbits.backups.storage.BackupStorage;
|
||||
import com.kingsrook.qqq.backend.core.actions.audits.AuditAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The catalog and download processes (spec 9.6 and 9.7): Import backups from
|
||||
** storage (a table action on backups_run) and Download (a record action on a run
|
||||
** or an artifact). Standard streamed ETL, Review and Result summary lines.
|
||||
*******************************************************************************/
|
||||
public final class CatalogProcesses
|
||||
{
|
||||
public static final String IMPORT_CATALOG = "backupImportCatalog";
|
||||
public static final String DOWNLOAD = "backupDownload";
|
||||
public static final String DOWNLOAD_ARTIFACT = "backupDownloadArtifact";
|
||||
|
||||
public static final Duration LINK_TTL = Duration.ofMinutes(15);
|
||||
|
||||
private static final QLogger LOG = QLogger.getLogger(CatalogProcesses.class);
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private CatalogProcesses()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The processes defined here.
|
||||
*******************************************************************************/
|
||||
public static List<QProcessMetaData> processes()
|
||||
{
|
||||
List<QProcessMetaData> processes = new ArrayList<>();
|
||||
|
||||
QProcessMetaData importProcess = definition(IMPORT_CATALOG, "Import backups from storage", BackupsTables.RUN, ImportExtract.class, ImportTransform.class, ImportLoad.class,
|
||||
"These are the backups found in storage that are not in the catalog yet. Nothing is written until you continue.");
|
||||
importProcess.withStep(0, new QFrontendStepMetaData()
|
||||
.withName("input")
|
||||
.withLabel("Where are the backups?")
|
||||
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
|
||||
.withFormField(new QFieldMetaData("bucketAlias", QFieldType.STRING).withLabel("Backup bucket alias").withIsRequired(true))
|
||||
.withFormField(new QFieldMetaData("prefix", QFieldType.STRING).withLabel("Folder (prefix)").withDefaultValue("qqq-backups/")));
|
||||
importProcess.setMinInputRecords(0);
|
||||
processes.add(BackupsProcesses.permission(importProcess, BackupsPermissions.ADMIN));
|
||||
|
||||
processes.add(BackupsProcesses.permission(definition(DOWNLOAD, "Download", BackupsTables.RUN, ExtractViaQueryStep.class, DownloadTransform.class, DownloadLoad.class,
|
||||
"These are the files you will get download links for. Nothing is changed."), BackupsPermissions.RESTORE));
|
||||
processes.add(BackupsProcesses.permission(definition(DOWNLOAD_ARTIFACT, "Download", BackupsTables.ARTIFACT, ExtractViaQueryStep.class, DownloadTransform.class, DownloadLoad.class,
|
||||
"These are the files you will get download links for. Nothing is changed."), BackupsPermissions.RESTORE));
|
||||
return (processes);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static QProcessMetaData definition(String name, String label, String table, Class<? extends AbstractExtractStep> extract,
|
||||
Class<? extends AbstractTransformStep> transform, Class<? extends AbstractLoadStep> load, String preview)
|
||||
{
|
||||
Map<String, Serializable> values = new HashMap<>();
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, table);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION, true);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_TRANSACTION_LEVEL, StreamedETLWithFrontendProcess.TRANSACTION_LEVEL_AUTO_COMMIT);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, preview);
|
||||
QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(extract, transform, load, values);
|
||||
process.setName(name);
|
||||
process.setLabel(label);
|
||||
process.setTableName(table);
|
||||
process.setPermissionRules(BackupsTables.permissionRules());
|
||||
return (process);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Lists the manifests under the bucket alias and prefix; one record per manifest.
|
||||
*******************************************************************************/
|
||||
public static class ImportExtract extends AbstractExtractStep
|
||||
{
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
String alias = input.getValueString("bucketAlias");
|
||||
if(alias == null || alias.isBlank())
|
||||
{
|
||||
throw (new QUserFacingException("Choose the backup bucket alias to import from."));
|
||||
}
|
||||
BackupStorage storage = BackupsRuntime.storageFor(alias);
|
||||
for(String key : new CatalogImporter().findManifestKeys(storage, input.getValueString("prefix")))
|
||||
{
|
||||
getRecordPipe().addRecord(new QRecord().withTableName(BackupsTables.RUN).withValue("manifestKey", key).withValue("storageBucketAlias", alias));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Review: what each manifest is (new, already in the catalog, or damaged).
|
||||
*******************************************************************************/
|
||||
public static class ImportTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final ProcessSummaryLine found = RestoreSummaries.line(Status.INFO, 0, "backup was found in storage", "backups were found in storage", "backup was found in storage", "backups were found in storage");
|
||||
private final ProcessSummaryLine newRuns = RestoreSummaries.line(Status.OK, 0, "backup will be added to the catalog", "backups will be added to the catalog", "backup was added to the catalog", "backups were added to the catalog");
|
||||
private final ProcessSummaryLine existing = RestoreSummaries.line(Status.INFO, 0, "backup is already in the catalog (skipped)", "backups are already in the catalog (skipped)", "backup was already in the catalog", "backups were already in the catalog");
|
||||
private final ProcessSummaryLine invalid = RestoreSummaries.line(Status.ERROR, 0, "backup has a manifest or checksum problem and will be skipped", "backups have a manifest or checksum problem and will be skipped",
|
||||
"backup had a manifest or checksum problem and was skipped", "backups had a manifest or checksum problem and were skipped");
|
||||
private final ArrayList<String> problems = new ArrayList<>();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
for(QRecord record : input.getRecords())
|
||||
{
|
||||
BackupStorage storage = BackupsRuntime.storageFor(record.getValueString("storageBucketAlias"));
|
||||
CatalogImporter.Inspection inspection = new CatalogImporter().inspect(storage, record.getValueString("manifestKey"));
|
||||
found.incrementCount();
|
||||
switch(inspection.status())
|
||||
{
|
||||
case CatalogImporter.Inspection.NEW ->
|
||||
{
|
||||
newRuns.incrementCount();
|
||||
output.getRecords().add(record);
|
||||
}
|
||||
case CatalogImporter.Inspection.EXISTS -> existing.incrementCount();
|
||||
default ->
|
||||
{
|
||||
invalid.incrementCount();
|
||||
for(String problem : inspection.problems())
|
||||
{
|
||||
problems.add(inspection.manifestKey() + ": " + problem);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
found.addSelfToListIfAnyCount(lines);
|
||||
newRuns.addSelfToListIfAnyCount(lines);
|
||||
existing.addSelfToListIfAnyCount(lines);
|
||||
invalid.setBulletsOfText(problems);
|
||||
invalid.addSelfToListIfAnyCount(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Execute: create the run and artifact rows from each new manifest.
|
||||
*******************************************************************************/
|
||||
public static class ImportLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final ProcessSummaryLine imported = RestoreSummaries.line(Status.OK, 0, "backup will be added to the catalog", "backups will be added to the catalog", "backup was added to the catalog", "backups were added to the catalog");
|
||||
private final ProcessSummaryLine skipped = RestoreSummaries.line(Status.WARNING, 0, "backup was not added", "backups were not added", "backup was not added", "backups were not added");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
CatalogImporter importer = new CatalogImporter();
|
||||
for(QRecord record : input.getRecords())
|
||||
{
|
||||
String alias = record.getValueString("storageBucketAlias");
|
||||
BackupStorage storage = BackupsRuntime.storageFor(alias);
|
||||
CatalogImporter.Inspection inspection = importer.inspect(storage, record.getValueString("manifestKey"));
|
||||
if(CatalogImporter.Inspection.NEW.equals(inspection.status()))
|
||||
{
|
||||
imported.incrementCountAndAddPrimaryKey(importer.importRun(alias, storage, inspection));
|
||||
}
|
||||
else
|
||||
{
|
||||
skipped.incrementCount();
|
||||
}
|
||||
output.getRecords().add(record);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
imported.addSelfToListIfAnyCount(lines);
|
||||
skipped.addSelfToListIfAnyCount(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One object to link to.
|
||||
*******************************************************************************/
|
||||
record DownloadItem(String auditTable, Integer auditId, BackupStorage storage, String key, String downloadName, boolean encrypted, long size)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The objects the selected records ask for: for a run, all of its artifacts; for
|
||||
** an artifact, itself plus the manifest and checksums of its run.
|
||||
*******************************************************************************/
|
||||
static List<DownloadItem> items(List<QRecord> records) throws QException
|
||||
{
|
||||
BackupCatalog catalog = new BackupCatalog();
|
||||
Map<String, DownloadItem> items = new LinkedHashMap<>();
|
||||
for(QRecord record : records)
|
||||
{
|
||||
boolean isArtifact = BackupsTables.ARTIFACT.equals(record.getTableName()) || record.getValue("artifactType") != null;
|
||||
Integer runId = isArtifact ? record.getValueInteger("backupRunId") : record.getValueInteger("id");
|
||||
QRecord run = catalog.get(BackupsTables.RUN, runId).orElseThrow(() -> new QUserFacingException("The backup run was not found."));
|
||||
String table = isArtifact ? BackupsTables.ARTIFACT : BackupsTables.RUN;
|
||||
BackupStorage storage = BackupsRuntime.storageFor(run.getValueString("storageBucketAlias"));
|
||||
|
||||
List<QRecord> artifacts = new ArrayList<>();
|
||||
if(isArtifact)
|
||||
{
|
||||
artifacts.add(record);
|
||||
for(QRecord other : catalog.where(BackupsTables.ARTIFACT, "backupRunId", runId))
|
||||
{
|
||||
if("MANIFEST".equals(other.getValueString("artifactType")) || "CHECKSUMS".equals(other.getValueString("artifactType")))
|
||||
{
|
||||
artifacts.add(other);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
artifacts.addAll(catalog.where(BackupsTables.ARTIFACT, "backupRunId", runId));
|
||||
}
|
||||
|
||||
for(QRecord artifact : artifacts)
|
||||
{
|
||||
if(!"COMPLETE".equals(artifact.getValueString("status")))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
String key = run.getValueString("runPrefix") + artifact.getValueString("objectKey");
|
||||
String name = run.getValueString("runKey") + "-" + artifact.getValueString("objectKey").replace('/', '-');
|
||||
items.putIfAbsent(key, new DownloadItem(table, isArtifact && artifact == record ? artifact.getValueInteger("id") : runId, storage, key, name,
|
||||
Boolean.TRUE.equals(artifact.getValueBoolean("encrypted")), artifact.getValueLong("bytesStored") == null ? 0 : artifact.getValueLong("bytesStored")));
|
||||
}
|
||||
}
|
||||
return (new ArrayList<>(items.values()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Review: what will be linked.
|
||||
*******************************************************************************/
|
||||
public static class DownloadTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final ProcessSummaryLine objects = RestoreSummaries.line(Status.OK, 0, "file will get a download link", "files will get download links", "file got a download link", "files got download links");
|
||||
private final ProcessSummaryLine encrypted = RestoreSummaries.line(Status.INFO, 0, "file is encrypted: you get the ciphertext only, and decrypt it with the command line tool",
|
||||
"files are encrypted: you get the ciphertext only, and decrypt them with the command line tool", "file is encrypted (ciphertext only)", "files are encrypted (ciphertext only)");
|
||||
private long bytes;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
for(DownloadItem item : items(input.getRecords()))
|
||||
{
|
||||
objects.incrementCount();
|
||||
bytes += item.size();
|
||||
if(item.encrypted())
|
||||
{
|
||||
encrypted.incrementCount();
|
||||
}
|
||||
}
|
||||
output.getRecords().addAll(input.getRecords());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
objects.addSelfToListIfAnyCount(lines);
|
||||
encrypted.addSelfToListIfAnyCount(lines);
|
||||
if(objects.getCount() != null && objects.getCount() > 0)
|
||||
{
|
||||
lines.add(RestoreSummaries.message(Status.INFO, "The links last " + LINK_TTL.toMinutes() + " minutes and the download is about " + bytes + " bytes. Each link is recorded in the audit."));
|
||||
}
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Execute: issue the presigned links (ciphertext only), each one audited.
|
||||
*******************************************************************************/
|
||||
public static class DownloadLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final ProcessSummaryLine issued = RestoreSummaries.line(Status.OK, 0, "download link was issued", "download links were issued", "download link was issued", "download links were issued");
|
||||
private final ArrayList<String> links = new ArrayList<>();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
String user = QContext.getQSession() == null || QContext.getQSession().getUser() == null ? "system" : QContext.getQSession().getUser().getIdReference();
|
||||
for(DownloadItem item : items(input.getRecords()))
|
||||
{
|
||||
Instant expires = BackupsRuntime.getClock().instant().plus(LINK_TTL);
|
||||
String url = item.storage().presignGet(item.key(), LINK_TTL, item.downloadName());
|
||||
links.add(item.downloadName() + (item.encrypted() ? " (encrypted)" : "") + ": " + url);
|
||||
issued.incrementCount();
|
||||
LOG.info("Issued a backup download link", logPair("object", item.key()), logPair("user", user), logPair("expires", expires));
|
||||
try
|
||||
{
|
||||
AuditAction.execute(item.auditTable(), item.auditId(), Map.of(), "Download link issued for " + item.key() + " to " + user + ", expires " + expires);
|
||||
}
|
||||
catch(RuntimeException e)
|
||||
{
|
||||
LOG.warn("Could not audit a download link", e, logPair("object", item.key()));
|
||||
}
|
||||
}
|
||||
output.getRecords().addAll(input.getRecords());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
issued.setBulletsOfText(links);
|
||||
issued.addSelfToListIfAnyCount(lines);
|
||||
if(issued.getCount() != null && issued.getCount() > 0)
|
||||
{
|
||||
lines.add(RestoreSummaries.message(Status.INFO, "The links expire in " + LINK_TTL.toMinutes() + " minutes. Encrypted files download as ciphertext: use the command line tool to decrypt them."));
|
||||
}
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.processes;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.permissions.BackupsPermissions;
|
||||
import com.kingsrook.qbits.backups.restore.RestoreEngine;
|
||||
import com.kingsrook.qbits.backups.restore.RestoreOutcome;
|
||||
import com.kingsrook.qbits.backups.restore.RestoreRequest;
|
||||
import com.kingsrook.qbits.backups.restore.RestoreSummaries;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The restore processes (spec 9.3): Restore... on a backup run, and Restore this
|
||||
** table on a backup artifact. Both are the standard streamed ETL. The first
|
||||
** screen takes the inputs; Review is a real dry run of the whole restore with the
|
||||
** per-table summary lines and a masked sample diff; Execute is the restore, and
|
||||
** Result shows what was done. Neither is in the nav.
|
||||
*******************************************************************************/
|
||||
public final class RestoreProcesses
|
||||
{
|
||||
public static final String RESTORE = "backupRestore";
|
||||
public static final String RESTORE_TABLE = "backupRestoreTable";
|
||||
|
||||
public static final String STEP_INPUT = "input";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private RestoreProcesses()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The processes defined here.
|
||||
*******************************************************************************/
|
||||
public static List<QProcessMetaData> processes()
|
||||
{
|
||||
List<QProcessMetaData> processes = new ArrayList<>();
|
||||
processes.add(BackupsProcesses.permission(restore(RESTORE, "Restore...", BackupsTables.RUN), BackupsPermissions.RESTORE));
|
||||
processes.add(BackupsProcesses.permission(restore(RESTORE_TABLE, "Restore this table", BackupsTables.ARTIFACT), BackupsPermissions.RESTORE));
|
||||
return (processes);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One restore process, sourced from the table whose records it is run on.
|
||||
*******************************************************************************/
|
||||
private static QProcessMetaData restore(String name, String label, String table)
|
||||
{
|
||||
Map<String, Serializable> values = new HashMap<>();
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, table);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION, true);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_TRANSACTION_LEVEL, StreamedETLWithFrontendProcess.TRANSACTION_LEVEL_AUTO_COMMIT);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, "This is what the restore will do. Nothing is written until you continue.");
|
||||
|
||||
QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(ExtractViaQueryStep.class, RestoreTransform.class, RestoreLoad.class, values);
|
||||
process.withStep(0, inputStep());
|
||||
process.setName(name);
|
||||
process.setLabel(label);
|
||||
process.setTableName(table);
|
||||
process.setPermissionRules(BackupsTables.permissionRules());
|
||||
return (process);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The first screen: what to restore and how.
|
||||
*******************************************************************************/
|
||||
private static QFrontendStepMetaData inputStep()
|
||||
{
|
||||
return (new QFrontendStepMetaData()
|
||||
.withName(STEP_INPUT)
|
||||
.withLabel("Restore options")
|
||||
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
|
||||
.withFormField(new QFieldMetaData("tables", QFieldType.STRING).withLabel("Tables (blank for every table in the backup)"))
|
||||
.withFormField(new QFieldMetaData("tableFilters", QFieldType.TEXT).withLabel("Only these records (JSON: table name to filter, optional)"))
|
||||
.withFormField(new QFieldMetaData("targetTable", QFieldType.STRING).withLabel("Restore into another table (optional, one table only)"))
|
||||
.withFormField(new QFieldMetaData("mode", QFieldType.STRING).withLabel("Mode").withIsRequired(true).withDefaultValue(RestoreRequest.MODE_INSERT_NEW)
|
||||
.withPossibleValueSourceName(BackupsTables.pvs("restore_mode")))
|
||||
.withFormField(new QFieldMetaData("matchOn", QFieldType.STRING).withLabel("Match records on (PRIMARY_KEY, or a unique key's fields)").withDefaultValue(RestoreRequest.MATCH_PRIMARY_KEY))
|
||||
.withFormField(new QFieldMetaData("idStrategy", QFieldType.STRING).withLabel("Ids").withIsRequired(true).withDefaultValue(RestoreRequest.IDS_PRESERVE)
|
||||
.withPossibleValueSourceName(BackupsTables.pvs("id_strategy")))
|
||||
.withFormField(new QFieldMetaData("suppressAutomations", QFieldType.BOOLEAN).withLabel("Do not trigger automations").withDefaultValue(true))
|
||||
.withFormField(new QFieldMetaData("publishEvents", QFieldType.BOOLEAN).withLabel("Publish change events").withDefaultValue(false))
|
||||
.withFormField(new QFieldMetaData("runCustomizers", QFieldType.BOOLEAN).withLabel("Run table customizers").withDefaultValue(false))
|
||||
.withFormField(new QFieldMetaData("perRecordAudit", QFieldType.BOOLEAN).withLabel("Audit each record (otherwise one summary per table)").withDefaultValue(false))
|
||||
.withFormField(new QFieldMetaData("restoreFiles", QFieldType.BOOLEAN).withLabel("Restore referenced files").withDefaultValue(true))
|
||||
.withFormField(new QFieldMetaData("fileTargetBucketAlias", QFieldType.STRING).withLabel("Restore files to another bucket alias (optional)"))
|
||||
.withFormField(new QFieldMetaData("fileTargetPrefix", QFieldType.STRING).withLabel("Restore files under this prefix (optional)"))
|
||||
.withFormField(new QFieldMetaData("preRestoreBackup", QFieldType.BOOLEAN).withLabel("Back up the target first").withDefaultValue(true))
|
||||
.withFormField(new QFieldMetaData("confirmationText", QFieldType.STRING).withLabel("Type the table name (or REPLACE <n> TABLES) to confirm a REPLACE")));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The backup runs the selected records belong to, with the tables chosen for each
|
||||
** (empty means the tables of the form). Artifact records choose their own table.
|
||||
*******************************************************************************/
|
||||
static Map<Integer, Set<String>> sources(List<QRecord> records) throws QException
|
||||
{
|
||||
Map<Integer, Set<String>> sources = new LinkedHashMap<>();
|
||||
for(QRecord record : records)
|
||||
{
|
||||
boolean artifact = BackupsTables.ARTIFACT.equals(record.getTableName()) || record.getValue("artifactType") != null;
|
||||
if(artifact)
|
||||
{
|
||||
String table = record.getValueString("tableName");
|
||||
if(table == null)
|
||||
{
|
||||
throw (new QUserFacingException("Only the data of a table can be restored: choose a table data artifact."));
|
||||
}
|
||||
sources.computeIfAbsent(record.getValueInteger("backupRunId"), k -> new LinkedHashSet<>()).add(table);
|
||||
}
|
||||
else
|
||||
{
|
||||
sources.computeIfAbsent(record.getValueInteger("id"), k -> new LinkedHashSet<>());
|
||||
}
|
||||
}
|
||||
return (sources);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Review (preview, validate) is the dry run; execute only re-checks the rails.
|
||||
*******************************************************************************/
|
||||
public static class RestoreTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final List<RestoreOutcome> outcomes = new ArrayList<>();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
boolean execute = StreamedETLWithFrontendProcess.STEP_NAME_EXECUTE.equals(input.getStepName());
|
||||
RestoreEngine engine = new RestoreEngine();
|
||||
for(Map.Entry<Integer, Set<String>> source : sources(input.getRecords()).entrySet())
|
||||
{
|
||||
RestoreRequest request = RestoreRequest.fromInput(input, source.getKey(), new ArrayList<>(source.getValue()));
|
||||
if(execute)
|
||||
{
|
||||
RestoreEngine.Plan plan = engine.plan(request);
|
||||
if(!plan.errors().isEmpty())
|
||||
{
|
||||
throw (new QUserFacingException(String.join(" ", plan.errors())));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
outcomes.add(engine.review(request));
|
||||
}
|
||||
}
|
||||
output.getRecords().addAll(input.getRecords());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
for(RestoreOutcome outcome : outcomes)
|
||||
{
|
||||
lines.addAll(RestoreSummaries.lines(outcome, false));
|
||||
}
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Execute: the restore itself, one backup run at a time.
|
||||
*******************************************************************************/
|
||||
public static class RestoreLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final List<RestoreOutcome> outcomes = new ArrayList<>();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
RestoreEngine engine = new RestoreEngine();
|
||||
for(Map.Entry<Integer, Set<String>> source : sources(input.getRecords()).entrySet())
|
||||
{
|
||||
RestoreRequest request = RestoreRequest.fromInput(input, source.getKey(), new ArrayList<>(source.getValue()));
|
||||
outcomes.add(engine.execute(request));
|
||||
}
|
||||
output.getRecords().addAll(input.getRecords());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
for(RestoreOutcome outcome : outcomes)
|
||||
{
|
||||
lines.addAll(RestoreSummaries.lines(outcome, true));
|
||||
}
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,359 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.restore;
|
||||
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.DigestInputStream;
|
||||
import java.security.MessageDigest;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Function;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.kingsrook.qbits.backups.BackupsRuntime;
|
||||
import com.kingsrook.qbits.backups.codec.ArtifactTransform;
|
||||
import com.kingsrook.qbits.backups.codec.CsvRowCodec;
|
||||
import com.kingsrook.qbits.backups.codec.HashingOutputStream;
|
||||
import com.kingsrook.qbits.backups.codec.JsonlRowCodec;
|
||||
import com.kingsrook.qbits.backups.manifest.BackupManifest;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qbits.backups.storage.BackupStorage;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A backup run opened for restore: the catalog row, its manifest (checked against
|
||||
** the manifest hash in the catalog), its storage, and readers for the rows and the
|
||||
** file index. Every part is checked against the stored SHA-256 in the manifest as
|
||||
** it is read, so a damaged part fails the restore instead of restoring bad rows.
|
||||
*******************************************************************************/
|
||||
public final class BackupSource
|
||||
{
|
||||
private static final ObjectMapper MAPPER = new ObjectMapper();
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////
|
||||
// Supplies the transform that decrypts the parts of an encrypted run. The //
|
||||
// default knows no keys, so restoring an encrypted run fails with a clear //
|
||||
// message until the crypto package (QQQ core PR 1107) provider is wired in. //
|
||||
/////////////////////////////////////////////////////////////////////////////////
|
||||
private static volatile Function<BackupManifest.EncryptionInfo, ArtifactTransform> transformResolver = null;
|
||||
|
||||
private final QRecord run;
|
||||
private final BackupStorage storage;
|
||||
private final String runPrefix;
|
||||
private final BackupManifest manifest;
|
||||
private final ArtifactTransform transform;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private BackupSource(QRecord run, BackupStorage storage, String runPrefix, BackupManifest manifest, ArtifactTransform transform)
|
||||
{
|
||||
this.run = run;
|
||||
this.storage = storage;
|
||||
this.runPrefix = runPrefix;
|
||||
this.manifest = manifest;
|
||||
this.transform = transform;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Install the resolver for the decrypting transform (null removes it).
|
||||
*******************************************************************************/
|
||||
public static void setTransformResolver(Function<BackupManifest.EncryptionInfo, ArtifactTransform> resolver)
|
||||
{
|
||||
transformResolver = resolver;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Open a run for restore.
|
||||
*******************************************************************************/
|
||||
public static BackupSource open(Integer runId) throws QException
|
||||
{
|
||||
QRecord run = new BackupCatalog().get(BackupsTables.RUN, runId).orElseThrow(() -> new QException("The backup run " + runId + " was not found"));
|
||||
String status = run.getValueString("status");
|
||||
if(!"SUCCEEDED".equals(status) && !"SUCCEEDED_WITH_WARNINGS".equals(status))
|
||||
{
|
||||
throw (new QException("The backup run " + run.getValueString("runKey") + " did not complete (status " + status + "), so it cannot be restored"));
|
||||
}
|
||||
String manifestKey = run.getValueString("manifestKey");
|
||||
if(manifestKey == null)
|
||||
{
|
||||
throw (new QException("The backup run " + run.getValueString("runKey") + " has no manifest"));
|
||||
}
|
||||
|
||||
BackupStorage storage = BackupsRuntime.storageFor(run.getValueString("storageBucketAlias"));
|
||||
try
|
||||
{
|
||||
byte[] bytes = readAll(storage.openRead(manifestKey));
|
||||
String expected = run.getValueString("manifestSha256");
|
||||
if(expected != null && !expected.equals(HashingOutputStream.sha256Hex(bytes)))
|
||||
{
|
||||
throw (new QException("The manifest of backup run " + run.getValueString("runKey") + " does not match its recorded checksum"));
|
||||
}
|
||||
BackupManifest manifest = BackupManifest.read(bytes);
|
||||
String runPrefix = run.getValueString("runPrefix");
|
||||
if(runPrefix == null)
|
||||
{
|
||||
runPrefix = manifestKey.substring(0, manifestKey.length() - "manifest.json".length());
|
||||
}
|
||||
return (new BackupSource(run, storage, runPrefix, manifest, transformFor(manifest)));
|
||||
}
|
||||
catch(IOException | RuntimeException e)
|
||||
{
|
||||
throw (new QException("Could not read the manifest of backup run " + run.getValueString("runKey") + ": " + e.getMessage(), e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static ArtifactTransform transformFor(BackupManifest manifest) throws QException
|
||||
{
|
||||
BackupManifest.EncryptionInfo encryption = manifest.encryption();
|
||||
if(encryption == null || encryption.mode() == null || "NONE".equals(encryption.mode()))
|
||||
{
|
||||
return (ArtifactTransform.IDENTITY);
|
||||
}
|
||||
Function<BackupManifest.EncryptionInfo, ArtifactTransform> resolver = transformResolver;
|
||||
ArtifactTransform resolved = resolver == null ? null : resolver.apply(encryption);
|
||||
if(resolved == null)
|
||||
{
|
||||
throw (new QException("This backup is encrypted (key " + encryption.keyRef() + "), and no key provider is available to restore it"));
|
||||
}
|
||||
return (resolved);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
static byte[] readAll(InputStream in) throws IOException
|
||||
{
|
||||
try(in)
|
||||
{
|
||||
return (in.readAllBytes());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The catalog row of the run.
|
||||
*******************************************************************************/
|
||||
public QRecord getRun()
|
||||
{
|
||||
return (run);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The manifest.
|
||||
*******************************************************************************/
|
||||
public BackupManifest getManifest()
|
||||
{
|
||||
return (manifest);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The storage the backup lives in.
|
||||
*******************************************************************************/
|
||||
public BackupStorage getStorage()
|
||||
{
|
||||
return (storage);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The transform that decrypts the stored parts (identity when not encrypted).
|
||||
*******************************************************************************/
|
||||
public ArtifactTransform getTransform()
|
||||
{
|
||||
return (transform);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The prefix of the run's objects.
|
||||
*******************************************************************************/
|
||||
public String getRunPrefix()
|
||||
{
|
||||
return (runPrefix);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stream the rows of one table, decoding each with the target table's field types.
|
||||
** The sink may throw an unchecked exception to stop.
|
||||
*******************************************************************************/
|
||||
public void readRows(BackupManifest.TableEntry entry, QTableMetaData decodeTable, Consumer<QRecord> sink) throws QException
|
||||
{
|
||||
List<BackupManifest.Part> parts = new ArrayList<>();
|
||||
for(BackupManifest.Part part : entry.parts())
|
||||
{
|
||||
if("JSONL".equals(part.format()))
|
||||
{
|
||||
parts.add(part);
|
||||
}
|
||||
}
|
||||
if(parts.isEmpty())
|
||||
{
|
||||
parts.addAll(entry.parts());
|
||||
}
|
||||
|
||||
for(BackupManifest.Part part : parts)
|
||||
{
|
||||
try
|
||||
{
|
||||
readPart(part, decodeTable, sink);
|
||||
}
|
||||
catch(IOException e)
|
||||
{
|
||||
throw (new QException("Could not read " + part.key() + " of the backup: " + e.getMessage(), e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private void readPart(BackupManifest.Part part, QTableMetaData decodeTable, Consumer<QRecord> sink) throws IOException, QException
|
||||
{
|
||||
MessageDigest digest = HashingOutputStream.newSha256();
|
||||
try(DigestInputStream stored = new DigestInputStream(storage.openRead(runPrefix + part.key()), digest))
|
||||
{
|
||||
InputStream plain = transform.unwrap(stored);
|
||||
InputStream text = "GZIP".equals(part.compression()) ? new GZIPInputStream(plain) : plain;
|
||||
|
||||
if("CSV".equals(part.format()))
|
||||
{
|
||||
String all = new String(text.readAllBytes(), StandardCharsets.UTF_8);
|
||||
List<String> records = CsvRowCodec.splitRecords(all);
|
||||
if(!records.isEmpty())
|
||||
{
|
||||
CsvRowCodec codec = new CsvRowCodec(decodeTable, null);
|
||||
List<String> header = CsvRowCodec.splitCells(records.get(0));
|
||||
for(int i = 1; i < records.size(); i++)
|
||||
{
|
||||
sink.accept(codec.toRecord(CsvRowCodec.splitCells(records.get(i)), header));
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
JsonlRowCodec codec = new JsonlRowCodec(decodeTable, null);
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(text, StandardCharsets.UTF_8));
|
||||
String line;
|
||||
while((line = reader.readLine()) != null)
|
||||
{
|
||||
if(!line.isEmpty())
|
||||
{
|
||||
sink.accept(codec.decode(line));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stored.transferTo(OutputStream.nullOutputStream());
|
||||
}
|
||||
|
||||
String actual = HashingOutputStream.hex(digest.digest());
|
||||
if(part.sha256Stored() != null && !part.sha256Stored().equals(actual))
|
||||
{
|
||||
throw (new IOException("checksum mismatch (the stored part is damaged or was changed)"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The file index entries of the given tables, keyed by table, record key and field.
|
||||
*******************************************************************************/
|
||||
public Map<String, JsonNode> fileIndex(Set<String> tables) throws QException
|
||||
{
|
||||
Map<String, JsonNode> index = new HashMap<>();
|
||||
if(manifest.files() == null || manifest.files().indexKeys() == null)
|
||||
{
|
||||
return (index);
|
||||
}
|
||||
for(String indexKey : manifest.files().indexKeys())
|
||||
{
|
||||
try(InputStream raw = storage.openRead(runPrefix + indexKey);
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(new GZIPInputStream(transform.unwrap(raw)), StandardCharsets.UTF_8)))
|
||||
{
|
||||
String line;
|
||||
while((line = reader.readLine()) != null)
|
||||
{
|
||||
if(line.isEmpty())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
JsonNode node = MAPPER.readTree(new ByteArrayInputStream(line.getBytes(StandardCharsets.UTF_8)));
|
||||
if(tables.contains(node.path("table").asText()))
|
||||
{
|
||||
index.put(indexKeyOf(node.path("table").asText(), node.path("recordKey").asText(), node.path("field").asText()), node);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch(IOException e)
|
||||
{
|
||||
throw (new QException("Could not read the file index " + indexKey + ": " + e.getMessage(), e));
|
||||
}
|
||||
}
|
||||
return (index);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The lookup key of a file index entry.
|
||||
*******************************************************************************/
|
||||
public static String indexKeyOf(String table, String recordKey, String field)
|
||||
{
|
||||
return (table + "\u0001" + recordKey + "\u0001" + field);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,362 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.restore;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Serializable;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import com.kingsrook.qbits.backups.codec.HashingOutputStream;
|
||||
import com.kingsrook.qbits.backups.manifest.BackupManifest;
|
||||
import com.kingsrook.qbits.backups.manifest.TableFingerprints;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qbits.backups.storage.BackupStorage;
|
||||
import com.kingsrook.qbits.backups.storage.StoredObjectInfo;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Rebuilds the catalog from storage (spec 9.6): every manifest is
|
||||
** self-describing, so after the database with the catalog is lost, the manifests
|
||||
** under a bucket alias and prefix are listed, validated, and turned back into
|
||||
** backups_run and backups_artifact rows (trigger IMPORTED). The content
|
||||
** addressed file store is re-registered in backups_file_object so dedup keeps
|
||||
** working. Importing is idempotent: a run already in the catalog is skipped.
|
||||
**
|
||||
** Validation: the manifest must be readable and of a known version, its run key
|
||||
** must match its folder, checksums.sha256 must exist and agree with the manifest
|
||||
** for every part, and every part must exist with the recorded size. The checksum
|
||||
** of the manifest itself is computed and stored on the run, so a later restore
|
||||
** detects a changed manifest.
|
||||
*******************************************************************************/
|
||||
public class CatalogImporter
|
||||
{
|
||||
private static final Pattern PART_NUMBER = Pattern.compile("part-(\\d+)");
|
||||
private static final Pattern SHA_KEY = Pattern.compile("^[0-9a-f]{64}$");
|
||||
|
||||
private final BackupCatalog catalog = new BackupCatalog();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** What was found for one manifest.
|
||||
*******************************************************************************/
|
||||
public record Inspection(String manifestKey, String status, List<String> problems, BackupManifest manifest, String runPrefix, String manifestSha256)
|
||||
{
|
||||
public static final String NEW = "NEW";
|
||||
public static final String EXISTS = "EXISTS";
|
||||
public static final String INVALID = "INVALID";
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The keys of every manifest under a prefix.
|
||||
*******************************************************************************/
|
||||
public List<String> findManifestKeys(BackupStorage storage, String prefix)
|
||||
{
|
||||
List<String> keys = new ArrayList<>();
|
||||
for(StoredObjectInfo info : storage.list(prefix == null ? "" : prefix))
|
||||
{
|
||||
if(info.key().endsWith("/manifest.json"))
|
||||
{
|
||||
keys.add(info.key());
|
||||
}
|
||||
}
|
||||
return (keys);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read and validate one manifest, and say whether the catalog already has it.
|
||||
*******************************************************************************/
|
||||
public Inspection inspect(BackupStorage storage, String manifestKey) throws QException
|
||||
{
|
||||
String runPrefix = manifestKey.substring(0, manifestKey.length() - "manifest.json".length());
|
||||
byte[] bytes;
|
||||
BackupManifest manifest;
|
||||
try
|
||||
{
|
||||
bytes = BackupSource.readAll(storage.openRead(manifestKey));
|
||||
manifest = BackupManifest.read(bytes);
|
||||
}
|
||||
catch(IOException | RuntimeException e)
|
||||
{
|
||||
return (invalid(manifestKey, runPrefix, "The manifest cannot be read: " + e.getMessage()));
|
||||
}
|
||||
|
||||
String sha = HashingOutputStream.sha256Hex(bytes);
|
||||
List<String> problems = new ArrayList<>();
|
||||
if(manifest.manifestVersion() > BackupManifest.CURRENT_VERSION)
|
||||
{
|
||||
problems.add("The manifest is version " + manifest.manifestVersion() + ", newer than this qbit understands");
|
||||
}
|
||||
String[] segments = runPrefix.split("/");
|
||||
if(manifest.runKey() == null || segments.length == 0 || !manifest.runKey().equals(segments[segments.length - 1]))
|
||||
{
|
||||
problems.add("The manifest's run key does not match its folder");
|
||||
}
|
||||
|
||||
if(!problems.isEmpty())
|
||||
{
|
||||
return (new Inspection(manifestKey, Inspection.INVALID, problems, manifest, runPrefix, sha));
|
||||
}
|
||||
|
||||
if(!catalog.where(BackupsTables.RUN, "runKey", manifest.runKey()).isEmpty())
|
||||
{
|
||||
return (new Inspection(manifestKey, Inspection.EXISTS, List.of(), manifest, runPrefix, sha));
|
||||
}
|
||||
|
||||
checkParts(storage, manifest, runPrefix, problems);
|
||||
return (new Inspection(manifestKey, problems.isEmpty() ? Inspection.NEW : Inspection.INVALID, problems, manifest, runPrefix, sha));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private Inspection invalid(String manifestKey, String runPrefix, String problem)
|
||||
{
|
||||
return (new Inspection(manifestKey, Inspection.INVALID, List.of(problem), null, runPrefix, null));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Check checksums.sha256 against the manifest, and that each part exists.
|
||||
*******************************************************************************/
|
||||
private void checkParts(BackupStorage storage, BackupManifest manifest, String runPrefix, List<String> problems)
|
||||
{
|
||||
Map<String, String> checksums = readChecksums(storage, runPrefix);
|
||||
if(checksums == null)
|
||||
{
|
||||
problems.add("checksums.sha256 is missing or unreadable");
|
||||
return;
|
||||
}
|
||||
|
||||
for(BackupManifest.TableEntry table : manifest.tables() == null ? List.<BackupManifest.TableEntry>of() : manifest.tables())
|
||||
{
|
||||
for(BackupManifest.Part part : table.parts())
|
||||
{
|
||||
String listed = checksums.get(part.key());
|
||||
if(listed == null || !listed.equals(part.sha256Stored()))
|
||||
{
|
||||
problems.add("The checksum of " + part.key() + " does not match the manifest");
|
||||
}
|
||||
else if(storage.head(runPrefix + part.key()).map(StoredObjectInfo::size).orElse(-1L) != part.bytesStored())
|
||||
{
|
||||
problems.add("The part " + part.key() + " is missing or has the wrong size");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Create the catalog rows of one valid manifest. Returns the new run id.
|
||||
*******************************************************************************/
|
||||
public Integer importRun(String bucketAlias, BackupStorage storage, Inspection inspection) throws QException
|
||||
{
|
||||
BackupManifest manifest = inspection.manifest();
|
||||
String runPrefix = inspection.runPrefix();
|
||||
List<BackupManifest.TableEntry> tables = manifest.tables() == null ? List.of() : manifest.tables();
|
||||
|
||||
Instant started = parse(manifest.startedAt());
|
||||
Instant ended = parse(manifest.finishedAt());
|
||||
Integer planId = null;
|
||||
if(manifest.plan() != null && manifest.plan().planKey() != null)
|
||||
{
|
||||
List<QRecord> plans = catalog.where(BackupsTables.PLAN, "planKey", manifest.plan().planKey());
|
||||
planId = plans.isEmpty() ? null : plans.get(0).getValueInteger("id");
|
||||
}
|
||||
|
||||
Map<String, Serializable> row = new LinkedHashMap<>();
|
||||
row.put("backupPlanId", planId);
|
||||
row.put("runKey", manifest.runKey());
|
||||
row.put("trigger", "IMPORTED");
|
||||
row.put("requestedByUserId", manifest.requestedBy());
|
||||
row.put("status", manifest.warnings() == null || manifest.warnings().isEmpty() ? "SUCCEEDED" : "SUCCEEDED_WITH_WARNINGS");
|
||||
row.put("startDate", started);
|
||||
row.put("endDate", ended);
|
||||
row.put("durationMs", started != null && ended != null ? ended.toEpochMilli() - started.toEpochMilli() : null);
|
||||
row.put("tablesPlanned", tables.size());
|
||||
row.put("tablesDone", tables.size());
|
||||
row.put("rowsTotal", manifest.totals() == null ? null : manifest.totals().rows());
|
||||
row.put("bytesStored", manifest.totals() == null ? null : manifest.totals().bytesStored());
|
||||
row.put("consistencyAchieved", manifest.consistency() == null ? null : manifest.consistency().achieved());
|
||||
row.put("storageBucketAlias", bucketAlias);
|
||||
row.put("runPrefix", runPrefix);
|
||||
row.put("manifestKey", inspection.manifestKey());
|
||||
row.put("manifestSha256", inspection.manifestSha256());
|
||||
row.put("securityScope", manifest.securityScope());
|
||||
row.put("qqqVersion", manifest.instance() == null ? null : manifest.instance().qqqVersion());
|
||||
row.put("appVersion", manifest.instance() == null ? null : manifest.instance().appVersion());
|
||||
row.put("instanceName", manifest.instance() == null ? null : manifest.instance().name());
|
||||
row.put("schemaFingerprint", TableFingerprints.combine(tables.stream().map(BackupManifest.TableEntry::fingerprint).toList()));
|
||||
boolean encrypted = manifest.encryption() != null && manifest.encryption().mode() != null && !"NONE".equals(manifest.encryption().mode());
|
||||
row.put("encryptionKeyRef", manifest.encryption() == null ? null : manifest.encryption().keyRef());
|
||||
row.put("encryptionKeyFingerprint", manifest.encryption() == null ? null : manifest.encryption().keyFingerprint());
|
||||
row.put("verificationStatus", "UNVERIFIED");
|
||||
row.put("warningCount", manifest.warnings() == null ? 0 : manifest.warnings().size());
|
||||
if(manifest.files() != null)
|
||||
{
|
||||
row.put("filesReferenced", (int) manifest.files().objects());
|
||||
row.put("filesMissing", (int) manifest.files().missing());
|
||||
row.put("fileBytesReferenced", manifest.files().bytes());
|
||||
}
|
||||
Integer runId = catalog.insert(BackupsTables.RUN, row);
|
||||
|
||||
Map<String, String> checksums = readChecksums(storage, runPrefix);
|
||||
if(checksums == null)
|
||||
{
|
||||
checksums = new HashMap<>();
|
||||
}
|
||||
for(BackupManifest.TableEntry table : tables)
|
||||
{
|
||||
for(int i = 0; i < table.parts().size(); i++)
|
||||
{
|
||||
BackupManifest.Part part = table.parts().get(i);
|
||||
Matcher matcher = PART_NUMBER.matcher(part.key());
|
||||
int number = matcher.find() ? Integer.parseInt(matcher.group(1)) : i + 1;
|
||||
boolean last = i == table.parts().size() - 1;
|
||||
catalog.insertArtifact(runId, "TABLE_DATA", table.name(), number, part.key(), part.format(), part.compression(), part.encrypted(), part.rows(),
|
||||
part.bytesRaw(), part.bytesStored(), part.sha256Stored(), part.sha256Raw(), last ? table.rowsHash() : null, null, null, table.fingerprint());
|
||||
}
|
||||
}
|
||||
if(manifest.files() != null && manifest.files().indexKeys() != null)
|
||||
{
|
||||
int number = 1;
|
||||
for(String key : manifest.files().indexKeys())
|
||||
{
|
||||
long size = storage.head(runPrefix + key).map(StoredObjectInfo::size).orElse(0L);
|
||||
catalog.insertArtifact(runId, "FILE_OBJECTS_INDEX", null, number++, key, "JSONL", "GZIP", encrypted, 0, 0, size, checksums.get(key), null, null, null, null, null);
|
||||
}
|
||||
}
|
||||
artifactOfKey(storage, runPrefix, "checksums.sha256", "CHECKSUMS", "TEXT", runId, null);
|
||||
artifactOfKey(storage, runPrefix, "manifest.json", "MANIFEST", "JSON", runId, inspection.manifestSha256());
|
||||
|
||||
registerFileStore(bucketAlias, storage, runPrefix, runId, manifest, encrypted);
|
||||
return (runId);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private void artifactOfKey(BackupStorage storage, String runPrefix, String key, String type, String format, Integer runId, String knownSha) throws QException
|
||||
{
|
||||
long size = storage.head(runPrefix + key).map(StoredObjectInfo::size).orElse(0L);
|
||||
String sha = knownSha;
|
||||
if(sha == null)
|
||||
{
|
||||
try
|
||||
{
|
||||
sha = HashingOutputStream.sha256Hex(BackupSource.readAll(storage.openRead(runPrefix + key)));
|
||||
}
|
||||
catch(IOException | RuntimeException e)
|
||||
{
|
||||
sha = null;
|
||||
}
|
||||
}
|
||||
catalog.insertArtifact(runId, type, null, 1, key, format, "NONE", false, 0, size, size, sha, sha, null, null, null, null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private Map<String, String> readChecksums(BackupStorage storage, String runPrefix)
|
||||
{
|
||||
Map<String, String> checksums = new HashMap<>();
|
||||
try
|
||||
{
|
||||
for(String line : new String(BackupSource.readAll(storage.openRead(runPrefix + "checksums.sha256")), StandardCharsets.UTF_8).split("\n"))
|
||||
{
|
||||
String[] parts = line.split(" ", 2);
|
||||
if(parts.length == 2)
|
||||
{
|
||||
checksums.put(parts[1].trim(), parts[0].trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
catch(IOException | RuntimeException e)
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
return (checksums);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Re-register the content-addressed files of the instance, so the next backup
|
||||
** deduplicates against them again.
|
||||
*******************************************************************************/
|
||||
private void registerFileStore(String bucketAlias, BackupStorage storage, String runPrefix, Integer runId, BackupManifest manifest, boolean encrypted)
|
||||
{
|
||||
String[] segments = runPrefix.split("/");
|
||||
if(segments.length < 6)
|
||||
{
|
||||
return;
|
||||
}
|
||||
String instancePrefix = String.join("/", java.util.Arrays.copyOf(segments, segments.length - 5)) + "/";
|
||||
String fingerprint = manifest.encryption() == null ? null : manifest.encryption().keyFingerprint();
|
||||
var registry = catalog.fileStoreRegistry(bucketAlias, runId, fingerprint);
|
||||
for(StoredObjectInfo info : storage.list(instancePrefix + "_files/sha256/"))
|
||||
{
|
||||
String name = info.key().substring(info.key().lastIndexOf('/') + 1);
|
||||
int dot = name.indexOf('.');
|
||||
String sha = dot < 0 ? name : name.substring(0, dot);
|
||||
if(SHA_KEY.matcher(sha).matches() && !registry.isKnown(sha))
|
||||
{
|
||||
registry.register(sha, info.size(), null, info.key(), encrypted);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static Instant parse(String text)
|
||||
{
|
||||
try
|
||||
{
|
||||
return (text == null ? null : Instant.parse(text));
|
||||
}
|
||||
catch(RuntimeException e)
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,651 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.restore;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.backups.BackupsQBitConfig;
|
||||
import com.kingsrook.qbits.backups.BackupsRuntime;
|
||||
import com.kingsrook.qbits.backups.codec.HashingOutputStream;
|
||||
import com.kingsrook.qbits.backups.manifest.BackupManifest;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.permissions.BackupsPermissionChecker;
|
||||
import com.kingsrook.qbits.backups.permissions.BackupsPermissions;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qbits.backups.run.BackupEngine;
|
||||
import com.kingsrook.qbits.backups.run.BackupRunFactory;
|
||||
import com.kingsrook.qbits.backups.run.BackupRunRequest;
|
||||
import com.kingsrook.qbits.backups.run.BackupRunResult;
|
||||
import com.kingsrook.qbits.backups.scope.PlanSettings;
|
||||
import com.kingsrook.qbits.backups.scope.ScopeResolver;
|
||||
import com.kingsrook.qbits.backups.scope.TableRole;
|
||||
import com.kingsrook.qbits.backups.scope.TableRoles;
|
||||
import com.kingsrook.qbits.backups.storage.BackupStorage;
|
||||
import com.kingsrook.qbits.backups.storage.WriteOptions;
|
||||
import com.kingsrook.qqq.backend.core.actions.audits.AuditAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Restores from a backup run (spec 9.3 to 9.5). Two entry points share the same
|
||||
** table pass: review() reads and compares without writing anything, and
|
||||
** execute() takes the pre-restore backup, writes in restore order, resyncs key
|
||||
** sequences, brings files back (re-hashing each against the index), writes one
|
||||
** summary audit per table, and records backups_restore_run and
|
||||
** backups_restore_run_table rows.
|
||||
**
|
||||
** Safety rails (spec 9.4) are enforced here, not only in the screens: the caller
|
||||
** needs backups.restore (REPLACE also backups.admin and the typed confirmation),
|
||||
** SECRET and BACKUPS tables are never restored, EPHEMERAL tables only when named,
|
||||
** and nothing restores while a backup of the same tables is running.
|
||||
*******************************************************************************/
|
||||
public class RestoreEngine
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(RestoreEngine.class);
|
||||
|
||||
private static final long RUNNING_HEARTBEAT_MINUTES = 10;
|
||||
|
||||
private final BackupCatalog catalog = new BackupCatalog();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** What a request resolves to: the source, the tables in restore order and the
|
||||
** table each is written to, with every rail violation as an error.
|
||||
*******************************************************************************/
|
||||
public record Plan(BackupSource source, List<String> order, Map<String, String> targets, List<String> errors, List<String> warnings, List<String> notes)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve a request and check every safety rail, without touching any data.
|
||||
*******************************************************************************/
|
||||
public Plan plan(RestoreRequest request)
|
||||
{
|
||||
List<String> errors = new ArrayList<>();
|
||||
List<String> warnings = new ArrayList<>();
|
||||
List<String> notes = new ArrayList<>();
|
||||
List<String> order = new ArrayList<>();
|
||||
Map<String, String> targets = new LinkedHashMap<>();
|
||||
QInstance instance = QContext.getQInstance();
|
||||
|
||||
if(!BackupsPermissionChecker.has(BackupsPermissions.RESTORE))
|
||||
{
|
||||
errors.add("Restoring needs the " + BackupsPermissions.RESTORE + " permission.");
|
||||
}
|
||||
boolean replace = RestoreRequest.MODE_REPLACE.equals(request.getMode());
|
||||
if(!List.of(RestoreRequest.MODE_INSERT_NEW, RestoreRequest.MODE_UPSERT, RestoreRequest.MODE_REPLACE).contains(request.getMode()))
|
||||
{
|
||||
errors.add("Unknown restore mode " + request.getMode() + ".");
|
||||
}
|
||||
if(!List.of(RestoreRequest.IDS_PRESERVE, RestoreRequest.IDS_REMAP).contains(request.getIdStrategy()))
|
||||
{
|
||||
errors.add("Unknown id strategy " + request.getIdStrategy() + ".");
|
||||
}
|
||||
if(replace && !BackupsPermissionChecker.has(BackupsPermissions.ADMIN))
|
||||
{
|
||||
errors.add("A REPLACE restore needs the " + BackupsPermissions.ADMIN + " permission as well.");
|
||||
}
|
||||
if(replace && RestoreRequest.IDS_REMAP.equals(request.getIdStrategy()))
|
||||
{
|
||||
errors.add("REPLACE cannot be combined with new ids: use PRESERVE ids.");
|
||||
}
|
||||
|
||||
BackupSource source = null;
|
||||
try
|
||||
{
|
||||
source = BackupSource.open(request.getSourceRunId());
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
errors.add(e.getMessage());
|
||||
return (new Plan(null, order, targets, errors, warnings, notes));
|
||||
}
|
||||
|
||||
boolean explicit = !request.getTables().isEmpty();
|
||||
List<String> names = new ArrayList<>();
|
||||
if(explicit)
|
||||
{
|
||||
for(String name : request.getTables())
|
||||
{
|
||||
if(source.getManifest().table(name) == null)
|
||||
{
|
||||
errors.add("The table " + name + " is not in this backup.");
|
||||
}
|
||||
else
|
||||
{
|
||||
names.add(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
source.getManifest().tables().forEach(t -> names.add(t.name()));
|
||||
}
|
||||
|
||||
if(request.getTargetTable() != null && names.size() != 1)
|
||||
{
|
||||
errors.add("A target table can only be chosen when exactly one table is restored.");
|
||||
}
|
||||
|
||||
for(String name : names)
|
||||
{
|
||||
String targetName = request.getTargetTable() != null && names.size() == 1 ? request.getTargetTable() : name;
|
||||
QTableMetaData target = instance.getTable(targetName);
|
||||
if(target == null)
|
||||
{
|
||||
errors.add("The table " + targetName + " does not exist in this instance.");
|
||||
continue;
|
||||
}
|
||||
TableRole role = TableRoles.roleOf(instance, target);
|
||||
if(role == TableRole.SECRET || role == TableRole.BACKUPS)
|
||||
{
|
||||
errors.add("The table " + targetName + " is a " + role + " table, and restores never touch those.");
|
||||
continue;
|
||||
}
|
||||
if(role == TableRole.EPHEMERAL)
|
||||
{
|
||||
if(!explicit)
|
||||
{
|
||||
notes.add("The table " + targetName + " holds temporary data and is left out unless you choose it.");
|
||||
continue;
|
||||
}
|
||||
warnings.add("The table " + targetName + " holds temporary data; you chose to restore it anyway.");
|
||||
}
|
||||
targets.put(name, targetName);
|
||||
}
|
||||
|
||||
List<String> manifestOrder = source.getManifest().restoreOrder() == null ? List.of() : source.getManifest().restoreOrder();
|
||||
for(String name : manifestOrder)
|
||||
{
|
||||
if(targets.containsKey(name))
|
||||
{
|
||||
order.add(name);
|
||||
}
|
||||
}
|
||||
for(String name : targets.keySet())
|
||||
{
|
||||
if(!order.contains(name))
|
||||
{
|
||||
order.add(name);
|
||||
}
|
||||
}
|
||||
|
||||
if(order.isEmpty() && errors.isEmpty())
|
||||
{
|
||||
errors.add("There are no tables to restore.");
|
||||
}
|
||||
|
||||
if(replace && errors.isEmpty())
|
||||
{
|
||||
String expected = RestoreRequest.expectedConfirmation(order);
|
||||
if(request.getConfirmationText() == null || !expected.equals(request.getConfirmationText().trim()))
|
||||
{
|
||||
errors.add("To confirm a REPLACE, type " + expected + ".");
|
||||
}
|
||||
}
|
||||
|
||||
String running = runningBackupFor(order);
|
||||
if(running != null)
|
||||
{
|
||||
errors.add(running);
|
||||
}
|
||||
|
||||
return (new Plan(source, order, targets, errors, warnings, notes));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A message when a backup that covers any of these tables is running, else null.
|
||||
*******************************************************************************/
|
||||
private String runningBackupFor(List<String> tables)
|
||||
{
|
||||
try
|
||||
{
|
||||
Instant cutoff = BackupsRuntime.getClock().instant().minusSeconds(RUNNING_HEARTBEAT_MINUTES * 60);
|
||||
for(QRecord run : catalog.where(BackupsTables.RUN, "status", "RUNNING"))
|
||||
{
|
||||
Instant heartbeat = run.getValueInstant("heartbeatDate");
|
||||
if(heartbeat == null || !heartbeat.isAfter(cutoff))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if(coversAny(run, tables))
|
||||
{
|
||||
return ("A backup of these tables is running (" + run.getValueString("runKey") + "). Wait for it to finish, then restore.");
|
||||
}
|
||||
}
|
||||
return (null);
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
return ("Could not check for running backups: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether a running backup covers any of the tables (a run without a plan is
|
||||
** assumed to).
|
||||
*******************************************************************************/
|
||||
private boolean coversAny(QRecord run, List<String> tables) throws QException
|
||||
{
|
||||
Integer planId = run.getValueInteger("backupPlanId");
|
||||
if(planId == null)
|
||||
{
|
||||
return (true);
|
||||
}
|
||||
QRecord plan = catalog.get(BackupsTables.PLAN, planId).orElse(null);
|
||||
if(plan == null)
|
||||
{
|
||||
return (true);
|
||||
}
|
||||
Set<String> covered = new HashSet<>(BackupRunFactory.resolveScope(plan).names());
|
||||
return (tables.stream().anyMatch(covered::contains));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Review: everything the restore would do, with nothing written.
|
||||
*******************************************************************************/
|
||||
public RestoreOutcome review(RestoreRequest request) throws QException
|
||||
{
|
||||
Plan plan = plan(request);
|
||||
if(!plan.errors().isEmpty())
|
||||
{
|
||||
RestoreOutcome outcome = new RestoreOutcome();
|
||||
outcome.getErrors().addAll(plan.errors());
|
||||
return (outcome);
|
||||
}
|
||||
|
||||
RestoreOutcome outcome = new RestorePass(request, plan.source(), true, null, plan.order(), plan.targets()).run();
|
||||
outcome.getWarnings().addAll(plan.warnings());
|
||||
outcome.getNotes().addAll(plan.notes());
|
||||
if(!request.getPreRestoreBackup())
|
||||
{
|
||||
outcome.getWarnings().add("The safety backup before the restore is turned off.");
|
||||
}
|
||||
if(RestoreRequest.IDS_PRESERVE.equals(request.getIdStrategy()) && outcome.sum(t -> t.getSequenceNeeded() ? 1 : 0) > 0 && !request.getWriter().canResyncSequences())
|
||||
{
|
||||
outcome.getWarnings().add("Key sequences cannot be resynced with this version of QQQ (it needs the sequence resync hook), so on databases with generated keys the next generated id may collide. Resync them after the restore.");
|
||||
}
|
||||
if(!request.getWriter().usesCoreRestoreFlag())
|
||||
{
|
||||
outcome.getNotes().add("This version of QQQ has no restore mode for writes, so rows are written with the standard actions: record security and validation apply, and modified dates may be set to now.");
|
||||
}
|
||||
return (outcome);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Execute the restore. Throws when a rail refuses it; otherwise returns the
|
||||
** outcome (also recorded in the restore run rows).
|
||||
*******************************************************************************/
|
||||
public RestoreOutcome execute(RestoreRequest request) throws QException
|
||||
{
|
||||
Plan plan = plan(request);
|
||||
if(!plan.errors().isEmpty())
|
||||
{
|
||||
throw (new QException(String.join(" ", plan.errors())));
|
||||
}
|
||||
|
||||
BackupsQBitConfig config = BackupsRuntime.getConfig();
|
||||
Instant start = BackupsRuntime.getClock().instant();
|
||||
|
||||
Map<String, Serializable> row = new LinkedHashMap<>();
|
||||
row.put("sourceBackupRunId", request.getSourceRunId());
|
||||
row.put("requestedByUserId", request.getRequestedBy());
|
||||
row.put("startDate", start);
|
||||
row.put("status", "RUNNING");
|
||||
row.put("mode", request.getMode());
|
||||
row.put("idStrategy", request.getIdStrategy());
|
||||
row.put("matchOn", request.getMatchOn());
|
||||
row.put("targetInstanceName", config.getInstanceName() != null ? config.getInstanceName() : "qqq");
|
||||
row.put("scopeSummary", scopeSummary(request, plan));
|
||||
row.put("automationsSuppressed", request.getSuppressAutomations());
|
||||
row.put("esbSuppressed", !request.getPublishEvents());
|
||||
row.put("confirmationText", RestoreRequest.MODE_REPLACE.equals(request.getMode()) ? request.getConfirmationText() : null);
|
||||
Integer restoreRunId = catalog.insert(BackupsTables.RESTORE_RUN, row);
|
||||
|
||||
RestoreOutcome outcome = null;
|
||||
try
|
||||
{
|
||||
Integer preRestoreRunId = null;
|
||||
if(request.getPreRestoreBackup())
|
||||
{
|
||||
preRestoreRunId = preRestoreBackup(request, plan);
|
||||
catalog.update(BackupsTables.RESTORE_RUN, restoreRunId, Map.of("preRestoreBackupRunId", preRestoreRunId));
|
||||
}
|
||||
|
||||
RestorePass pass = new RestorePass(request, plan.source(), false, restoreRunId, plan.order(), plan.targets());
|
||||
outcome = pass.run();
|
||||
outcome.setRestoreRunId(restoreRunId);
|
||||
outcome.setPreRestoreRunId(preRestoreRunId);
|
||||
outcome.getWarnings().addAll(plan.warnings());
|
||||
outcome.getNotes().addAll(plan.notes());
|
||||
|
||||
resyncSequences(request, plan, outcome);
|
||||
restoreFiles(request, plan, pass, outcome, restoreRunId);
|
||||
recordResults(request, plan, outcome, restoreRunId, start);
|
||||
deleteIdMap(restoreRunId);
|
||||
return (outcome);
|
||||
}
|
||||
catch(QException | RuntimeException e)
|
||||
{
|
||||
LOG.warn("Restore failed", e, logPair("restoreRunId", restoreRunId));
|
||||
String message = e.getClass().getSimpleName() + ": " + e.getMessage();
|
||||
try
|
||||
{
|
||||
Map<String, Serializable> failed = new LinkedHashMap<>();
|
||||
failed.put("status", "FAILED");
|
||||
failed.put("endDate", BackupsRuntime.getClock().instant());
|
||||
failed.put("errorMessage", message.length() > 2000 ? message.substring(0, 2000) : message);
|
||||
catalog.update(BackupsTables.RESTORE_RUN, restoreRunId, failed);
|
||||
}
|
||||
catch(QException inner)
|
||||
{
|
||||
LOG.warn("Could not record the failed restore", inner);
|
||||
}
|
||||
throw (e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private String scopeSummary(RestoreRequest request, Plan plan)
|
||||
{
|
||||
StringBuilder sb = new StringBuilder(String.join(", ", plan.order()));
|
||||
if(!request.getFilters().isEmpty())
|
||||
{
|
||||
sb.append(" | filters on ").append(String.join(", ", request.getFilters().keySet()));
|
||||
}
|
||||
if(request.getTargetTable() != null)
|
||||
{
|
||||
sb.append(" | into ").append(request.getTargetTable());
|
||||
}
|
||||
if(!request.getPreRestoreBackup())
|
||||
{
|
||||
sb.append(" | no pre-restore backup");
|
||||
}
|
||||
return (sb.toString());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Back up the target scope (with its files) before writing anything.
|
||||
*******************************************************************************/
|
||||
private Integer preRestoreBackup(RestoreRequest request, Plan plan) throws QException
|
||||
{
|
||||
BackupSource source = plan.source();
|
||||
BackupsQBitConfig config = BackupsRuntime.getConfig();
|
||||
String alias = source.getRun().getValueString("storageBucketAlias");
|
||||
|
||||
PlanSettings settings = PlanSettings.adHoc().with("scopeType", "TABLES").with("planKey", "pre-restore");
|
||||
Map<String, Serializable> original = source.getManifest().plan() == null ? Map.of() : source.getManifest().plan().settings();
|
||||
for(String key : List.of("prefix", "format", "compression", "pageSize", "targetPartBytes", "fileObjectPolicy", "heavyFieldPolicy", "consistency"))
|
||||
{
|
||||
if(original != null && original.get(key) != null)
|
||||
{
|
||||
settings.with(key, original.get(key));
|
||||
}
|
||||
}
|
||||
boolean encrypts = source.getTransform().encrypts();
|
||||
if(encrypts)
|
||||
{
|
||||
settings.with("encryptionMode", "CLIENT_AES256_GCM").with("sensitiveFieldPolicy", "INCLUDE_ENCRYPTED_ONLY");
|
||||
}
|
||||
|
||||
List<ScopeResolver.ScopeRule> rules = new ArrayList<>();
|
||||
for(String name : plan.order())
|
||||
{
|
||||
rules.add(new ScopeResolver.ScopeRule(true, "TABLE", plan.targets().get(name), request.filterFor(name), false, null));
|
||||
}
|
||||
|
||||
BackupRunRequest backup = new BackupRunRequest()
|
||||
.withSettings(settings)
|
||||
.withScope(new ScopeResolver(QContext.getQInstance()).resolve(settings, rules))
|
||||
.withTrigger("PRE_RESTORE")
|
||||
.withRequestedBy(request.getRequestedBy())
|
||||
.withBucketAlias(alias)
|
||||
.withStorage(source.getStorage())
|
||||
.withSourceResolver(BackupsRuntime::storageFor)
|
||||
.withInstanceName(config.getInstanceName() != null ? config.getInstanceName() : "qqq")
|
||||
.withPlanName("Pre-restore backup")
|
||||
.withClock(BackupsRuntime.getClock());
|
||||
BackupManifest.EncryptionInfo encryption = source.getManifest().encryption();
|
||||
backup.withTransform(source.getTransform(), encrypts && encryption != null ? encryption.keyRef() : null, encrypts && encryption != null ? encryption.keyFingerprint() : null);
|
||||
|
||||
BackupRunResult result = new BackupEngine().execute(backup);
|
||||
if(!result.isUsable())
|
||||
{
|
||||
throw (new QException("The backup taken before the restore failed, so nothing was restored: " + result.getErrorMessage()));
|
||||
}
|
||||
return (result.getRunId());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Advance the key sequences of the tables written with their original ids.
|
||||
*******************************************************************************/
|
||||
private void resyncSequences(RestoreRequest request, Plan plan, RestoreOutcome outcome) throws QException
|
||||
{
|
||||
List<QTableMetaData> tables = new ArrayList<>();
|
||||
for(RestoreOutcome.TableOutcome table : outcome.getTables().values())
|
||||
{
|
||||
if(table.getSequenceNeeded())
|
||||
{
|
||||
tables.add(QContext.getQInstance().getTable(table.getTargetName()));
|
||||
}
|
||||
}
|
||||
if(tables.isEmpty())
|
||||
{
|
||||
return;
|
||||
}
|
||||
if(!request.getWriter().canResyncSequences())
|
||||
{
|
||||
outcome.getWarnings().add("Key sequences were not resynced: this version of QQQ has no resync hook.");
|
||||
return;
|
||||
}
|
||||
List<String> resynced = request.getWriter().resyncSequences(tables);
|
||||
outcome.setSequencesResynced(resynced.size());
|
||||
for(RestoreOutcome.TableOutcome table : outcome.getTables().values())
|
||||
{
|
||||
table.setSequenceResynced(resynced.contains(table.getTargetName()));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Copy files back from the content-addressed store, re-hashing each against the
|
||||
** index. A conflicting file (different content at the target) is only replaced
|
||||
** by a REPLACE restore.
|
||||
*******************************************************************************/
|
||||
private void restoreFiles(RestoreRequest request, Plan plan, RestorePass pass, RestoreOutcome outcome, Integer restoreRunId)
|
||||
{
|
||||
for(RestorePass.FileWork work : pass.getFileWork())
|
||||
{
|
||||
if(work.conflict() && !RestoreRequest.MODE_REPLACE.equals(request.getMode()))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
RestoreOutcome.TableOutcome table = outcome.getTables().get(work.table());
|
||||
try
|
||||
{
|
||||
copyFileBack(plan.source(), work, restoreRunId);
|
||||
table.addFilesRestored(1);
|
||||
table.addFileBytesRestored(Math.max(0, work.size()));
|
||||
}
|
||||
catch(IOException | RuntimeException e)
|
||||
{
|
||||
outcome.addFilesFailed(1);
|
||||
table.getProblems().add("The file " + work.targetKey() + " was not restored: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Write one file to a temporary key, check its SHA-256 against the index, and
|
||||
** only then put it in place.
|
||||
*******************************************************************************/
|
||||
private void copyFileBack(BackupSource source, RestorePass.FileWork work, Integer restoreRunId) throws IOException
|
||||
{
|
||||
BackupStorage target = BackupsRuntime.storageFor(work.targetAlias());
|
||||
String temp = work.targetKey() + ".qqq-restore-" + restoreRunId;
|
||||
|
||||
String observed;
|
||||
try(InputStream in = source.getTransform().unwrap(source.getStorage().openRead(work.casKey()));
|
||||
HashingOutputStream out = new HashingOutputStream(target.openWrite(temp, WriteOptions.create().withMetadata("qqq-restore-run", String.valueOf(restoreRunId)))))
|
||||
{
|
||||
in.transferTo(out);
|
||||
out.close();
|
||||
observed = out.getSha256();
|
||||
}
|
||||
|
||||
if(!work.sha256().equals(observed))
|
||||
{
|
||||
target.deletePermanent(temp);
|
||||
throw (new IOException("checksum mismatch after reading the stored copy"));
|
||||
}
|
||||
target.copy(temp, work.targetKey());
|
||||
target.deletePermanent(temp);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One summary audit per table, the per-table rows, and the totals on the run.
|
||||
*******************************************************************************/
|
||||
private void recordResults(RestoreRequest request, Plan plan, RestoreOutcome outcome, Integer restoreRunId, Instant start) throws QException
|
||||
{
|
||||
String runKey = plan.source().getRun().getValueString("runKey");
|
||||
for(RestoreOutcome.TableOutcome t : outcome.getTables().values())
|
||||
{
|
||||
Map<String, Serializable> row = new LinkedHashMap<>();
|
||||
row.put("backupRestoreRunId", restoreRunId);
|
||||
row.put("tableName", t.getName());
|
||||
row.put("targetTableName", t.getTargetName());
|
||||
row.put("rowsInserted", t.getInserted());
|
||||
row.put("rowsUpdated", t.getUpdated());
|
||||
row.put("rowsUnchanged", t.getUnchanged());
|
||||
row.put("rowsDeleted", t.getDeleted());
|
||||
row.put("rowsSkipped", t.getSkipped() + t.getConflicts());
|
||||
row.put("rowsErrored", t.getErrored());
|
||||
row.put("schemaStatus", t.getSchemaStatus());
|
||||
row.put("unmappedFields", String.join(", ", t.getUnmappedFields()));
|
||||
row.put("sequenceResynced", t.getSequenceResynced());
|
||||
row.put("status", t.getStatus());
|
||||
catalog.insert(BackupsTables.RESTORE_RUN_TABLE, row);
|
||||
|
||||
if(!"SKIPPED".equals(t.getStatus()))
|
||||
{
|
||||
summaryAudit(t, runKey, restoreRunId);
|
||||
}
|
||||
}
|
||||
|
||||
boolean warnings = outcome.sum(t -> t.getErrored()) > 0 || outcome.getFilesFailed() > 0 || !outcome.getWarnings().isEmpty()
|
||||
|| outcome.getTables().values().stream().anyMatch(t -> "SKIPPED".equals(t.getStatus()));
|
||||
|
||||
Map<String, Serializable> totals = new LinkedHashMap<>();
|
||||
totals.put("status", warnings ? "SUCCEEDED_WITH_WARNINGS" : "SUCCEEDED");
|
||||
totals.put("endDate", BackupsRuntime.getClock().instant());
|
||||
totals.put("rowsInserted", outcome.sum(RestoreOutcome.TableOutcome::getInserted));
|
||||
totals.put("rowsUpdated", outcome.sum(RestoreOutcome.TableOutcome::getUpdated));
|
||||
totals.put("rowsUnchanged", outcome.sum(RestoreOutcome.TableOutcome::getUnchanged));
|
||||
totals.put("rowsDeleted", outcome.sum(RestoreOutcome.TableOutcome::getDeleted));
|
||||
totals.put("rowsSkipped", outcome.sum(t -> t.getSkipped() + t.getConflicts()));
|
||||
totals.put("rowsErrored", outcome.sum(RestoreOutcome.TableOutcome::getErrored));
|
||||
totals.put("filesRestored", (int) outcome.sum(t -> t.getFilesRestored()));
|
||||
totals.put("filesAlreadyPresent", (int) outcome.sum(t -> t.getFilesPresent()));
|
||||
totals.put("filesConflicting", (int) outcome.sum(t -> t.getFilesConflicting()));
|
||||
totals.put("filesMissingFromBackup", (int) outcome.sum(t -> t.getFilesMissing()));
|
||||
totals.put("fileBytesRestored", outcome.sum(RestoreOutcome.TableOutcome::getFileBytesRestored));
|
||||
totals.put("sequencesResynced", outcome.getSequencesResynced());
|
||||
catalog.update(BackupsTables.RESTORE_RUN, restoreRunId, totals);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The one audit entry for a table in a restore (replaces per-record audit).
|
||||
*******************************************************************************/
|
||||
private void summaryAudit(RestoreOutcome.TableOutcome t, String runKey, Integer restoreRunId)
|
||||
{
|
||||
try
|
||||
{
|
||||
AuditAction.execute(t.getTargetName(), null, Map.of(), "Restore " + restoreRunId + " from backup " + runKey + ": "
|
||||
+ t.getInserted() + " inserted, " + t.getUpdated() + " updated, " + t.getDeleted() + " deleted, " + t.getUnchanged() + " unchanged");
|
||||
}
|
||||
catch(RuntimeException e)
|
||||
{
|
||||
LOG.warn("Could not write the restore summary audit", e, logPair("table", t.getTargetName()));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The id map rows are only needed while the restore runs.
|
||||
*******************************************************************************/
|
||||
private void deleteIdMap(Integer restoreRunId)
|
||||
{
|
||||
try
|
||||
{
|
||||
DeleteInput input = new DeleteInput(BackupsTables.ID_MAP);
|
||||
input.setQueryFilter(new QQueryFilter(new QFilterCriteria("restoreRunId", QCriteriaOperator.EQUALS, restoreRunId)));
|
||||
input.withPermissionBypass("backups restore id map");
|
||||
new DeleteAction().execute(input);
|
||||
}
|
||||
catch(QException | RuntimeException e)
|
||||
{
|
||||
LOG.warn("Could not clear the id map of a restore", e, logPair("restoreRunId", restoreRunId));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,710 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.restore;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** What a restore pass found (Review) or did (Execute): the per-table counts, the
|
||||
** schema comparison, the sample diffs and the file counts. Review and Execute
|
||||
** share one algorithm, so the Review numbers are the numbers the restore produces.
|
||||
** Nothing here ever holds a sensitive value: sample diffs are masked when built.
|
||||
*******************************************************************************/
|
||||
public class RestoreOutcome
|
||||
{
|
||||
private final Map<String, TableOutcome> tables = new LinkedHashMap<>();
|
||||
private final List<String> warnings = new ArrayList<>();
|
||||
private final List<String> errors = new ArrayList<>();
|
||||
private final List<String> notes = new ArrayList<>();
|
||||
|
||||
private Integer preRestoreRunId;
|
||||
private Integer restoreRunId;
|
||||
private int sequencesResynced;
|
||||
private int filesFailed;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One changed record, as masked "field: old -> new" texts.
|
||||
*******************************************************************************/
|
||||
public record Sample(String key, List<String> changes)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The result for one table.
|
||||
*******************************************************************************/
|
||||
public static class TableOutcome
|
||||
{
|
||||
private final String name;
|
||||
private final String targetName;
|
||||
|
||||
private String status = "PENDING";
|
||||
private String schemaStatus = "MATCH";
|
||||
private final List<String> schemaNotes = new ArrayList<>();
|
||||
private final List<String> unmappedFields = new ArrayList<>();
|
||||
private final List<String> unmappedReferences = new ArrayList<>();
|
||||
private final List<String> problems = new ArrayList<>();
|
||||
private final List<Sample> samples = new ArrayList<>();
|
||||
private long rows;
|
||||
private long inserted;
|
||||
private long updated;
|
||||
private long unchanged;
|
||||
private long skipped;
|
||||
private long deleted;
|
||||
private long conflicts;
|
||||
private long orphans;
|
||||
private long errored;
|
||||
private boolean sequenceNeeded;
|
||||
private boolean sequenceResynced;
|
||||
private long updatedSampleTotal;
|
||||
private int filesToRestore;
|
||||
private int filesPresent;
|
||||
private int filesConflicting;
|
||||
private int filesMissing;
|
||||
private int filesRestored;
|
||||
private long fileBytesRestored;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor.
|
||||
*******************************************************************************/
|
||||
public TableOutcome(String name, String targetName)
|
||||
{
|
||||
this.name = name;
|
||||
this.targetName = targetName;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for name (the table in the backup)
|
||||
*******************************************************************************/
|
||||
public String getName()
|
||||
{
|
||||
return (name);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for targetName (the table written)
|
||||
*******************************************************************************/
|
||||
public String getTargetName()
|
||||
{
|
||||
return (targetName);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for status
|
||||
*******************************************************************************/
|
||||
public String getStatus()
|
||||
{
|
||||
return (status);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for status
|
||||
*******************************************************************************/
|
||||
public void setStatus(String status)
|
||||
{
|
||||
this.status = status;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for schemaStatus
|
||||
*******************************************************************************/
|
||||
public String getSchemaStatus()
|
||||
{
|
||||
return (schemaStatus);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for schemaStatus
|
||||
*******************************************************************************/
|
||||
public void setSchemaStatus(String schemaStatus)
|
||||
{
|
||||
this.schemaStatus = schemaStatus;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for schemaNotes
|
||||
*******************************************************************************/
|
||||
public List<String> getSchemaNotes()
|
||||
{
|
||||
return (schemaNotes);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for unmappedFields (in the backup, not in the target)
|
||||
*******************************************************************************/
|
||||
public List<String> getUnmappedFields()
|
||||
{
|
||||
return (unmappedFields);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for unmappedReferences (id fields that metadata cannot trace)
|
||||
*******************************************************************************/
|
||||
public List<String> getUnmappedReferences()
|
||||
{
|
||||
return (unmappedReferences);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for problems (error messages, never row values)
|
||||
*******************************************************************************/
|
||||
public List<String> getProblems()
|
||||
{
|
||||
return (problems);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for samples
|
||||
*******************************************************************************/
|
||||
public List<Sample> getSamples()
|
||||
{
|
||||
return (samples);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for rows
|
||||
*******************************************************************************/
|
||||
public long getRows()
|
||||
{
|
||||
return (rows);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Add rows read from the backup.
|
||||
*******************************************************************************/
|
||||
public void addRows(long n)
|
||||
{
|
||||
rows += n;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for inserted
|
||||
*******************************************************************************/
|
||||
public long getInserted()
|
||||
{
|
||||
return (inserted);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Add inserted
|
||||
*******************************************************************************/
|
||||
public void addInserted(long n)
|
||||
{
|
||||
inserted += n;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for updated
|
||||
*******************************************************************************/
|
||||
public long getUpdated()
|
||||
{
|
||||
return (updated);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Add updated
|
||||
*******************************************************************************/
|
||||
public void addUpdated(long n)
|
||||
{
|
||||
updated += n;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for unchanged
|
||||
*******************************************************************************/
|
||||
public long getUnchanged()
|
||||
{
|
||||
return (unchanged);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Add unchanged
|
||||
*******************************************************************************/
|
||||
public void addUnchanged(long n)
|
||||
{
|
||||
unchanged += n;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for skipped (INSERT_NEW rows that exist with different content)
|
||||
*******************************************************************************/
|
||||
public long getSkipped()
|
||||
{
|
||||
return (skipped);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Add skipped
|
||||
*******************************************************************************/
|
||||
public void addSkipped(long n)
|
||||
{
|
||||
skipped += n;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for deleted
|
||||
*******************************************************************************/
|
||||
public long getDeleted()
|
||||
{
|
||||
return (deleted);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Add deleted
|
||||
*******************************************************************************/
|
||||
public void addDeleted(long n)
|
||||
{
|
||||
deleted += n;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for conflicts (rows that collide on a unique key)
|
||||
*******************************************************************************/
|
||||
public long getConflicts()
|
||||
{
|
||||
return (conflicts);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Add conflicts
|
||||
*******************************************************************************/
|
||||
public void addConflicts(long n)
|
||||
{
|
||||
conflicts += n;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for orphans (rows whose parent is in neither the backup nor the target)
|
||||
*******************************************************************************/
|
||||
public long getOrphans()
|
||||
{
|
||||
return (orphans);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Add orphans
|
||||
*******************************************************************************/
|
||||
public void addOrphans(long n)
|
||||
{
|
||||
orphans += n;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for errored
|
||||
*******************************************************************************/
|
||||
public long getErrored()
|
||||
{
|
||||
return (errored);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Add errored
|
||||
*******************************************************************************/
|
||||
public void addErrored(long n)
|
||||
{
|
||||
errored += n;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for sequenceNeeded
|
||||
*******************************************************************************/
|
||||
public boolean getSequenceNeeded()
|
||||
{
|
||||
return (sequenceNeeded);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for sequenceNeeded
|
||||
*******************************************************************************/
|
||||
public void setSequenceNeeded(boolean sequenceNeeded)
|
||||
{
|
||||
this.sequenceNeeded = sequenceNeeded;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for sequenceResynced
|
||||
*******************************************************************************/
|
||||
public boolean getSequenceResynced()
|
||||
{
|
||||
return (sequenceResynced);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for sequenceResynced
|
||||
*******************************************************************************/
|
||||
public void setSequenceResynced(boolean sequenceResynced)
|
||||
{
|
||||
this.sequenceResynced = sequenceResynced;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for the number of updated records seen (the sample list is capped)
|
||||
*******************************************************************************/
|
||||
public long getUpdatedSampleTotal()
|
||||
{
|
||||
return (updatedSampleTotal);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Count one more changed record (sampled or not).
|
||||
*******************************************************************************/
|
||||
public void countChangedRecord()
|
||||
{
|
||||
updatedSampleTotal++;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for filesToRestore
|
||||
*******************************************************************************/
|
||||
public int getFilesToRestore()
|
||||
{
|
||||
return (filesToRestore);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Add filesToRestore
|
||||
*******************************************************************************/
|
||||
public void addFilesToRestore(int n)
|
||||
{
|
||||
filesToRestore += n;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for filesPresent
|
||||
*******************************************************************************/
|
||||
public int getFilesPresent()
|
||||
{
|
||||
return (filesPresent);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Add filesPresent
|
||||
*******************************************************************************/
|
||||
public void addFilesPresent(int n)
|
||||
{
|
||||
filesPresent += n;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for filesConflicting
|
||||
*******************************************************************************/
|
||||
public int getFilesConflicting()
|
||||
{
|
||||
return (filesConflicting);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Add filesConflicting
|
||||
*******************************************************************************/
|
||||
public void addFilesConflicting(int n)
|
||||
{
|
||||
filesConflicting += n;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for filesMissing (missing from the backup)
|
||||
*******************************************************************************/
|
||||
public int getFilesMissing()
|
||||
{
|
||||
return (filesMissing);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Add filesMissing
|
||||
*******************************************************************************/
|
||||
public void addFilesMissing(int n)
|
||||
{
|
||||
filesMissing += n;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for filesRestored
|
||||
*******************************************************************************/
|
||||
public int getFilesRestored()
|
||||
{
|
||||
return (filesRestored);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Add filesRestored
|
||||
*******************************************************************************/
|
||||
public void addFilesRestored(int n)
|
||||
{
|
||||
filesRestored += n;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for fileBytesRestored
|
||||
*******************************************************************************/
|
||||
public long getFileBytesRestored()
|
||||
{
|
||||
return (fileBytesRestored);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Add fileBytesRestored
|
||||
*******************************************************************************/
|
||||
public void addFileBytesRestored(long n)
|
||||
{
|
||||
fileBytesRestored += n;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The outcome of one table, created on first use.
|
||||
*******************************************************************************/
|
||||
public TableOutcome table(String name, String targetName)
|
||||
{
|
||||
return (tables.computeIfAbsent(name, k -> new TableOutcome(name, targetName)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The tables, in restore order.
|
||||
*******************************************************************************/
|
||||
public Map<String, TableOutcome> getTables()
|
||||
{
|
||||
return (tables);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Run-level warnings.
|
||||
*******************************************************************************/
|
||||
public List<String> getWarnings()
|
||||
{
|
||||
return (warnings);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Run-level errors (the restore does not run when there are any).
|
||||
*******************************************************************************/
|
||||
public List<String> getErrors()
|
||||
{
|
||||
return (errors);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Run-level informational notes.
|
||||
*******************************************************************************/
|
||||
public List<String> getNotes()
|
||||
{
|
||||
return (notes);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for preRestoreRunId
|
||||
*******************************************************************************/
|
||||
public Integer getPreRestoreRunId()
|
||||
{
|
||||
return (preRestoreRunId);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for preRestoreRunId
|
||||
*******************************************************************************/
|
||||
public void setPreRestoreRunId(Integer preRestoreRunId)
|
||||
{
|
||||
this.preRestoreRunId = preRestoreRunId;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for restoreRunId
|
||||
*******************************************************************************/
|
||||
public Integer getRestoreRunId()
|
||||
{
|
||||
return (restoreRunId);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for restoreRunId
|
||||
*******************************************************************************/
|
||||
public void setRestoreRunId(Integer restoreRunId)
|
||||
{
|
||||
this.restoreRunId = restoreRunId;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for sequencesResynced
|
||||
*******************************************************************************/
|
||||
public int getSequencesResynced()
|
||||
{
|
||||
return (sequencesResynced);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for sequencesResynced
|
||||
*******************************************************************************/
|
||||
public void setSequencesResynced(int sequencesResynced)
|
||||
{
|
||||
this.sequencesResynced = sequencesResynced;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for filesFailed
|
||||
*******************************************************************************/
|
||||
public int getFilesFailed()
|
||||
{
|
||||
return (filesFailed);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Add filesFailed
|
||||
*******************************************************************************/
|
||||
public void addFilesFailed(int n)
|
||||
{
|
||||
filesFailed += n;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A total over the tables.
|
||||
*******************************************************************************/
|
||||
public long sum(java.util.function.ToLongFunction<TableOutcome> getter)
|
||||
{
|
||||
return (tables.values().stream().mapToLong(getter).sum());
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,612 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.restore;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.backups.files.FileReferenceProvider;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Everything a restore needs to know (spec 9.3): the source run, the tables and
|
||||
** per-table filters, the target, the mode, how rows are matched, what happens to
|
||||
** ids, and the options. Built from the first screen of the process.
|
||||
*******************************************************************************/
|
||||
public class RestoreRequest
|
||||
{
|
||||
public static final String MODE_INSERT_NEW = "INSERT_NEW";
|
||||
public static final String MODE_UPSERT = "UPSERT";
|
||||
public static final String MODE_REPLACE = "REPLACE";
|
||||
|
||||
public static final String IDS_PRESERVE = "PRESERVE";
|
||||
public static final String IDS_REMAP = "REMAP_NEW_IDS";
|
||||
|
||||
public static final String MATCH_PRIMARY_KEY = "PRIMARY_KEY";
|
||||
|
||||
private Integer sourceRunId;
|
||||
private List<String> tables = new ArrayList<>();
|
||||
private Map<String, String> filters = new LinkedHashMap<>();
|
||||
private String targetTable;
|
||||
private String mode = MODE_INSERT_NEW;
|
||||
private String matchOn = MATCH_PRIMARY_KEY;
|
||||
private String idStrategy = IDS_PRESERVE;
|
||||
private boolean suppressAutomations = true;
|
||||
private boolean publishEvents = false;
|
||||
private boolean runCustomizers = false;
|
||||
private boolean perRecordAudit = false;
|
||||
private boolean restoreFiles = true;
|
||||
private boolean preRestoreBackup = true;
|
||||
private String fileTargetBucketAlias;
|
||||
private String fileTargetPrefix;
|
||||
private String confirmationText;
|
||||
private String requestedBy = "system";
|
||||
private int pageSize = 1000;
|
||||
private List<FileReferenceProvider> fileProviders = new ArrayList<>(List.of(new FileReferenceProvider.FromTableMetaData()));
|
||||
private RestoreWriter writer = new StandardRestoreWriter();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build from the values of the first screen of the process. The tables value is
|
||||
** a comma separated list (blank means every table in the backup); the filters
|
||||
** value is a JSON object of table name to query filter.
|
||||
*******************************************************************************/
|
||||
public static RestoreRequest fromInput(RunBackendStepInput input, Integer runId, List<String> forcedTables) throws com.kingsrook.qqq.backend.core.exceptions.QException
|
||||
{
|
||||
RestoreRequest request = new RestoreRequest();
|
||||
request.sourceRunId = runId;
|
||||
if(forcedTables != null && !forcedTables.isEmpty())
|
||||
{
|
||||
request.tables = new ArrayList<>(forcedTables);
|
||||
}
|
||||
else
|
||||
{
|
||||
request.tables = splitList(input.getValueString("tables"));
|
||||
}
|
||||
request.filters = parseFilters(input.getValueString("tableFilters"));
|
||||
request.targetTable = blankToNull(input.getValueString("targetTable"));
|
||||
request.mode = orDefault(input.getValueString("mode"), MODE_INSERT_NEW);
|
||||
request.matchOn = orDefault(input.getValueString("matchOn"), MATCH_PRIMARY_KEY);
|
||||
request.idStrategy = orDefault(input.getValueString("idStrategy"), IDS_PRESERVE);
|
||||
request.suppressAutomations = bool(input, "suppressAutomations", true);
|
||||
request.publishEvents = bool(input, "publishEvents", false);
|
||||
request.runCustomizers = bool(input, "runCustomizers", false);
|
||||
request.perRecordAudit = bool(input, "perRecordAudit", false);
|
||||
request.restoreFiles = bool(input, "restoreFiles", true);
|
||||
request.preRestoreBackup = bool(input, "preRestoreBackup", true);
|
||||
request.fileTargetBucketAlias = blankToNull(input.getValueString("fileTargetBucketAlias"));
|
||||
request.fileTargetPrefix = blankToNull(input.getValueString("fileTargetPrefix"));
|
||||
request.confirmationText = input.getValueString("confirmationText");
|
||||
com.kingsrook.qqq.backend.core.model.session.QSession session = com.kingsrook.qqq.backend.core.context.QContext.getQSession();
|
||||
request.requestedBy = session == null || session.getUser() == null ? "system" : session.getUser().getIdReference();
|
||||
return (request);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The text the user must type to confirm a REPLACE of these tables.
|
||||
*******************************************************************************/
|
||||
public static String expectedConfirmation(List<String> tables)
|
||||
{
|
||||
return (tables.size() == 1 ? tables.get(0) : "REPLACE " + tables.size() + " TABLES");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The filter json for a table, or null.
|
||||
*******************************************************************************/
|
||||
public String filterFor(String table)
|
||||
{
|
||||
return (filters.get(table));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static List<String> splitList(String text)
|
||||
{
|
||||
List<String> rs = new ArrayList<>();
|
||||
if(StringUtils.hasContent(text))
|
||||
{
|
||||
for(String part : text.split(","))
|
||||
{
|
||||
if(!part.isBlank())
|
||||
{
|
||||
rs.add(part.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
return (rs);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static Map<String, String> parseFilters(String json) throws com.kingsrook.qqq.backend.core.exceptions.QException
|
||||
{
|
||||
Map<String, String> rs = new LinkedHashMap<>();
|
||||
if(!StringUtils.hasContent(json))
|
||||
{
|
||||
return (rs);
|
||||
}
|
||||
try
|
||||
{
|
||||
com.fasterxml.jackson.databind.JsonNode node = new com.fasterxml.jackson.databind.ObjectMapper().readTree(json);
|
||||
for(var it = node.fields(); it.hasNext(); )
|
||||
{
|
||||
var entry = it.next();
|
||||
rs.put(entry.getKey(), entry.getValue().isTextual() ? entry.getValue().asText() : entry.getValue().toString());
|
||||
}
|
||||
}
|
||||
catch(java.io.IOException e)
|
||||
{
|
||||
throw (new com.kingsrook.qqq.backend.core.exceptions.QException("The per-table filters are not valid JSON: " + e.getMessage()));
|
||||
}
|
||||
return (rs);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static boolean bool(RunBackendStepInput input, String name, boolean defaultValue)
|
||||
{
|
||||
Boolean value = input.getValueBoolean(name);
|
||||
return (value == null ? defaultValue : value);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static String orDefault(String value, String defaultValue)
|
||||
{
|
||||
return (StringUtils.hasContent(value) ? value : defaultValue);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static String blankToNull(String value)
|
||||
{
|
||||
return (StringUtils.hasContent(value) ? value : null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for sourceRunId
|
||||
*******************************************************************************/
|
||||
public Integer getSourceRunId()
|
||||
{
|
||||
return (sourceRunId);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for sourceRunId
|
||||
*******************************************************************************/
|
||||
public RestoreRequest withSourceRunId(Integer sourceRunId)
|
||||
{
|
||||
this.sourceRunId = sourceRunId;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for tables (empty means every table in the backup)
|
||||
*******************************************************************************/
|
||||
public List<String> getTables()
|
||||
{
|
||||
return (tables);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for tables
|
||||
*******************************************************************************/
|
||||
public RestoreRequest withTables(List<String> tables)
|
||||
{
|
||||
this.tables = new ArrayList<>(tables);
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for filters
|
||||
*******************************************************************************/
|
||||
public Map<String, String> getFilters()
|
||||
{
|
||||
return (filters);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for one table filter (json of a QQueryFilter)
|
||||
*******************************************************************************/
|
||||
public RestoreRequest withFilter(String table, String filterJson)
|
||||
{
|
||||
this.filters.put(table, filterJson);
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for targetTable
|
||||
*******************************************************************************/
|
||||
public String getTargetTable()
|
||||
{
|
||||
return (targetTable);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for targetTable
|
||||
*******************************************************************************/
|
||||
public RestoreRequest withTargetTable(String targetTable)
|
||||
{
|
||||
this.targetTable = targetTable;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for mode
|
||||
*******************************************************************************/
|
||||
public String getMode()
|
||||
{
|
||||
return (mode);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for mode
|
||||
*******************************************************************************/
|
||||
public RestoreRequest withMode(String mode)
|
||||
{
|
||||
this.mode = mode;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for matchOn
|
||||
*******************************************************************************/
|
||||
public String getMatchOn()
|
||||
{
|
||||
return (matchOn);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for matchOn
|
||||
*******************************************************************************/
|
||||
public RestoreRequest withMatchOn(String matchOn)
|
||||
{
|
||||
this.matchOn = matchOn;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for idStrategy
|
||||
*******************************************************************************/
|
||||
public String getIdStrategy()
|
||||
{
|
||||
return (idStrategy);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for idStrategy
|
||||
*******************************************************************************/
|
||||
public RestoreRequest withIdStrategy(String idStrategy)
|
||||
{
|
||||
this.idStrategy = idStrategy;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for suppressAutomations
|
||||
*******************************************************************************/
|
||||
public boolean getSuppressAutomations()
|
||||
{
|
||||
return (suppressAutomations);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for suppressAutomations
|
||||
*******************************************************************************/
|
||||
public RestoreRequest withSuppressAutomations(boolean suppressAutomations)
|
||||
{
|
||||
this.suppressAutomations = suppressAutomations;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for publishEvents
|
||||
*******************************************************************************/
|
||||
public boolean getPublishEvents()
|
||||
{
|
||||
return (publishEvents);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for publishEvents
|
||||
*******************************************************************************/
|
||||
public RestoreRequest withPublishEvents(boolean publishEvents)
|
||||
{
|
||||
this.publishEvents = publishEvents;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for runCustomizers
|
||||
*******************************************************************************/
|
||||
public boolean getRunCustomizers()
|
||||
{
|
||||
return (runCustomizers);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for runCustomizers
|
||||
*******************************************************************************/
|
||||
public RestoreRequest withRunCustomizers(boolean runCustomizers)
|
||||
{
|
||||
this.runCustomizers = runCustomizers;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for perRecordAudit
|
||||
*******************************************************************************/
|
||||
public boolean getPerRecordAudit()
|
||||
{
|
||||
return (perRecordAudit);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for perRecordAudit
|
||||
*******************************************************************************/
|
||||
public RestoreRequest withPerRecordAudit(boolean perRecordAudit)
|
||||
{
|
||||
this.perRecordAudit = perRecordAudit;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for restoreFiles
|
||||
*******************************************************************************/
|
||||
public boolean getRestoreFiles()
|
||||
{
|
||||
return (restoreFiles);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for restoreFiles
|
||||
*******************************************************************************/
|
||||
public RestoreRequest withRestoreFiles(boolean restoreFiles)
|
||||
{
|
||||
this.restoreFiles = restoreFiles;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for preRestoreBackup
|
||||
*******************************************************************************/
|
||||
public boolean getPreRestoreBackup()
|
||||
{
|
||||
return (preRestoreBackup);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for preRestoreBackup
|
||||
*******************************************************************************/
|
||||
public RestoreRequest withPreRestoreBackup(boolean preRestoreBackup)
|
||||
{
|
||||
this.preRestoreBackup = preRestoreBackup;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for fileTargetBucketAlias
|
||||
*******************************************************************************/
|
||||
public String getFileTargetBucketAlias()
|
||||
{
|
||||
return (fileTargetBucketAlias);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for fileTargetBucketAlias
|
||||
*******************************************************************************/
|
||||
public RestoreRequest withFileTargetBucketAlias(String fileTargetBucketAlias)
|
||||
{
|
||||
this.fileTargetBucketAlias = fileTargetBucketAlias;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for fileTargetPrefix
|
||||
*******************************************************************************/
|
||||
public String getFileTargetPrefix()
|
||||
{
|
||||
return (fileTargetPrefix);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for fileTargetPrefix
|
||||
*******************************************************************************/
|
||||
public RestoreRequest withFileTargetPrefix(String fileTargetPrefix)
|
||||
{
|
||||
this.fileTargetPrefix = fileTargetPrefix;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for confirmationText
|
||||
*******************************************************************************/
|
||||
public String getConfirmationText()
|
||||
{
|
||||
return (confirmationText);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for confirmationText
|
||||
*******************************************************************************/
|
||||
public RestoreRequest withConfirmationText(String confirmationText)
|
||||
{
|
||||
this.confirmationText = confirmationText;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for requestedBy
|
||||
*******************************************************************************/
|
||||
public String getRequestedBy()
|
||||
{
|
||||
return (requestedBy);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for requestedBy
|
||||
*******************************************************************************/
|
||||
public RestoreRequest withRequestedBy(String requestedBy)
|
||||
{
|
||||
this.requestedBy = requestedBy;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for pageSize
|
||||
*******************************************************************************/
|
||||
public int getPageSize()
|
||||
{
|
||||
return (pageSize);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for pageSize
|
||||
*******************************************************************************/
|
||||
public RestoreRequest withPageSize(int pageSize)
|
||||
{
|
||||
this.pageSize = pageSize;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for fileProviders
|
||||
*******************************************************************************/
|
||||
public List<FileReferenceProvider> getFileProviders()
|
||||
{
|
||||
return (fileProviders);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for writer
|
||||
*******************************************************************************/
|
||||
public RestoreWriter getWriter()
|
||||
{
|
||||
return (writer);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for writer (tests, or a backend specific writer)
|
||||
*******************************************************************************/
|
||||
public RestoreRequest withWriter(RestoreWriter writer)
|
||||
{
|
||||
this.writer = writer;
|
||||
return (this);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.restore;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Turns a RestoreOutcome into the standard Review and Result summary lines
|
||||
** (spec 9.3). One group of lines per table, in restore order. Every line has
|
||||
** its future wording (Review) and its past wording (Result); sample diffs are
|
||||
** bullets on one line per table, with sensitive values already masked.
|
||||
*******************************************************************************/
|
||||
public final class RestoreSummaries
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private RestoreSummaries()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A line with all four wordings.
|
||||
*******************************************************************************/
|
||||
public static ProcessSummaryLine line(Status status, long count, String singularFuture, String pluralFuture, String singularPast, String pluralPast)
|
||||
{
|
||||
ProcessSummaryLine line = new ProcessSummaryLine(status, (int) Math.min(Integer.MAX_VALUE, count), null);
|
||||
line.setSingularFutureMessage(singularFuture);
|
||||
line.setPluralFutureMessage(pluralFuture);
|
||||
line.setSingularPastMessage(singularPast);
|
||||
line.setPluralPastMessage(pluralPast);
|
||||
return (line);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A one-off message line (the same text before and after).
|
||||
*******************************************************************************/
|
||||
public static ProcessSummaryLine message(Status status, String text)
|
||||
{
|
||||
return (line(status, 1, text, text, text, text));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The lines of an outcome.
|
||||
*******************************************************************************/
|
||||
public static List<ProcessSummaryLineInterface> lines(RestoreOutcome outcome, boolean isResult)
|
||||
{
|
||||
List<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
|
||||
for(String error : outcome.getErrors())
|
||||
{
|
||||
lines.add(message(Status.ERROR, error));
|
||||
}
|
||||
|
||||
for(RestoreOutcome.TableOutcome t : outcome.getTables().values())
|
||||
{
|
||||
tableLines(lines, t, isResult);
|
||||
}
|
||||
|
||||
long sequences = outcome.sum(t -> t.getSequenceNeeded() ? 1 : 0);
|
||||
if(sequences > 0 && !isResult)
|
||||
{
|
||||
lines.add(line(Status.INFO, sequences, "table will have its key sequence resynced", "tables will have their key sequences resynced",
|
||||
"table had its key sequence resynced", "tables had their key sequences resynced"));
|
||||
}
|
||||
if(isResult && outcome.getSequencesResynced() > 0)
|
||||
{
|
||||
lines.add(line(Status.OK, outcome.getSequencesResynced(), "key sequence was resynced", "key sequences were resynced",
|
||||
"key sequence was resynced", "key sequences were resynced"));
|
||||
}
|
||||
if(isResult && outcome.getPreRestoreRunId() != null)
|
||||
{
|
||||
lines.add(message(Status.INFO, "A backup of the target tables was taken first (backup run " + outcome.getPreRestoreRunId() + "), so this restore can be undone."));
|
||||
}
|
||||
if(isResult && outcome.getFilesFailed() > 0)
|
||||
{
|
||||
lines.add(line(Status.ERROR, outcome.getFilesFailed(), "file could not be restored", "files could not be restored", "file could not be restored", "files could not be restored"));
|
||||
}
|
||||
|
||||
for(String warning : outcome.getWarnings())
|
||||
{
|
||||
lines.add(message(Status.WARNING, warning));
|
||||
}
|
||||
for(String note : outcome.getNotes())
|
||||
{
|
||||
lines.add(message(Status.INFO, note));
|
||||
}
|
||||
return (lines);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static void tableLines(List<ProcessSummaryLineInterface> lines, RestoreOutcome.TableOutcome t, boolean isResult)
|
||||
{
|
||||
String in = " " + t.getTargetName();
|
||||
|
||||
if("SKIPPED".equals(t.getStatus()))
|
||||
{
|
||||
lines.add(message(Status.ERROR, "The table " + t.getName() + " will not be restored: its schema does not fit the backup"));
|
||||
}
|
||||
for(String note : t.getSchemaNotes())
|
||||
{
|
||||
lines.add(message("MISMATCH".equals(t.getSchemaStatus()) ? Status.ERROR : Status.WARNING, "Schema of " + t.getName() + ": " + note));
|
||||
}
|
||||
for(String reference : t.getUnmappedReferences())
|
||||
{
|
||||
lines.add(message(Status.WARNING, "The field " + reference + " looks like a reference but cannot be traced, so it is not rewritten to the new ids"));
|
||||
}
|
||||
|
||||
add(lines, line(Status.OK, t.getInserted(), "record will be inserted into" + in, "records will be inserted into" + in,
|
||||
"record was inserted into" + in, "records were inserted into" + in));
|
||||
add(lines, line(Status.OK, t.getUpdated(), "record will be updated in" + in, "records will be updated in" + in,
|
||||
"record was updated in" + in, "records were updated in" + in));
|
||||
add(lines, line(Status.INFO, t.getUnchanged(), "record is unchanged in" + in, "records are unchanged in" + in,
|
||||
"record was already the same in" + in, "records were already the same in" + in));
|
||||
add(lines, line(Status.INFO, t.getSkipped(), "record exists with different content and will be left alone in" + in,
|
||||
"records exist with different content and will be left alone in" + in,
|
||||
"record existed with different content and was left alone in" + in, "records existed with different content and were left alone in" + in));
|
||||
add(lines, line(Status.WARNING, t.getDeleted(), "record will be deleted from" + in + " (REPLACE)", "records will be deleted from" + in + " (REPLACE)",
|
||||
"record was deleted from" + in + " (REPLACE)", "records were deleted from" + in + " (REPLACE)"));
|
||||
add(lines, line(Status.WARNING, t.getConflicts(), "record conflicts on a unique key in" + in + " and will be skipped",
|
||||
"records conflict on a unique key in" + in + " and will be skipped",
|
||||
"record conflicted on a unique key in" + in + " and was skipped", "records conflicted on a unique key in" + in + " and were skipped"));
|
||||
add(lines, line(Status.WARNING, t.getOrphans(), "record in" + in + " references a parent that is missing from the backup and the target",
|
||||
"records in" + in + " reference a parent that is missing from the backup and the target",
|
||||
"record in" + in + " referenced a parent that is missing from the backup and the target",
|
||||
"records in" + in + " referenced a parent that is missing from the backup and the target"));
|
||||
if(isResult)
|
||||
{
|
||||
add(lines, line(Status.ERROR, t.getErrored(), "record could not be written to" + in, "records could not be written to" + in,
|
||||
"record could not be written to" + in, "records could not be written to" + in));
|
||||
for(String problem : t.getProblems())
|
||||
{
|
||||
lines.add(message(Status.ERROR, problem));
|
||||
}
|
||||
}
|
||||
|
||||
fileLines(lines, t, isResult);
|
||||
|
||||
if(!t.getSamples().isEmpty() && !isResult)
|
||||
{
|
||||
ProcessSummaryLine sample = line(Status.INFO, t.getSamples().size(),
|
||||
"changed record in" + in + " (field: now -> from the backup; sensitive values are hidden)",
|
||||
"changed records in" + in + " (field: now -> from the backup; sensitive values are hidden)", "", "");
|
||||
ArrayList<String> bullets = new ArrayList<>();
|
||||
for(RestoreOutcome.Sample s : t.getSamples())
|
||||
{
|
||||
bullets.add("#" + s.key() + " " + String.join("; ", s.changes()));
|
||||
}
|
||||
sample.setBulletsOfText(bullets);
|
||||
lines.add(sample);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static void fileLines(List<ProcessSummaryLineInterface> lines, RestoreOutcome.TableOutcome t, boolean isResult)
|
||||
{
|
||||
String in = " for" + " " + t.getTargetName();
|
||||
if(isResult)
|
||||
{
|
||||
add(lines, line(Status.OK, t.getFilesRestored(), "file was restored" + in, "files were restored" + in, "file was restored" + in, "files were restored" + in));
|
||||
}
|
||||
else
|
||||
{
|
||||
add(lines, line(Status.OK, t.getFilesToRestore(), "file will be restored" + in, "files will be restored" + in, "file was restored" + in, "files were restored" + in));
|
||||
}
|
||||
add(lines, line(Status.INFO, t.getFilesPresent(), "file is already there with the same checksum" + in + " and is skipped",
|
||||
"files are already there with the same checksum" + in + " and are skipped",
|
||||
"file was already there with the same checksum" + in, "files were already there with the same checksum" + in));
|
||||
add(lines, line(Status.WARNING, t.getFilesConflicting(), "file exists with different content" + in + " (kept, unless the mode is REPLACE)",
|
||||
"files exist with different content" + in + " (kept, unless the mode is REPLACE)",
|
||||
"file existed with different content" + in, "files existed with different content" + in));
|
||||
add(lines, line(Status.WARNING, t.getFilesMissing(), "file is missing from the backup" + in, "files are missing from the backup" + in,
|
||||
"file was missing from the backup" + in, "files were missing from the backup" + in));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static void add(List<ProcessSummaryLineInterface> lines, ProcessSummaryLine line)
|
||||
{
|
||||
if(line.getCount() != null && line.getCount() > 0)
|
||||
{
|
||||
lines.add(line);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.restore;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** How a restore writes (spec 5.3 and 5.4). The default implementation uses the
|
||||
** plain Insert, Update and Delete actions, and selects the core RestoreWriteFlag
|
||||
** (QRun-IO/qqq#1111) and the core key-sequence resync (#1112) reflectively when the
|
||||
** running QQQ has them. A backend specific writer can be supplied on the request.
|
||||
**
|
||||
** Whatever the implementation, record security, field security, unique keys and
|
||||
** required-field validation still apply to every write.
|
||||
*******************************************************************************/
|
||||
public interface RestoreWriter
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Insert records (restore mode). Returns the output records, with any errors.
|
||||
*******************************************************************************/
|
||||
List<QRecord> insert(QTableMetaData table, List<QRecord> records, RestoreRequest options, QBackendTransaction transaction) throws QException;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Update records (restore mode). Returns the output records, with any errors.
|
||||
*******************************************************************************/
|
||||
List<QRecord> update(QTableMetaData table, List<QRecord> records, RestoreRequest options, QBackendTransaction transaction) throws QException;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Delete records by primary key (REPLACE). Returns the number deleted.
|
||||
*******************************************************************************/
|
||||
long delete(QTableMetaData table, List<Serializable> primaryKeys, RestoreRequest options, QBackendTransaction transaction) throws QException;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether the writes honor the core restore flag (so created and modified dates
|
||||
** are kept as supplied).
|
||||
*******************************************************************************/
|
||||
boolean usesCoreRestoreFlag();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether key sequences can be resynced on this QQQ (the core hook is present).
|
||||
*******************************************************************************/
|
||||
boolean canResyncSequences();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resync key sequences after explicit-id inserts. Returns the names of the tables
|
||||
** whose sequence was advanced.
|
||||
*******************************************************************************/
|
||||
List<String> resyncSequences(Collection<QTableMetaData> tables) throws QException;
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.restore;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.ActionFlag;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The default RestoreWriter: the standard Insert, Update and Delete actions.
|
||||
**
|
||||
** Stand-ins (replace with the core API when QQQ 4.1.0-RC.5 ships):
|
||||
** <ul>
|
||||
** <li>RestoreWriteFlag (QRun-IO/qqq#1111) is created reflectively when the class is
|
||||
** present. Without it, the DML audit is omitted (a summary audit is written per
|
||||
** table instead), but created and modified dates may be set by the normal defaults.</li>
|
||||
** <li>resyncKeySequences (QRun-IO/qqq#1112) is called reflectively on the backend
|
||||
** module when it exists; otherwise it is a no-op that the Review reports.</li>
|
||||
** </ul>
|
||||
*******************************************************************************/
|
||||
public class StandardRestoreWriter implements RestoreWriter
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(StandardRestoreWriter.class);
|
||||
|
||||
private static final String FLAG_CLASS = "com.kingsrook.qqq.backend.core.model.actions.tables.RestoreWriteFlag";
|
||||
|
||||
private static final Class<?> FLAG = lookup(FLAG_CLASS);
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static Class<?> lookup(String name)
|
||||
{
|
||||
try
|
||||
{
|
||||
return (Class.forName(name));
|
||||
}
|
||||
catch(ClassNotFoundException | LinkageError e)
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> insert(QTableMetaData table, List<QRecord> records, RestoreRequest options, QBackendTransaction transaction) throws QException
|
||||
{
|
||||
InsertInput input = new InsertInput(table.getName());
|
||||
input.setRecords(records);
|
||||
input.setOmitDmlAudit(!options.getPerRecordAudit());
|
||||
if(transaction != null)
|
||||
{
|
||||
input.setTransaction(transaction);
|
||||
}
|
||||
addFlag(options, input::withFlag);
|
||||
return (new InsertAction().execute(input).getRecords());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> update(QTableMetaData table, List<QRecord> records, RestoreRequest options, QBackendTransaction transaction) throws QException
|
||||
{
|
||||
UpdateInput input = new UpdateInput(table.getName());
|
||||
input.setRecords(records);
|
||||
input.setOmitDmlAudit(!options.getPerRecordAudit());
|
||||
if(transaction != null)
|
||||
{
|
||||
input.setTransaction(transaction);
|
||||
}
|
||||
addFlag(options, input::withFlag);
|
||||
return (new UpdateAction().execute(input).getRecords());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public long delete(QTableMetaData table, List<Serializable> primaryKeys, RestoreRequest options, QBackendTransaction transaction) throws QException
|
||||
{
|
||||
DeleteInput input = new DeleteInput(table.getName());
|
||||
input.setPrimaryKeys(primaryKeys);
|
||||
input.setOmitDmlAudit(!options.getPerRecordAudit());
|
||||
if(transaction != null)
|
||||
{
|
||||
input.setTransaction(transaction);
|
||||
}
|
||||
addFlag(options, input::withFlag);
|
||||
DeleteOutput output = new DeleteAction().execute(input);
|
||||
if(output.getRecordsWithErrors() != null && !output.getRecordsWithErrors().isEmpty())
|
||||
{
|
||||
QRecord first = output.getRecordsWithErrors().get(0);
|
||||
throw (new QException("Could not delete " + output.getRecordsWithErrors().size() + " records of " + table.getName() + ": "
|
||||
+ (first.getErrors().isEmpty() ? "unknown error" : first.getErrors().get(0).getMessage())));
|
||||
}
|
||||
return (output.getDeletedRecordCount());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Create the core flag (when the running QQQ has it) and hand it to the input.
|
||||
*******************************************************************************/
|
||||
private void addFlag(RestoreRequest options, java.util.function.Consumer<ActionFlag> sink) throws QException
|
||||
{
|
||||
if(FLAG == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
try
|
||||
{
|
||||
Object flag = FLAG.getDeclaredConstructor().newInstance();
|
||||
FLAG.getMethod("withSuppressAutomations", boolean.class).invoke(flag, options.getSuppressAutomations());
|
||||
FLAG.getMethod("withRunCustomizers", boolean.class).invoke(flag, options.getRunCustomizers());
|
||||
FLAG.getMethod("withPerRecordAudit", boolean.class).invoke(flag, options.getPerRecordAudit());
|
||||
FLAG.getMethod("withPublishEvents", boolean.class).invoke(flag, options.getPublishEvents());
|
||||
sink.accept((ActionFlag) flag);
|
||||
}
|
||||
catch(ReflectiveOperationException | ClassCastException e)
|
||||
{
|
||||
throw (new QException("Could not use the core restore flag: " + e.getMessage(), e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean usesCoreRestoreFlag()
|
||||
{
|
||||
return (FLAG != null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean canResyncSequences()
|
||||
{
|
||||
return (resyncMethod() != null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static Method resyncMethod()
|
||||
{
|
||||
try
|
||||
{
|
||||
return (QBackendModuleInterface.class.getMethod("resyncKeySequences", QBackendMetaData.class, Collection.class));
|
||||
}
|
||||
catch(NoSuchMethodException | LinkageError e)
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<String> resyncSequences(Collection<QTableMetaData> tables) throws QException
|
||||
{
|
||||
Method method = resyncMethod();
|
||||
if(method == null)
|
||||
{
|
||||
return (List.of());
|
||||
}
|
||||
|
||||
Map<String, List<QTableMetaData>> byBackend = new LinkedHashMap<>();
|
||||
for(QTableMetaData table : tables)
|
||||
{
|
||||
byBackend.computeIfAbsent(table.getBackendName(), k -> new ArrayList<>()).add(table);
|
||||
}
|
||||
|
||||
List<String> resynced = new ArrayList<>();
|
||||
for(Map.Entry<String, List<QTableMetaData>> entry : byBackend.entrySet())
|
||||
{
|
||||
QBackendMetaData backend = QContext.getQInstance().getBackend(entry.getKey());
|
||||
try
|
||||
{
|
||||
QBackendModuleInterface module = new QBackendModuleDispatcher().getQBackendModule(backend);
|
||||
for(Object result : (List<?>) method.invoke(module, backend, entry.getValue()))
|
||||
{
|
||||
Object status = result.getClass().getMethod("getStatus").invoke(result);
|
||||
if("RESYNCED".equals(String.valueOf(status)))
|
||||
{
|
||||
resynced.add(String.valueOf(result.getClass().getMethod("getTableName").invoke(result)));
|
||||
}
|
||||
}
|
||||
}
|
||||
catch(ReflectiveOperationException e)
|
||||
{
|
||||
LOG.warn("Could not resync key sequences", e, logPair("backend", entry.getKey()));
|
||||
throw (new QException("Could not resync key sequences on backend " + entry.getKey() + ": " + e.getMessage(), e));
|
||||
}
|
||||
}
|
||||
return (resynced);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.restore;
|
||||
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.backups.BackupsTestUtils;
|
||||
import com.kingsrook.qbits.backups.codec.HashingOutputStream;
|
||||
import com.kingsrook.qbits.backups.manifest.BackupManifest;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qbits.backups.run.BackupRunResult;
|
||||
import com.kingsrook.qbits.backups.storage.StoredObjectInfo;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Import backups from storage (spec 9.6): the database with the catalog is gone,
|
||||
** a fresh instance imports the catalog from the manifests in storage and restores
|
||||
** from it, files included.
|
||||
*******************************************************************************/
|
||||
class CatalogImportTest
|
||||
{
|
||||
private String originalPostsHash;
|
||||
private BackupManifest originalManifest;
|
||||
private Map<String, byte[]> snapshot;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Back up an instance, keep what is in the backup bucket, then start over: a fresh
|
||||
** instance (empty tables, empty catalog, empty media) that only has the bucket.
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp() throws Exception
|
||||
{
|
||||
BackupsTestUtils.init();
|
||||
BackupsTestUtils.loadBlog(30);
|
||||
BackupRunResult run = RestoreTestSupport.backup(Map.of());
|
||||
originalPostsHash = RestoreTestSupport.contentHash("blogPost");
|
||||
originalManifest = RestoreTestSupport.manifest(run);
|
||||
|
||||
snapshot = new LinkedHashMap<>();
|
||||
for(StoredObjectInfo info : BackupsTestUtils.BACKUPS.list(""))
|
||||
{
|
||||
snapshot.put(info.key(), BackupsTestUtils.BACKUPS.getBytes(info.key()));
|
||||
}
|
||||
|
||||
BackupsTestUtils.init();
|
||||
snapshot.forEach(BackupsTestUtils.BACKUPS::putBytes);
|
||||
assertThat(new BackupCatalog().query(BackupsTables.RUN, new QQueryFilter())).as("a fresh catalog").isEmpty();
|
||||
assertThat(RestoreTestSupport.rows("blogPost")).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The manifests are found, validated and imported (trigger IMPORTED), the content
|
||||
** addressed files are registered again, and importing twice changes nothing.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testImportRebuildsTheCatalog() throws Exception
|
||||
{
|
||||
CatalogImporter importer = new CatalogImporter();
|
||||
List<String> keys = importer.findManifestKeys(BackupsTestUtils.BACKUPS, "qqq-backups/");
|
||||
assertThat(keys).hasSize(1);
|
||||
|
||||
CatalogImporter.Inspection inspection = importer.inspect(BackupsTestUtils.BACKUPS, keys.get(0));
|
||||
assertThat(inspection.status()).isEqualTo(CatalogImporter.Inspection.NEW);
|
||||
assertThat(inspection.manifestSha256()).isEqualTo(HashingOutputStream.sha256Hex(BackupsTestUtils.BACKUPS.getBytes(keys.get(0))));
|
||||
|
||||
Integer runId = importer.importRun("backups", BackupsTestUtils.BACKUPS, inspection);
|
||||
QRecord run = new BackupCatalog().get(BackupsTables.RUN, runId).orElseThrow();
|
||||
assertThat(run.getValueString("trigger")).isEqualTo("IMPORTED");
|
||||
assertThat(run.getValueString("runKey")).isEqualTo(originalManifest.runKey());
|
||||
assertThat(run.getValueString("status")).isEqualTo("SUCCEEDED_WITH_WARNINGS");
|
||||
assertThat(run.getValueString("manifestKey")).isEqualTo(keys.get(0));
|
||||
assertThat(run.getValueString("manifestSha256")).isEqualTo(inspection.manifestSha256());
|
||||
assertThat(run.getValueString("storageBucketAlias")).isEqualTo("backups");
|
||||
assertThat(run.getValueString("instanceName")).isEqualTo("testapp");
|
||||
assertThat(run.getValueLong("rowsTotal")).isEqualTo(originalManifest.totals().rows());
|
||||
assertThat(run.getValueInteger("tablesPlanned")).isEqualTo(originalManifest.tables().size());
|
||||
assertThat(run.getValueString("verificationStatus")).isEqualTo("UNVERIFIED");
|
||||
|
||||
List<QRecord> artifacts = new BackupCatalog().where(BackupsTables.ARTIFACT, "backupRunId", runId);
|
||||
long parts = originalManifest.tables().stream().mapToLong(t -> t.parts().size()).sum();
|
||||
assertThat(artifacts).hasSize((int) parts + 3);
|
||||
assertThat(artifacts).extracting(a -> a.getValueString("artifactType")).contains("MANIFEST", "CHECKSUMS", "FILE_OBJECTS_INDEX", "TABLE_DATA");
|
||||
QRecord posts = artifacts.stream().filter(a -> "blogPost".equals(a.getValueString("tableName"))).findFirst().orElseThrow();
|
||||
assertThat(posts.getValueString("sha256Stored")).isEqualTo(originalManifest.table("blogPost").parts().get(0).sha256Stored());
|
||||
assertThat(posts.getValueString("rowsHash")).isEqualTo(originalManifest.table("blogPost").rowsHash());
|
||||
|
||||
assertThat(new BackupCatalog().query(BackupsTables.FILE_OBJECT, new QQueryFilter())).as("the file store is registered again").hasSize(3);
|
||||
|
||||
assertThat(importer.inspect(BackupsTestUtils.BACKUPS, keys.get(0)).status()).isEqualTo(CatalogImporter.Inspection.EXISTS);
|
||||
assertThat(new BackupCatalog().query(BackupsTables.RUN, new QQueryFilter())).hasSize(1);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The point of the catalog import: restore from the imported run into the fresh
|
||||
** instance and get the original rows (same hash) and the files (same checksum).
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRestoreFromTheImportedCatalog() throws Exception
|
||||
{
|
||||
CatalogImporter importer = new CatalogImporter();
|
||||
CatalogImporter.Inspection inspection = importer.inspect(BackupsTestUtils.BACKUPS, importer.findManifestKeys(BackupsTestUtils.BACKUPS, "qqq-backups/").get(0));
|
||||
Integer runId = importer.importRun("backups", BackupsTestUtils.BACKUPS, inspection);
|
||||
|
||||
RestoreRequest request = new RestoreRequest().withSourceRunId(runId).withMode(RestoreRequest.MODE_INSERT_NEW).withRequestedBy("tester");
|
||||
RestoreOutcome review = new RestoreEngine().review(request);
|
||||
assertThat(review.getErrors()).isEmpty();
|
||||
assertThat(review.getTables().get("author").getInserted()).isEqualTo(3);
|
||||
assertThat(review.getTables().get("blogPost").getInserted()).isEqualTo(30);
|
||||
assertThat(review.getTables().get("blogPost").getFilesToRestore()).isEqualTo(3);
|
||||
|
||||
RestoreOutcome outcome = new RestoreEngine().execute(request);
|
||||
assertThat(outcome.getTables().get("blogPost").getInserted()).isEqualTo(30);
|
||||
assertThat(outcome.getTables().get("blogPost").getFilesRestored()).isEqualTo(3);
|
||||
assertThat(outcome.getFilesFailed()).isZero();
|
||||
|
||||
assertThat(RestoreTestSupport.rows("author")).hasSize(3);
|
||||
assertThat(RestoreTestSupport.contentHash("blogPost")).isEqualTo(originalPostsHash);
|
||||
for(int i = 0; i < 3; i++)
|
||||
{
|
||||
assertThat(new String(BackupsTestUtils.MEDIA.getBytes("media/photo-" + i + ".jpg"))).isEqualTo("photo bytes " + i);
|
||||
}
|
||||
assertThat(new BackupCatalog().where(BackupsTables.RESTORE_RUN, "sourceBackupRunId", runId)).hasSize(1);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A manifest whose parts are damaged is reported, not imported.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testDamagedBackupsAreNotImported() throws Exception
|
||||
{
|
||||
CatalogImporter importer = new CatalogImporter();
|
||||
String key = importer.findManifestKeys(BackupsTestUtils.BACKUPS, "qqq-backups/").get(0);
|
||||
String partKey = key.substring(0, key.length() - "manifest.json".length()) + originalManifest.table("blogPost").parts().get(0).key();
|
||||
BackupsTestUtils.BACKUPS.corrupt(partKey, new byte[] { 1, 2, 3 });
|
||||
|
||||
CatalogImporter.Inspection inspection = importer.inspect(BackupsTestUtils.BACKUPS, key);
|
||||
assertThat(inspection.status()).isEqualTo(CatalogImporter.Inspection.INVALID);
|
||||
assertThat(inspection.problems()).anyMatch(p -> p.contains("wrong size"));
|
||||
assertThat(new BackupCatalog().query(BackupsTables.RUN, new QQueryFilter())).isEmpty();
|
||||
|
||||
BackupsTestUtils.BACKUPS.putBytes(key, "not a manifest".getBytes());
|
||||
assertThat(importer.inspect(BackupsTestUtils.BACKUPS, key).problems()).anyMatch(p -> p.contains("cannot be read"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Restore refuses a part that does not match its stored checksum.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRestoreRefusesADamagedPart() throws Exception
|
||||
{
|
||||
CatalogImporter importer = new CatalogImporter();
|
||||
String key = importer.findManifestKeys(BackupsTestUtils.BACKUPS, "qqq-backups/").get(0);
|
||||
Integer runId = importer.importRun("backups", BackupsTestUtils.BACKUPS, importer.inspect(BackupsTestUtils.BACKUPS, key));
|
||||
|
||||
String partKey = key.substring(0, key.length() - "manifest.json".length()) + originalManifest.table("author").parts().get(0).key();
|
||||
byte[] same = BackupsTestUtils.BACKUPS.getBytes(partKey).clone();
|
||||
same[same.length / 2] ^= 0x55;
|
||||
BackupsTestUtils.BACKUPS.corrupt(partKey, same);
|
||||
|
||||
RestoreRequest request = new RestoreRequest().withSourceRunId(runId).withTables(List.of("author")).withMode(RestoreRequest.MODE_INSERT_NEW).withPreRestoreBackup(false);
|
||||
assertThatThrownBy(() -> new RestoreEngine().execute(request)).hasMessageContaining("Could not read");
|
||||
assertThat(RestoreTestSupport.rows("author")).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The import process: Review lists what will be added, Result says what was.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testImportProcess() throws Exception
|
||||
{
|
||||
RunProcessInput input = new RunProcessInput();
|
||||
input.setProcessName("backupImportCatalog");
|
||||
input.addValue("bucketAlias", "backups");
|
||||
input.addValue("prefix", "qqq-backups/");
|
||||
|
||||
RunProcessOutput output = new RunProcessAction().execute(input);
|
||||
assertThat(output.getException()).isEmpty();
|
||||
assertThat(output.getProcessState().getNextStepName()).isPresent().get().isEqualTo("input");
|
||||
input.addValue("processUUID", output.getProcessUUID());
|
||||
|
||||
input.setStartAfterStep("input");
|
||||
output = new RunProcessAction().execute(input);
|
||||
assertThat(output.getException()).isEmpty();
|
||||
List<ProcessSummaryLineInterface> review = RestoreProcessTest.lines(output, StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY);
|
||||
assertThat(RestoreTestSupport.line(review, false, "backup will be added to the catalog").getCount()).isEqualTo(1);
|
||||
assertThat(new BackupCatalog().query(BackupsTables.RUN, new QQueryFilter())).as("Review writes nothing").isEmpty();
|
||||
|
||||
input.setStartAfterStep("review");
|
||||
output = new RunProcessAction().execute(input);
|
||||
assertThat(output.getException()).isEmpty();
|
||||
List<ProcessSummaryLineInterface> result = RestoreProcessTest.lines(output, StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY);
|
||||
ProcessSummaryLine imported = RestoreTestSupport.line(result, true, "backup was added to the catalog");
|
||||
assertThat(imported.getCount()).isEqualTo(1);
|
||||
assertThat(imported.getPrimaryKeys()).hasSize(1);
|
||||
assertThat(new BackupCatalog().query(BackupsTables.RUN, new QQueryFilter())).hasSize(1);
|
||||
|
||||
RunProcessInput again = new RunProcessInput();
|
||||
again.setProcessName("backupImportCatalog");
|
||||
again.addValue("bucketAlias", "backups");
|
||||
again.addValue("prefix", "qqq-backups/");
|
||||
output = new RunProcessAction().execute(again);
|
||||
again.addValue("processUUID", output.getProcessUUID());
|
||||
again.setStartAfterStep("input");
|
||||
output = new RunProcessAction().execute(again);
|
||||
List<ProcessSummaryLineInterface> second = RestoreProcessTest.lines(output, StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY);
|
||||
assertThat(RestoreTestSupport.line(second, false, "already in the catalog").getCount()).isEqualTo(1);
|
||||
assertThat(RestoreTestSupport.line(second, false, "will be added")).isNull();
|
||||
assertThat(second.stream().map(l -> ((ProcessSummaryLine) l).getStatus())).doesNotContain(Status.ERROR);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,515 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.restore;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.backups.BackupsTestUtils;
|
||||
import com.kingsrook.qbits.backups.codec.HashingOutputStream;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qbits.backups.run.BackupRunResult;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** End to end tests of restore (spec 13): back up 30 posts, then delete 10, edit
|
||||
** 10 and add 10; Review each mode and check the summary lines; restore and check
|
||||
** counts, hashes and files.
|
||||
*******************************************************************************/
|
||||
class RestoreModesTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp() throws Exception
|
||||
{
|
||||
BackupsTestUtils.init();
|
||||
BackupsTestUtils.loadBlog(30);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Review of each mode reports the counts the restore will produce, in the standard
|
||||
** summary lines, and writes nothing.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testReviewEachMode() throws Exception
|
||||
{
|
||||
BackupRunResult run = RestoreTestSupport.backup(Map.of());
|
||||
RestoreTestSupport.mutate();
|
||||
assertThat(RestoreTestSupport.rows("blogPost")).hasSize(30);
|
||||
|
||||
RestoreOutcome insertNew = new RestoreEngine().review(RestoreTestSupport.request(run, RestoreRequest.MODE_INSERT_NEW, "blogPost"));
|
||||
RestoreOutcome.TableOutcome t = insertNew.getTables().get("blogPost");
|
||||
assertThat(insertNew.getErrors()).isEmpty();
|
||||
assertThat(t.getRows()).isEqualTo(30);
|
||||
assertThat(t.getInserted()).isEqualTo(10);
|
||||
assertThat(t.getUpdated()).isZero();
|
||||
assertThat(t.getUnchanged()).isEqualTo(10);
|
||||
assertThat(t.getSkipped()).isEqualTo(10);
|
||||
assertThat(t.getDeleted()).isZero();
|
||||
assertThat(t.getSchemaStatus()).isEqualTo("MATCH");
|
||||
List<ProcessSummaryLineInterface> lines = RestoreSummaries.lines(insertNew, false);
|
||||
assertThat(RestoreTestSupport.line(lines, false, "records will be inserted into blogPost").getCount()).isEqualTo(10);
|
||||
assertThat(RestoreTestSupport.line(lines, false, "records are unchanged in blogPost").getCount()).isEqualTo(10);
|
||||
assertThat(RestoreTestSupport.line(lines, false, "left alone in blogPost").getCount()).isEqualTo(10);
|
||||
assertThat(RestoreTestSupport.line(lines, false, "key sequence")).isNotNull();
|
||||
|
||||
RestoreOutcome upsert = new RestoreEngine().review(RestoreTestSupport.request(run, RestoreRequest.MODE_UPSERT, "blogPost"));
|
||||
t = upsert.getTables().get("blogPost");
|
||||
assertThat(t.getInserted()).isEqualTo(10);
|
||||
assertThat(t.getUpdated()).isEqualTo(10);
|
||||
assertThat(t.getUnchanged()).isEqualTo(10);
|
||||
assertThat(t.getSkipped()).isZero();
|
||||
assertThat(t.getDeleted()).isZero();
|
||||
lines = RestoreSummaries.lines(upsert, false);
|
||||
assertThat(RestoreTestSupport.line(lines, false, "records will be updated in blogPost").getCount()).isEqualTo(10);
|
||||
assertThat(t.getSamples()).hasSize(10);
|
||||
assertThat(t.getSamples().get(0).changes()).contains("title: Edited 11 -> Post 11");
|
||||
|
||||
RestoreOutcome replace = new RestoreEngine().review(RestoreTestSupport.request(run, RestoreRequest.MODE_REPLACE, "blogPost").withConfirmationText("blogPost"));
|
||||
assertThat(replace.getErrors()).isEmpty();
|
||||
t = replace.getTables().get("blogPost");
|
||||
assertThat(t.getDeleted()).isEqualTo(10);
|
||||
assertThat(t.getInserted()).isEqualTo(10);
|
||||
assertThat(t.getUpdated()).isEqualTo(10);
|
||||
assertThat(t.getUnchanged()).isEqualTo(10);
|
||||
ProcessSummaryLine deleted = RestoreTestSupport.line(RestoreSummaries.lines(replace, false), false, "will be deleted from blogPost (REPLACE)");
|
||||
assertThat(deleted.getCount()).isEqualTo(10);
|
||||
|
||||
assertThat(RestoreTestSupport.rows("blogPost")).as("Review writes nothing").hasSize(30);
|
||||
assertThat(new BackupCatalog().query(BackupsTables.RESTORE_RUN, new com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter())).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** INSERT_NEW brings back the deleted rows and leaves edits and additions alone.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testExecuteInsertNew() throws Exception
|
||||
{
|
||||
BackupRunResult run = RestoreTestSupport.backup(Map.of());
|
||||
RestoreTestSupport.mutate();
|
||||
|
||||
RestoreOutcome outcome = new RestoreEngine().execute(RestoreTestSupport.request(run, RestoreRequest.MODE_INSERT_NEW, "blogPost"));
|
||||
assertThat(outcome.getTables().get("blogPost").getInserted()).isEqualTo(10);
|
||||
|
||||
List<QRecord> rows = RestoreTestSupport.rows("blogPost");
|
||||
assertThat(rows).hasSize(40);
|
||||
assertThat(rows.get(0).getValueString("title")).isEqualTo("Post 1");
|
||||
assertThat(rows.stream().filter(r -> r.getValueInteger("id") == 15).findFirst().orElseThrow().getValueString("title")).isEqualTo("Edited 15");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** UPSERT rolls edits back and restores deleted rows, keeps additions; the file that
|
||||
** was removed comes back with the right checksum, and a file with different content
|
||||
** is kept (a conflict).
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testExecuteUpsertWithFiles() throws Exception
|
||||
{
|
||||
BackupRunResult run = RestoreTestSupport.backup(Map.of());
|
||||
RestoreTestSupport.mutate();
|
||||
String original1 = "photo bytes 1";
|
||||
BackupsTestUtils.MEDIA.removeIgnoringLock("media/photo-1.jpg");
|
||||
BackupsTestUtils.MEDIA.putBytes("media/photo-2.jpg", "tampered".getBytes());
|
||||
|
||||
RestoreOutcome review = new RestoreEngine().review(RestoreTestSupport.request(run, RestoreRequest.MODE_UPSERT, "blogPost"));
|
||||
RestoreOutcome.TableOutcome rt = review.getTables().get("blogPost");
|
||||
assertThat(rt.getFilesToRestore()).isEqualTo(1);
|
||||
assertThat(rt.getFilesPresent()).isEqualTo(1);
|
||||
assertThat(rt.getFilesConflicting()).isEqualTo(1);
|
||||
assertThat(rt.getFilesMissing()).isZero();
|
||||
assertThat(RestoreTestSupport.line(RestoreSummaries.lines(review, false), false, "different content for blogPost")).isNotNull();
|
||||
|
||||
RestoreOutcome outcome = new RestoreEngine().execute(RestoreTestSupport.request(run, RestoreRequest.MODE_UPSERT, "blogPost"));
|
||||
RestoreOutcome.TableOutcome t = outcome.getTables().get("blogPost");
|
||||
assertThat(t.getInserted()).isEqualTo(10);
|
||||
assertThat(t.getUpdated()).isEqualTo(10);
|
||||
assertThat(t.getFilesRestored()).isEqualTo(1);
|
||||
assertThat(outcome.getFilesFailed()).isZero();
|
||||
|
||||
List<QRecord> rows = RestoreTestSupport.rows("blogPost");
|
||||
assertThat(rows).hasSize(40);
|
||||
assertThat(rows.stream().filter(r -> r.getValueInteger("id") == 15).findFirst().orElseThrow().getValueString("title")).isEqualTo("Post 15");
|
||||
|
||||
byte[] restored = BackupsTestUtils.MEDIA.getBytes("media/photo-1.jpg");
|
||||
assertThat(new String(restored)).isEqualTo(original1);
|
||||
assertThat(HashingOutputStream.sha256Hex(restored)).isEqualTo(HashingOutputStream.sha256Hex(original1.getBytes()));
|
||||
assertThat(new String(BackupsTestUtils.MEDIA.getBytes("media/photo-2.jpg"))).as("a conflicting file is kept").isEqualTo("tampered");
|
||||
assertThat(BackupsTestUtils.MEDIA.list("media/").stream().filter(o -> o.key().contains("qqq-restore"))).as("no temporary object is left").isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** REPLACE needs the typed confirmation, deletes what is not in the backup, takes a
|
||||
** pre-restore backup first, brings the table back to exactly the backed up rows
|
||||
** (same hash), overwrites a conflicting file, and records the run rows.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testExecuteReplace() throws Exception
|
||||
{
|
||||
BackupRunResult run = RestoreTestSupport.backup(Map.of());
|
||||
String originalHash = RestoreTestSupport.contentHash("blogPost");
|
||||
RestoreTestSupport.mutate();
|
||||
assertThat(RestoreTestSupport.contentHash("blogPost")).isNotEqualTo(originalHash);
|
||||
BackupsTestUtils.MEDIA.putBytes("media/photo-2.jpg", "tampered".getBytes());
|
||||
|
||||
RestoreRequest request = RestoreTestSupport.request(run, RestoreRequest.MODE_REPLACE, "blogPost");
|
||||
assertThatThrownBy(() -> new RestoreEngine().execute(request)).hasMessageContaining("type blogPost");
|
||||
assertThat(RestoreTestSupport.rows("blogPost")).hasSize(30);
|
||||
|
||||
RestoreOutcome outcome = new RestoreEngine().execute(request.withConfirmationText("blogPost"));
|
||||
RestoreOutcome.TableOutcome t = outcome.getTables().get("blogPost");
|
||||
assertThat(t.getDeleted()).isEqualTo(10);
|
||||
assertThat(t.getInserted()).isEqualTo(10);
|
||||
assertThat(t.getUpdated()).isEqualTo(10);
|
||||
|
||||
assertThat(RestoreTestSupport.rows("blogPost")).hasSize(30);
|
||||
assertThat(RestoreTestSupport.contentHash("blogPost")).as("the rows are exactly the backed up rows again").isEqualTo(originalHash);
|
||||
assertThat(new String(BackupsTestUtils.MEDIA.getBytes("media/photo-2.jpg"))).as("REPLACE overwrites a conflicting file").isEqualTo("photo bytes 2");
|
||||
|
||||
QRecord restoreRun = new BackupCatalog().get(BackupsTables.RESTORE_RUN, outcome.getRestoreRunId()).orElseThrow();
|
||||
assertThat(restoreRun.getValueString("status")).isIn("SUCCEEDED", "SUCCEEDED_WITH_WARNINGS");
|
||||
assertThat(restoreRun.getValueString("mode")).isEqualTo("REPLACE");
|
||||
assertThat(restoreRun.getValueLong("rowsDeleted")).isEqualTo(10L);
|
||||
assertThat(restoreRun.getValueLong("rowsInserted")).isEqualTo(10L);
|
||||
assertThat(restoreRun.getValueInteger("filesRestored")).isEqualTo(1);
|
||||
assertThat(restoreRun.getValueString("confirmationText")).isEqualTo("blogPost");
|
||||
assertThat(restoreRun.getValueInteger("preRestoreBackupRunId")).isNotNull();
|
||||
QRecord pre = new BackupCatalog().get(BackupsTables.RUN, restoreRun.getValueInteger("preRestoreBackupRunId")).orElseThrow();
|
||||
assertThat(pre.getValueString("trigger")).isEqualTo("PRE_RESTORE");
|
||||
assertThat(pre.getValueString("status")).startsWith("SUCCEEDED");
|
||||
List<QRecord> tableRows = new BackupCatalog().where(BackupsTables.RESTORE_RUN_TABLE, "backupRestoreRunId", outcome.getRestoreRunId());
|
||||
assertThat(tableRows).hasSize(1);
|
||||
assertThat(tableRows.get(0).getValueString("tableName")).isEqualTo("blogPost");
|
||||
assertThat(tableRows.get(0).getValueLong("rowsDeleted")).isEqualTo(10L);
|
||||
assertThat(tableRows.get(0).getValueString("schemaStatus")).isEqualTo("MATCH");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Turning the pre-restore backup off is allowed and is recorded on the restore run.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testPreRestoreBackupCanBeTurnedOff() throws Exception
|
||||
{
|
||||
BackupRunResult run = RestoreTestSupport.backup(Map.of());
|
||||
RestoreTestSupport.mutate();
|
||||
RestoreOutcome outcome = new RestoreEngine().execute(RestoreTestSupport.request(run, RestoreRequest.MODE_INSERT_NEW, "blogPost").withPreRestoreBackup(false));
|
||||
QRecord restoreRun = new BackupCatalog().get(BackupsTables.RESTORE_RUN, outcome.getRestoreRunId()).orElseThrow();
|
||||
assertThat(restoreRun.getValueInteger("preRestoreBackupRunId")).isNull();
|
||||
assertThat(restoreRun.getValueString("scopeSummary")).contains("no pre-restore backup");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** REMAP_NEW_IDS lets the target assign ids and rewrites the references of later
|
||||
** tables through the map; the id map is cleared when the restore ends.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRemapNewIds() throws Exception
|
||||
{
|
||||
BackupRunResult run = RestoreTestSupport.backup(Map.of());
|
||||
|
||||
RestoreRequest request = RestoreTestSupport.request(run, RestoreRequest.MODE_INSERT_NEW, "author", "blogPost").withIdStrategy(RestoreRequest.IDS_REMAP);
|
||||
RestoreOutcome review = new RestoreEngine().review(request);
|
||||
assertThat(review.getTables().get("author").getInserted()).isEqualTo(3);
|
||||
assertThat(review.getTables().get("blogPost").getInserted()).isEqualTo(30);
|
||||
assertThat(review.getTables().get("blogPost").getOrphans()).as("the parents are in the backup").isZero();
|
||||
|
||||
RestoreOutcome outcome = new RestoreEngine().execute(request);
|
||||
assertThat(outcome.getTables().get("author").getInserted()).isEqualTo(3);
|
||||
|
||||
List<QRecord> authors = RestoreTestSupport.rows("author");
|
||||
assertThat(authors).hasSize(6);
|
||||
List<QRecord> posts = RestoreTestSupport.rows("blogPost");
|
||||
assertThat(posts).hasSize(60);
|
||||
QRecord newPost1 = posts.stream().filter(p -> p.getValueInteger("id") > 30 && "Post 1".equals(p.getValueString("title"))).findFirst().orElseThrow();
|
||||
QRecord newAuthor = authors.stream().filter(a -> a.getValueInteger("id") == newPost1.getValueInteger("authorId")).findFirst().orElseThrow();
|
||||
assertThat(newAuthor.getValueInteger("id")).as("the post points at a new author").isGreaterThan(3);
|
||||
assertThat(newAuthor.getValueString("name")).as("old author 2 was Grace").isEqualTo("Grace");
|
||||
assertThat(new BackupCatalog().query(BackupsTables.ID_MAP, new com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter())).isEmpty();
|
||||
assertThat(new BackupCatalog().get(BackupsTables.RESTORE_RUN, outcome.getRestoreRunId()).orElseThrow().getValueString("idStrategy")).isEqualTo("REMAP_NEW_IDS");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** REMAP with a unique key reuses rows that already exist (the target ids differ),
|
||||
** and the later tables point at the existing parents.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRemapMatchingOnAUniqueKey() throws Exception
|
||||
{
|
||||
RestoreTestSupport.uniqueKey("author", "name");
|
||||
BackupRunResult run = RestoreTestSupport.backup(Map.of());
|
||||
|
||||
com.kingsrook.qqq.backend.core.actions.tables.DeleteAction delete = new com.kingsrook.qqq.backend.core.actions.tables.DeleteAction();
|
||||
for(String table : List.of("blogPost", "author"))
|
||||
{
|
||||
com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput input = new com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput(table);
|
||||
List<Serializable> ids = new ArrayList<>();
|
||||
for(QRecord row : RestoreTestSupport.rows(table))
|
||||
{
|
||||
ids.add(row.getValue("id"));
|
||||
}
|
||||
input.setPrimaryKeys(ids);
|
||||
delete.execute(input);
|
||||
}
|
||||
BackupsTestUtils.insert("author", List.of(Map.of("id", 10, "name", "Linus"), Map.of("id", 20, "name", "Ada", "bio", "other bio"), Map.of("id", 30, "name", "Grace", "bio", "")));
|
||||
|
||||
RestoreRequest request = RestoreTestSupport.request(run, RestoreRequest.MODE_UPSERT, "author", "blogPost").withIdStrategy(RestoreRequest.IDS_REMAP).withMatchOn("name");
|
||||
RestoreOutcome review = new RestoreEngine().review(request);
|
||||
assertThat(review.getTables().get("author").getInserted()).isZero();
|
||||
assertThat(review.getTables().get("author").getUpdated()).as("only Ada's bio differs").isEqualTo(1);
|
||||
assertThat(review.getTables().get("author").getUnchanged()).isEqualTo(2);
|
||||
|
||||
new RestoreEngine().execute(request);
|
||||
assertThat(RestoreTestSupport.rows("author")).hasSize(3);
|
||||
assertThat(RestoreTestSupport.rows("author").stream().filter(a -> a.getValueInteger("id") == 20).findFirst().orElseThrow().getValueString("bio"))
|
||||
.startsWith("Line one");
|
||||
List<QRecord> posts = RestoreTestSupport.rows("blogPost");
|
||||
assertThat(posts).hasSize(30);
|
||||
QRecord post1 = posts.stream().filter(p -> "Post 1".equals(p.getValueString("title"))).findFirst().orElseThrow();
|
||||
assertThat(post1.getValueInteger("authorId")).as("Post 1 belonged to Grace, who is now id 30").isEqualTo(30);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unique-key conflicts and orphaned references are counted in Review.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testConflictsAndOrphans() throws Exception
|
||||
{
|
||||
RestoreTestSupport.uniqueKey("author", "name");
|
||||
BackupRunResult run = RestoreTestSupport.backup(Map.of());
|
||||
|
||||
com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput input = new com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput("author");
|
||||
input.setPrimaryKeys(List.of(1, 2));
|
||||
new com.kingsrook.qqq.backend.core.actions.tables.DeleteAction().execute(input);
|
||||
BackupsTestUtils.insert("author", List.of(Map.of("id", 5, "name", "Ada")));
|
||||
|
||||
RestoreOutcome authors = new RestoreEngine().review(RestoreTestSupport.request(run, RestoreRequest.MODE_INSERT_NEW, "author"));
|
||||
RestoreOutcome.TableOutcome a = authors.getTables().get("author");
|
||||
assertThat(a.getConflicts()).as("Ada id 1 collides with the existing Ada id 5").isEqualTo(1);
|
||||
assertThat(a.getInserted()).as("Grace id 2 can come back").isEqualTo(1);
|
||||
assertThat(a.getUnchanged()).isEqualTo(1);
|
||||
assertThat(RestoreTestSupport.line(RestoreSummaries.lines(authors, false), false, "conflicts on a unique key")).isNotNull();
|
||||
|
||||
RestoreOutcome posts = new RestoreEngine().review(RestoreTestSupport.request(run, RestoreRequest.MODE_INSERT_NEW, "blogPost"));
|
||||
assertThat(posts.getTables().get("blogPost").getOrphans()).as("posts of author 1 and 2: those ids are in neither the backup selection nor the target").isEqualTo(20);
|
||||
assertThat(RestoreTestSupport.line(RestoreSummaries.lines(posts, false), false, "reference a parent that is missing")).isNotNull();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The schema is compared with the backup: extra and removed fields are compatible
|
||||
** (the removed ones are skipped); a changed type blocks that table.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testSchemaComparison() throws Exception
|
||||
{
|
||||
BackupRunResult run = RestoreTestSupport.backup(Map.of());
|
||||
RestoreTestSupport.mutate();
|
||||
|
||||
com.kingsrook.qqq.backend.core.context.QContext.getQInstance().getTable("author").getFields().remove("bio");
|
||||
com.kingsrook.qqq.backend.core.context.QContext.getQInstance().getTable("blogPost").getField("price").setType(QFieldType.STRING);
|
||||
|
||||
RestoreOutcome review = new RestoreEngine().review(RestoreTestSupport.request(run, RestoreRequest.MODE_UPSERT, "author", "blogPost"));
|
||||
RestoreOutcome.TableOutcome author = review.getTables().get("author");
|
||||
assertThat(author.getSchemaStatus()).isEqualTo("COMPATIBLE");
|
||||
assertThat(author.getUnmappedFields()).containsExactly("bio");
|
||||
assertThat(RestoreTestSupport.line(RestoreSummaries.lines(review, false), false, "field bio is in the backup but not in the target and will be skipped")).isNotNull();
|
||||
|
||||
RestoreOutcome.TableOutcome posts = review.getTables().get("blogPost");
|
||||
assertThat(posts.getSchemaStatus()).isEqualTo("MISMATCH");
|
||||
assertThat(posts.getStatus()).isEqualTo("SKIPPED");
|
||||
assertThat(posts.getInserted()).isZero();
|
||||
assertThat(RestoreTestSupport.line(RestoreSummaries.lines(review, false), false, "changed type from DECIMAL to STRING").getStatus())
|
||||
.isEqualTo(com.kingsrook.qqq.backend.core.model.actions.processes.Status.ERROR);
|
||||
|
||||
RestoreOutcome outcome = new RestoreEngine().execute(RestoreTestSupport.request(run, RestoreRequest.MODE_UPSERT, "author", "blogPost"));
|
||||
assertThat(outcome.getTables().get("blogPost").getStatus()).isEqualTo("SKIPPED");
|
||||
assertThat(RestoreTestSupport.rows("blogPost")).as("the incompatible table is left alone").hasSize(30);
|
||||
List<QRecord> tableRows = new BackupCatalog().where(BackupsTables.RESTORE_RUN_TABLE, "backupRestoreRunId", outcome.getRestoreRunId());
|
||||
assertThat(tableRows).extracting(r -> r.getValueString("schemaStatus")).containsExactlyInAnyOrder("COMPATIBLE", "MISMATCH");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Sensitive values are masked in the sample diff, even when they are in the backup.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testSensitiveValuesAreNeverShown() throws Exception
|
||||
{
|
||||
BackupRunResult run = RestoreTestSupport.backup(Map.of("sensitiveFieldPolicy", "INCLUDE"));
|
||||
UpdateInput update = new UpdateInput("blogPost");
|
||||
update.setRecords(List.of(new QRecord().withValue("id", 1).withValue("secretNote", "brand-new-secret")));
|
||||
new UpdateAction().execute(update);
|
||||
|
||||
RestoreOutcome review = new RestoreEngine().review(RestoreTestSupport.request(run, RestoreRequest.MODE_UPSERT, "blogPost"));
|
||||
RestoreOutcome.TableOutcome t = review.getTables().get("blogPost");
|
||||
assertThat(t.getUpdated()).isEqualTo(1);
|
||||
assertThat(t.getSamples()).hasSize(1);
|
||||
assertThat(t.getSamples().get(0).changes()).containsExactly("secretNote: ******** -> ********");
|
||||
|
||||
String everything = RestoreSummaries.lines(review, false).stream().map(l -> String.valueOf(((ProcessSummaryLine) l).getBulletsOfText())).reduce("", String::concat);
|
||||
assertThat(everything).doesNotContain("brand-new-secret").doesNotContain("do-not-leak");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Per table filters restrict the restore to selected records.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testFilteredRestore() throws Exception
|
||||
{
|
||||
BackupRunResult run = RestoreTestSupport.backup(Map.of());
|
||||
RestoreTestSupport.mutate();
|
||||
|
||||
RestoreRequest request = RestoreTestSupport.request(run, RestoreRequest.MODE_UPSERT, "blogPost")
|
||||
.withFilter("blogPost", "{\"criteria\":[{\"fieldName\":\"id\",\"operator\":\"IN\",\"values\":[2,3,12]}]}");
|
||||
RestoreOutcome outcome = new RestoreEngine().execute(request);
|
||||
RestoreOutcome.TableOutcome t = outcome.getTables().get("blogPost");
|
||||
assertThat(t.getRows()).isEqualTo(3);
|
||||
assertThat(t.getInserted()).isEqualTo(2);
|
||||
assertThat(t.getUpdated()).isEqualTo(1);
|
||||
assertThat(RestoreTestSupport.rows("blogPost")).hasSize(32);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Writes go through the RestoreWriter in page transactions.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testWritesUseThePluggableWriterInPageTransactions() throws Exception
|
||||
{
|
||||
BackupRunResult run = RestoreTestSupport.backup(Map.of());
|
||||
RestoreTestSupport.mutate();
|
||||
|
||||
List<String> calls = new ArrayList<>();
|
||||
StandardRestoreWriter standard = new StandardRestoreWriter();
|
||||
RestoreWriter recording = new RestoreWriter()
|
||||
{
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> insert(com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData table, List<QRecord> records, RestoreRequest options, QBackendTransaction tx)
|
||||
throws com.kingsrook.qqq.backend.core.exceptions.QException
|
||||
{
|
||||
calls.add("insert:" + records.size() + ":" + (tx != null));
|
||||
return (standard.insert(table, records, options, tx));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> update(com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData table, List<QRecord> records, RestoreRequest options, QBackendTransaction tx)
|
||||
throws com.kingsrook.qqq.backend.core.exceptions.QException
|
||||
{
|
||||
calls.add("update:" + records.size() + ":" + (tx != null));
|
||||
return (standard.update(table, records, options, tx));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public long delete(com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData table, List<Serializable> primaryKeys, RestoreRequest options, QBackendTransaction tx)
|
||||
throws com.kingsrook.qqq.backend.core.exceptions.QException
|
||||
{
|
||||
return (standard.delete(table, primaryKeys, options, tx));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean usesCoreRestoreFlag()
|
||||
{
|
||||
return (standard.usesCoreRestoreFlag());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean canResyncSequences()
|
||||
{
|
||||
return (true);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<String> resyncSequences(java.util.Collection<com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData> tables)
|
||||
throws com.kingsrook.qqq.backend.core.exceptions.QException
|
||||
{
|
||||
tables.forEach(t -> calls.add("resync:" + t.getName()));
|
||||
return (tables.stream().map(com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData::getName).toList());
|
||||
}
|
||||
};
|
||||
|
||||
RestoreOutcome outcome = new RestoreEngine().execute(RestoreTestSupport.request(run, RestoreRequest.MODE_UPSERT, "blogPost").withWriter(recording).withPageSize(8));
|
||||
assertThat(calls).contains("insert:8:true", "insert:2:true", "update:6:true", "update:4:true");
|
||||
assertThat(calls).filteredOn(c -> !c.startsWith("resync")).allMatch(c -> c.endsWith(":true"));
|
||||
assertThat(calls).as("the key sequence of the table that got its original ids is resynced, after the writes").endsWith("resync:blogPost");
|
||||
assertThat(outcome.getSequencesResynced()).isEqualTo(1);
|
||||
assertThat(new BackupCatalog().get(BackupsTables.RESTORE_RUN, outcome.getRestoreRunId()).orElseThrow().getValueInteger("sequencesResynced")).isEqualTo(1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.restore;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.backups.BackupsTestUtils;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qbits.backups.run.BackupRunResult;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The restore and download processes, run the way the frontend runs them: the
|
||||
** first screen, Review (the validation summary), Execute and Result.
|
||||
*******************************************************************************/
|
||||
class RestoreProcessTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp() throws Exception
|
||||
{
|
||||
BackupsTestUtils.init();
|
||||
BackupsTestUtils.loadBlog(30);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@SuppressWarnings("unchecked")
|
||||
static List<ProcessSummaryLineInterface> lines(RunProcessOutput output, String field)
|
||||
{
|
||||
return (new ArrayList<>((List<ProcessSummaryLineInterface>) output.getValues().get(field)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Restore... on a backup run: Review shows the per-table lines and a sample diff,
|
||||
** Execute restores, Result shows what was done.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRestoreProcessEndToEnd() throws Exception
|
||||
{
|
||||
BackupRunResult run = RestoreTestSupport.backup(Map.of());
|
||||
RestoreTestSupport.mutate();
|
||||
|
||||
RunProcessInput input = new RunProcessInput();
|
||||
input.setProcessName("backupRestore");
|
||||
input.addValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, run.getRunId())));
|
||||
input.addValue("tables", "blogPost");
|
||||
input.addValue("mode", "UPSERT");
|
||||
|
||||
RunProcessOutput output = new RunProcessAction().execute(input);
|
||||
assertThat(output.getException()).isEmpty();
|
||||
assertThat(output.getProcessState().getNextStepName()).as("the first screen asks for the options").isPresent().get().isEqualTo("input");
|
||||
input.addValue("processUUID", output.getProcessUUID());
|
||||
|
||||
input.setStartAfterStep("input");
|
||||
output = new RunProcessAction().execute(input);
|
||||
assertThat(output.getException()).isEmpty();
|
||||
assertThat(output.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review");
|
||||
|
||||
List<ProcessSummaryLineInterface> review = lines(output, StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY);
|
||||
assertThat(RestoreTestSupport.line(review, false, "records will be inserted into blogPost").getCount()).isEqualTo(10);
|
||||
assertThat(RestoreTestSupport.line(review, false, "records will be updated in blogPost").getCount()).isEqualTo(10);
|
||||
assertThat(RestoreTestSupport.line(review, false, "records are unchanged in blogPost").getCount()).isEqualTo(10);
|
||||
ProcessSummaryLine sample = RestoreTestSupport.line(review, false, "changed records in blogPost");
|
||||
assertThat(sample.getBulletsOfText()).hasSize(10);
|
||||
assertThat(sample.getBulletsOfText().get(0)).contains("title: Edited 11 -> Post 11");
|
||||
assertThat(RestoreTestSupport.rows("blogPost")).as("Review writes nothing").hasSize(30);
|
||||
|
||||
input.setStartAfterStep("review");
|
||||
output = new RunProcessAction().execute(input);
|
||||
assertThat(output.getException()).isEmpty();
|
||||
assertThat(output.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result");
|
||||
|
||||
List<ProcessSummaryLineInterface> result = lines(output, StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY);
|
||||
assertThat(RestoreTestSupport.line(result, true, "records were inserted into blogPost").getCount()).isEqualTo(10);
|
||||
assertThat(RestoreTestSupport.line(result, true, "records were updated in blogPost").getCount()).isEqualTo(10);
|
||||
assertThat(RestoreTestSupport.line(result, true, "A backup of the target tables was taken first")).isNotNull();
|
||||
assertThat(result.stream().map(l -> ((ProcessSummaryLine) l).getStatus())).doesNotContain(Status.ERROR);
|
||||
|
||||
assertThat(RestoreTestSupport.rows("blogPost")).hasSize(40);
|
||||
assertThat(new BackupCatalog().where(BackupsTables.RESTORE_RUN, "sourceBackupRunId", run.getRunId())).hasSize(1);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Restore this table, on a table's data artifact, restores just that table.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRestoreThisTableOnAnArtifact() throws Exception
|
||||
{
|
||||
BackupRunResult run = RestoreTestSupport.backup(Map.of());
|
||||
RestoreTestSupport.mutate();
|
||||
QRecord artifact = new BackupCatalog().where(BackupsTables.ARTIFACT, "backupRunId", run.getRunId()).stream()
|
||||
.filter(a -> "blogPost".equals(a.getValueString("tableName"))).findFirst().orElseThrow();
|
||||
|
||||
RunProcessInput input = new RunProcessInput();
|
||||
input.setProcessName("backupRestoreTable");
|
||||
input.addValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, artifact.getValueInteger("id"))));
|
||||
input.addValue("mode", "INSERT_NEW");
|
||||
|
||||
RunProcessOutput output = new RunProcessAction().execute(input);
|
||||
input.addValue("processUUID", output.getProcessUUID());
|
||||
input.setStartAfterStep("input");
|
||||
output = new RunProcessAction().execute(input);
|
||||
assertThat(output.getException()).isEmpty();
|
||||
assertThat(RestoreTestSupport.line(lines(output, StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY), false, "records will be inserted into blogPost").getCount()).isEqualTo(10);
|
||||
assertThat(RestoreTestSupport.line(lines(output, StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY), false, "author")).as("only that table").isNull();
|
||||
|
||||
input.setStartAfterStep("review");
|
||||
new RunProcessAction().execute(input);
|
||||
assertThat(RestoreTestSupport.rows("blogPost")).hasSize(40);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The Review of a REPLACE without the typed confirmation is an error line, and
|
||||
** Execute refuses.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testReplaceWithoutConfirmationIsRefused() throws Exception
|
||||
{
|
||||
BackupRunResult run = RestoreTestSupport.backup(Map.of());
|
||||
RestoreTestSupport.mutate();
|
||||
|
||||
RunProcessInput input = new RunProcessInput();
|
||||
input.setProcessName("backupRestore");
|
||||
input.addValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, run.getRunId())));
|
||||
input.addValue("tables", "blogPost");
|
||||
input.addValue("mode", "REPLACE");
|
||||
|
||||
RunProcessOutput output = new RunProcessAction().execute(input);
|
||||
input.addValue("processUUID", output.getProcessUUID());
|
||||
input.setStartAfterStep("input");
|
||||
output = new RunProcessAction().execute(input);
|
||||
ProcessSummaryLine error = RestoreTestSupport.line(lines(output, StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY), false, "To confirm a REPLACE, type blogPost");
|
||||
assertThat(error).isNotNull();
|
||||
assertThat(error.getStatus()).isEqualTo(Status.ERROR);
|
||||
|
||||
input.setStartAfterStep("review");
|
||||
assertThatThrownBy(() -> new RunProcessAction().execute(input)).hasMessageContaining("To confirm a REPLACE, type blogPost");
|
||||
assertThat(RestoreTestSupport.rows("blogPost")).hasSize(30);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Download: presigned links for the run's objects (ciphertext as stored), each one
|
||||
** a separate link, with the manifest and checksums among them.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testDownloadIssuesLinks() throws Exception
|
||||
{
|
||||
BackupRunResult run = RestoreTestSupport.backup(Map.of());
|
||||
int artifacts = new BackupCatalog().where(BackupsTables.ARTIFACT, "backupRunId", run.getRunId()).size();
|
||||
|
||||
RunProcessInput input = new RunProcessInput();
|
||||
input.setProcessName("backupDownload");
|
||||
input.addValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, run.getRunId())));
|
||||
|
||||
RunProcessOutput output = new RunProcessAction().execute(input);
|
||||
assertThat(output.getException()).isEmpty();
|
||||
input.addValue("processUUID", output.getProcessUUID());
|
||||
if(output.getProcessState().getNextStepName().orElse("").equals("review"))
|
||||
{
|
||||
input.setStartAfterStep("review");
|
||||
}
|
||||
else
|
||||
{
|
||||
input.setStartAfterStep(output.getProcessState().getNextStepName().orElse(null));
|
||||
}
|
||||
|
||||
List<ProcessSummaryLineInterface> review = lines(output, StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY);
|
||||
assertThat(RestoreTestSupport.line(review, false, "files will get download links").getCount()).isEqualTo(artifacts);
|
||||
|
||||
output = new RunProcessAction().execute(input);
|
||||
assertThat(output.getException()).isEmpty();
|
||||
ProcessSummaryLine issued = RestoreTestSupport.line(lines(output, StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY), true, "download links were issued");
|
||||
assertThat(issued.getCount()).isEqualTo(artifacts);
|
||||
assertThat(issued.getBulletsOfText()).hasSize(artifacts);
|
||||
assertThat(issued.getBulletsOfText()).anyMatch(b -> b.contains("manifest.json") && b.contains("memory://") && b.contains("ttl=900"));
|
||||
assertThat(issued.getBulletsOfText()).anyMatch(b -> b.contains("checksums.sha256"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Download on one artifact links that artifact plus the manifest and checksums.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testDownloadOfOneArtifact() throws Exception
|
||||
{
|
||||
BackupRunResult run = RestoreTestSupport.backup(Map.of());
|
||||
QRecord artifact = new BackupCatalog().where(BackupsTables.ARTIFACT, "backupRunId", run.getRunId()).stream()
|
||||
.filter(a -> "blogPost".equals(a.getValueString("tableName"))).findFirst().orElseThrow();
|
||||
|
||||
RunProcessInput input = new RunProcessInput();
|
||||
input.setProcessName("backupDownloadArtifact");
|
||||
input.addValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, artifact.getValueInteger("id"))));
|
||||
RunProcessOutput output = new RunProcessAction().execute(input);
|
||||
input.addValue("processUUID", output.getProcessUUID());
|
||||
input.setStartAfterStep("review");
|
||||
output = new RunProcessAction().execute(input);
|
||||
|
||||
ProcessSummaryLine issued = RestoreTestSupport.line(lines(output, StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY), true, "download links were issued");
|
||||
assertThat(issued.getCount()).isEqualTo(3);
|
||||
assertThat(issued.getBulletsOfText()).anyMatch(b -> b.contains("blogPost-part-00001"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.restore;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.backups.BackupsRuntime;
|
||||
import com.kingsrook.qbits.backups.BackupsTestUtils;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qbits.backups.run.BackupRunResult;
|
||||
import com.kingsrook.qbits.backups.scope.BackupTableMetaData;
|
||||
import com.kingsrook.qbits.backups.scope.TableRole;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The safety rails of restore (spec 9.4) and who may run it (spec 11).
|
||||
*******************************************************************************/
|
||||
class RestoreSafetyTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp() throws Exception
|
||||
{
|
||||
BackupsTestUtils.init();
|
||||
BackupsTestUtils.loadBlog(12);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A restore is refused while a backup that covers the same tables is running.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRefusesWhileABackupOfTheSameTablesIsRunning() throws Exception
|
||||
{
|
||||
BackupRunResult run = RestoreTestSupport.backup(Map.of());
|
||||
Integer planId = new BackupCatalog().where(BackupsTables.PLAN, "planKey", "nightly").get(0).getValueInteger("id");
|
||||
|
||||
Map<String, Serializable> live = new LinkedHashMap<>();
|
||||
live.put("backupPlanId", planId);
|
||||
live.put("runKey", "live-one");
|
||||
live.put("status", "RUNNING");
|
||||
live.put("startDate", BackupsRuntime.getClock().instant());
|
||||
live.put("heartbeatDate", BackupsRuntime.getClock().instant());
|
||||
Integer liveId = new BackupCatalog().insert(BackupsTables.RUN, live);
|
||||
|
||||
RestoreRequest request = RestoreTestSupport.request(run, RestoreRequest.MODE_INSERT_NEW, "blogPost");
|
||||
assertThat(new RestoreEngine().plan(request).errors()).anyMatch(e -> e.contains("is running"));
|
||||
assertThatThrownBy(() -> new RestoreEngine().execute(request)).hasMessageContaining("is running");
|
||||
assertThat(new BackupCatalog().where(BackupsTables.RESTORE_RUN, "sourceBackupRunId", run.getRunId())).as("nothing was started").isEmpty();
|
||||
|
||||
new BackupCatalog().update(BackupsTables.RUN, liveId, Map.of("heartbeatDate", BackupsRuntime.getClock().instant().minusSeconds(3600)));
|
||||
assertThat(new RestoreEngine().plan(request).errors()).as("a run with an old heartbeat is not running").isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** REPLACE needs the typed confirmation: the table name, or REPLACE n TABLES.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testReplaceNeedsTheTypedConfirmation() throws Exception
|
||||
{
|
||||
BackupRunResult run = RestoreTestSupport.backup(Map.of());
|
||||
|
||||
RestoreRequest one = RestoreTestSupport.request(run, RestoreRequest.MODE_REPLACE, "blogPost");
|
||||
assertThat(new RestoreEngine().plan(one).errors()).anyMatch(e -> e.contains("type blogPost"));
|
||||
assertThat(new RestoreEngine().plan(one.withConfirmationText("blogpost")).errors()).as("case matters").isNotEmpty();
|
||||
assertThat(new RestoreEngine().plan(one.withConfirmationText("blogPost")).errors()).isEmpty();
|
||||
|
||||
RestoreRequest two = RestoreTestSupport.request(run, RestoreRequest.MODE_REPLACE, "author", "blogPost");
|
||||
assertThat(new RestoreEngine().plan(two).errors()).anyMatch(e -> e.contains("REPLACE 2 TABLES"));
|
||||
assertThat(new RestoreEngine().plan(two.withConfirmationText("REPLACE 2 TABLES")).errors()).isEmpty();
|
||||
|
||||
assertThat(new RestoreEngine().plan(RestoreTestSupport.request(run, RestoreRequest.MODE_REPLACE, "blogPost").withConfirmationText("blogPost")
|
||||
.withIdStrategy(RestoreRequest.IDS_REMAP)).errors()).anyMatch(e -> e.contains("REPLACE cannot be combined with new ids"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** REPLACE also needs backups.admin, not just backups.restore.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testReplaceNeedsAdmin() throws Exception
|
||||
{
|
||||
BackupRunResult run = RestoreTestSupport.backup(Map.of());
|
||||
BackupsRuntime.getConfig().withOpenWhenAnonymous(false);
|
||||
RestoreRequest replace = RestoreTestSupport.request(run, RestoreRequest.MODE_REPLACE, "blogPost").withConfirmationText("blogPost");
|
||||
|
||||
QContext.setQSession(new QSession().withPermissions("backups.view", "backups.restore"));
|
||||
assertThat(new RestoreEngine().plan(RestoreTestSupport.request(run, RestoreRequest.MODE_UPSERT, "blogPost")).errors()).isEmpty();
|
||||
assertThat(new RestoreEngine().plan(replace).errors()).anyMatch(e -> e.contains("backups.admin"));
|
||||
|
||||
QContext.setQSession(new QSession().withPermissions("backups.view", "backups.restore", "backups.admin"));
|
||||
assertThat(new RestoreEngine().plan(replace).errors()).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Restores never touch SECRET or backups tables, and temporary (EPHEMERAL) tables
|
||||
** only when they are named.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testNeverRestoresSecretBackupsOrUnnamedEphemeralTables() throws Exception
|
||||
{
|
||||
BackupRunResult run = RestoreTestSupport.backup(Map.of("includeRoles", "DATA,AUDIT,EPHEMERAL"));
|
||||
assertThat(RestoreTestSupport.manifest(run).table("userSession")).isNotNull();
|
||||
|
||||
RestoreEngine.Plan all = new RestoreEngine().plan(new RestoreRequest().withSourceRunId(run.getRunId()));
|
||||
assertThat(all.errors()).isEmpty();
|
||||
assertThat(all.order()).containsExactly("author", "blogPost");
|
||||
assertThat(all.notes()).anyMatch(n -> n.contains("userSession") && n.contains("temporary"));
|
||||
|
||||
RestoreEngine.Plan named = new RestoreEngine().plan(new RestoreRequest().withSourceRunId(run.getRunId()).withTables(List.of("userSession")));
|
||||
assertThat(named.order()).containsExactly("userSession");
|
||||
assertThat(named.warnings()).anyMatch(w -> w.contains("temporary data"));
|
||||
|
||||
assertThat(new RestoreEngine().plan(new RestoreRequest().withSourceRunId(run.getRunId()).withTables(List.of("backups_plan"))).errors())
|
||||
.anyMatch(e -> e.contains("not in this backup"));
|
||||
|
||||
BackupTableMetaData.ofOrWithNew(QContext.getQInstance().getTable("author")).withRole(TableRole.SECRET);
|
||||
RestoreEngine.Plan secret = new RestoreEngine().plan(new RestoreRequest().withSourceRunId(run.getRunId()).withTables(List.of("author")));
|
||||
assertThat(secret.errors()).anyMatch(e -> e.contains("SECRET") && e.contains("never touch"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Security: without backups.restore nobody can run a restore, through the process
|
||||
** (the core permission check) or by calling the engine. With the permission they can.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testRestoreNeedsThePermission() throws Exception
|
||||
{
|
||||
BackupRunResult run = RestoreTestSupport.backup(Map.of());
|
||||
RestoreTestSupport.mutate();
|
||||
int rowsBefore = RestoreTestSupport.rows("blogPost").size();
|
||||
|
||||
BackupsRuntime.getConfig().withOpenWhenAnonymous(false);
|
||||
QContext.getQInstance().setEnforcePermissionsInCoreActions(true);
|
||||
QContext.setQSession(new QSession().withPermissions("backups.view", "backups.run"));
|
||||
|
||||
RunProcessInput input = new RunProcessInput();
|
||||
input.setProcessName("backupRestore");
|
||||
input.addValue("tables", "blogPost");
|
||||
input.addValue("mode", "UPSERT");
|
||||
assertThatThrownBy(() -> new RunProcessAction().execute(input)).isInstanceOf(QPermissionDeniedException.class).hasMessageContaining("backups.restore");
|
||||
|
||||
RestoreRequest request = RestoreTestSupport.request(run, RestoreRequest.MODE_UPSERT, "blogPost");
|
||||
assertThatThrownBy(() -> new RestoreEngine().execute(request)).hasMessageContaining("backups.restore");
|
||||
assertThat(new RestoreEngine().review(request).getErrors()).anyMatch(e -> e.contains("backups.restore"));
|
||||
|
||||
QContext.setQSession(new QSession());
|
||||
assertThat(RestoreTestSupport.rows("blogPost")).as("nothing was written").hasSize(rowsBefore);
|
||||
|
||||
QContext.setQSession(new QSession().withPermissions("backups.view", "backups.restore"));
|
||||
assertThat(new RestoreEngine().plan(request).errors()).isEmpty();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.backups.restore;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.backups.BackupsTestUtils;
|
||||
import com.kingsrook.qbits.backups.codec.HashingOutputStream;
|
||||
import com.kingsrook.qbits.backups.codec.JsonlRowCodec;
|
||||
import com.kingsrook.qbits.backups.manifest.BackupManifest;
|
||||
import com.kingsrook.qbits.backups.model.BackupsTables;
|
||||
import com.kingsrook.qbits.backups.run.BackupCatalog;
|
||||
import com.kingsrook.qbits.backups.run.BackupEngine;
|
||||
import com.kingsrook.qbits.backups.run.BackupRunFactory;
|
||||
import com.kingsrook.qbits.backups.run.BackupRunResult;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Helpers shared by the restore tests: take a backup, mutate the data the way
|
||||
** the spec's acceptance does (delete 10, edit 10, add 10), and read tables back.
|
||||
*******************************************************************************/
|
||||
final class RestoreTestSupport
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private RestoreTestSupport()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Create the nightly plan (with overrides) and run it.
|
||||
*******************************************************************************/
|
||||
static BackupRunResult backup(Map<String, Serializable> overrides) throws Exception
|
||||
{
|
||||
Map<String, Serializable> values = new LinkedHashMap<>(Map.of("name", "Nightly", "planKey", "nightly", "isActive", true,
|
||||
"scopeType", "INSTANCE", "storageBucketAlias", "backups", "slaHours", 26));
|
||||
values.putAll(overrides);
|
||||
BackupCatalog catalog = new BackupCatalog();
|
||||
Integer planId = catalog.insert(BackupsTables.PLAN, values);
|
||||
BackupRunResult result = new BackupEngine().execute(BackupRunFactory.forPlan(catalog.get(BackupsTables.PLAN, planId).orElseThrow(), "MANUAL"));
|
||||
assertThat(result.isUsable()).as(result.getErrorMessage()).isTrue();
|
||||
return (result);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Back up again with the existing nightly plan.
|
||||
*******************************************************************************/
|
||||
static BackupRunResult backupAgain() throws Exception
|
||||
{
|
||||
QRecord plan = new BackupCatalog().where(BackupsTables.PLAN, "planKey", "nightly").get(0);
|
||||
BackupRunResult result = new BackupEngine().execute(BackupRunFactory.forPlan(plan, "MANUAL"));
|
||||
assertThat(result.isUsable()).as(result.getErrorMessage()).isTrue();
|
||||
return (result);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The manifest of a run.
|
||||
*******************************************************************************/
|
||||
static BackupManifest manifest(BackupRunResult result) throws Exception
|
||||
{
|
||||
return (BackupManifest.read(BackupsTestUtils.BACKUPS.getBytes(result.getManifestKey())));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Delete posts 1..10, edit posts 11..20 (title), add posts 31..40.
|
||||
*******************************************************************************/
|
||||
static void mutate() throws Exception
|
||||
{
|
||||
DeleteInput delete = new DeleteInput("blogPost");
|
||||
List<Serializable> ids = new ArrayList<>();
|
||||
for(int i = 1; i <= 10; i++)
|
||||
{
|
||||
ids.add(i);
|
||||
}
|
||||
delete.setPrimaryKeys(ids);
|
||||
new DeleteAction().execute(delete);
|
||||
|
||||
List<QRecord> edits = new ArrayList<>();
|
||||
for(int i = 11; i <= 20; i++)
|
||||
{
|
||||
edits.add(new QRecord().withValue("id", i).withValue("title", "Edited " + i));
|
||||
}
|
||||
UpdateInput update = new UpdateInput("blogPost");
|
||||
update.setRecords(edits);
|
||||
new UpdateAction().execute(update);
|
||||
|
||||
List<Map<String, Serializable>> added = new ArrayList<>();
|
||||
for(int i = 31; i <= 40; i++)
|
||||
{
|
||||
Map<String, Serializable> row = new LinkedHashMap<>();
|
||||
row.put("id", i);
|
||||
row.put("authorId", 1);
|
||||
row.put("title", "Added " + i);
|
||||
row.put("createDate", Instant.parse("2026-10-13T00:00:00Z"));
|
||||
added.add(row);
|
||||
}
|
||||
BackupsTestUtils.insert("blogPost", added);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** All the rows of a table, by id.
|
||||
*******************************************************************************/
|
||||
static List<QRecord> rows(String table) throws QException
|
||||
{
|
||||
QueryInput input = new QueryInput(table);
|
||||
input.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderBy("id", true)));
|
||||
return (new QueryAction().execute(input).getRecords());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A SHA-256 over the canonical rows of a table (by id), leaving out the sensitive
|
||||
** fields (they are not backed up) and, until the core restore flag ships, the created
|
||||
** and modified dates (without the flag the standard actions set them to now).
|
||||
*******************************************************************************/
|
||||
static String contentHash(String table) throws Exception
|
||||
{
|
||||
QTableMetaData meta = QContext.getQInstance().getTable(table);
|
||||
List<String> left = new ArrayList<>();
|
||||
meta.getFields().values().stream().filter(f -> f.getIsSensitive()).forEach(f -> left.add(f.getName()));
|
||||
if(!new StandardRestoreWriter().usesCoreRestoreFlag())
|
||||
{
|
||||
left.addAll(List.of("createDate", "modifyDate"));
|
||||
}
|
||||
JsonlRowCodec codec = new JsonlRowCodec(meta, left);
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for(QRecord row : rows(table))
|
||||
{
|
||||
sb.append(codec.encode(row)).append('\n');
|
||||
}
|
||||
return (HashingOutputStream.sha256Hex(sb.toString().getBytes(StandardCharsets.UTF_8)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A request over some tables, restoring from a run.
|
||||
*******************************************************************************/
|
||||
static RestoreRequest request(BackupRunResult run, String mode, String... tables)
|
||||
{
|
||||
return (new RestoreRequest().withSourceRunId(run.getRunId()).withMode(mode).withTables(List.of(tables)).withRequestedBy("tester"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The summary line whose message contains the text.
|
||||
*******************************************************************************/
|
||||
static ProcessSummaryLine line(List<ProcessSummaryLineInterface> lines, boolean result, String text)
|
||||
{
|
||||
for(ProcessSummaryLineInterface line : lines)
|
||||
{
|
||||
ProcessSummaryLine candidate = (ProcessSummaryLine) line;
|
||||
candidate.pickMessage(result);
|
||||
if(candidate.getMessage() != null && candidate.getMessage().contains(text))
|
||||
{
|
||||
return (candidate);
|
||||
}
|
||||
}
|
||||
return (null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Declare a unique key on a table of the fixture.
|
||||
*******************************************************************************/
|
||||
static void uniqueKey(String table, String... fields)
|
||||
{
|
||||
QContext.getQInstance().getTable(table).withUniqueKey(new UniqueKey(fields));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user