From b928920989303d1f8d017b8541264155c0c9d88c Mon Sep 17 00:00:00 2001 From: James Maes Date: Mon, 12 Oct 2026 16:42:57 -0500 Subject: [PATCH] feat(restore): restore, catalog import and download processes backupRestore (Restore... on backups_run) and backupRestoreTable (Restore this table on backups_artifact): inputs for tables, per-table filters, target table, mode (INSERT_NEW, UPSERT, REPLACE), match key, id strategy (PRESERVE, REMAP_NEW_IDS with backups_id_map and reference rewriting) and options. Review is a real dry run with per-table summary lines (inserted, updated, unchanged, skipped, deleted, conflicts, orphans, schema, sequences, files) and a masked sample diff. Execute takes the pre-restore backup, writes in manifest order in page transactions behind a RestoreWriter (core RestoreWriteFlag and resyncKeySequences selected reflectively), restores files through the content-addressed store with a SHA-256 re-check, and records backups_restore_run and backups_restore_run_table rows. Safety rails enforced in the engine. backupImportCatalog rebuilds backups_run and backups_artifact from the manifests under a bucket alias and prefix (trigger IMPORTED). backupDownload issues audited presigned links, ciphertext only. Co-Authored-By: Claude Sonnet 5.5 --- .../backups/processes/BackupsProcesses.java | 2 + .../backups/processes/CatalogProcesses.java | 443 ++++++ .../backups/processes/RestoreProcesses.java | 269 ++++ .../qbits/backups/restore/BackupSource.java | 359 +++++ .../backups/restore/CatalogImporter.java | 362 +++++ .../qbits/backups/restore/RestoreEngine.java | 651 ++++++++ .../qbits/backups/restore/RestoreOutcome.java | 710 +++++++++ .../qbits/backups/restore/RestorePass.java | 1304 +++++++++++++++++ .../qbits/backups/restore/RestoreRequest.java | 612 ++++++++ .../backups/restore/RestoreSummaries.java | 222 +++ .../qbits/backups/restore/RestoreWriter.java | 81 + .../restore/StandardRestoreWriter.java | 257 ++++ .../backups/restore/CatalogImportTest.java | 252 ++++ .../backups/restore/RestoreModesTest.java | 515 +++++++ .../backups/restore/RestoreProcessTest.java | 249 ++++ .../backups/restore/RestoreSafetyTest.java | 190 +++ .../backups/restore/RestoreTestSupport.java | 220 +++ 17 files changed, 6698 insertions(+) create mode 100644 src/main/java/com/kingsrook/qbits/backups/processes/CatalogProcesses.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/processes/RestoreProcesses.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/restore/BackupSource.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/restore/CatalogImporter.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/restore/RestoreEngine.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/restore/RestoreOutcome.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/restore/RestorePass.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/restore/RestoreRequest.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/restore/RestoreSummaries.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/restore/RestoreWriter.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/restore/StandardRestoreWriter.java create mode 100644 src/test/java/com/kingsrook/qbits/backups/restore/CatalogImportTest.java create mode 100644 src/test/java/com/kingsrook/qbits/backups/restore/RestoreModesTest.java create mode 100644 src/test/java/com/kingsrook/qbits/backups/restore/RestoreProcessTest.java create mode 100644 src/test/java/com/kingsrook/qbits/backups/restore/RestoreSafetyTest.java create mode 100644 src/test/java/com/kingsrook/qbits/backups/restore/RestoreTestSupport.java diff --git a/src/main/java/com/kingsrook/qbits/backups/processes/BackupsProcesses.java b/src/main/java/com/kingsrook/qbits/backups/processes/BackupsProcesses.java index 1c0843c..435f860 100644 --- a/src/main/java/com/kingsrook/qbits/backups/processes/BackupsProcesses.java +++ b/src/main/java/com/kingsrook/qbits/backups/processes/BackupsProcesses.java @@ -74,6 +74,8 @@ public final class BackupsProcesses processes.add(permission(etl(RUN_PLAN, "Run backup now", BackupsTables.PLAN, RunPlanTransform.class, RunPlanLoad.class, "This is a preview of what the backup will include. Nothing is written until you continue."), BackupsPermissions.RUN)); processes.addAll(VerifyProcesses.processes()); + processes.addAll(RestoreProcesses.processes()); + processes.addAll(CatalogProcesses.processes()); return (processes); } diff --git a/src/main/java/com/kingsrook/qbits/backups/processes/CatalogProcesses.java b/src/main/java/com/kingsrook/qbits/backups/processes/CatalogProcesses.java new file mode 100644 index 0000000..a56d21a --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/processes/CatalogProcesses.java @@ -0,0 +1,443 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.processes; + + +import java.io.Serializable; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.backups.BackupsRuntime; +import com.kingsrook.qbits.backups.model.BackupsTables; +import com.kingsrook.qbits.backups.permissions.BackupsPermissions; +import com.kingsrook.qbits.backups.restore.CatalogImporter; +import com.kingsrook.qbits.backups.restore.RestoreSummaries; +import com.kingsrook.qbits.backups.run.BackupCatalog; +import com.kingsrook.qbits.backups.storage.BackupStorage; +import com.kingsrook.qqq.backend.core.actions.audits.AuditAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** The catalog and download processes (spec 9.6 and 9.7): Import backups from + ** storage (a table action on backups_run) and Download (a record action on a run + ** or an artifact). Standard streamed ETL, Review and Result summary lines. + *******************************************************************************/ +public final class CatalogProcesses +{ + public static final String IMPORT_CATALOG = "backupImportCatalog"; + public static final String DOWNLOAD = "backupDownload"; + public static final String DOWNLOAD_ARTIFACT = "backupDownloadArtifact"; + + public static final Duration LINK_TTL = Duration.ofMinutes(15); + + private static final QLogger LOG = QLogger.getLogger(CatalogProcesses.class); + + + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private CatalogProcesses() + { + } + + + + /******************************************************************************* + ** The processes defined here. + *******************************************************************************/ + public static List processes() + { + List processes = new ArrayList<>(); + + QProcessMetaData importProcess = definition(IMPORT_CATALOG, "Import backups from storage", BackupsTables.RUN, ImportExtract.class, ImportTransform.class, ImportLoad.class, + "These are the backups found in storage that are not in the catalog yet. Nothing is written until you continue."); + importProcess.withStep(0, new QFrontendStepMetaData() + .withName("input") + .withLabel("Where are the backups?") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM)) + .withFormField(new QFieldMetaData("bucketAlias", QFieldType.STRING).withLabel("Backup bucket alias").withIsRequired(true)) + .withFormField(new QFieldMetaData("prefix", QFieldType.STRING).withLabel("Folder (prefix)").withDefaultValue("qqq-backups/"))); + importProcess.setMinInputRecords(0); + processes.add(BackupsProcesses.permission(importProcess, BackupsPermissions.ADMIN)); + + processes.add(BackupsProcesses.permission(definition(DOWNLOAD, "Download", BackupsTables.RUN, ExtractViaQueryStep.class, DownloadTransform.class, DownloadLoad.class, + "These are the files you will get download links for. Nothing is changed."), BackupsPermissions.RESTORE)); + processes.add(BackupsProcesses.permission(definition(DOWNLOAD_ARTIFACT, "Download", BackupsTables.ARTIFACT, ExtractViaQueryStep.class, DownloadTransform.class, DownloadLoad.class, + "These are the files you will get download links for. Nothing is changed."), BackupsPermissions.RESTORE)); + return (processes); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QProcessMetaData definition(String name, String label, String table, Class extract, + Class transform, Class load, String preview) + { + Map values = new HashMap<>(); + values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, table); + values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table); + values.put(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION, true); + values.put(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true); + values.put(StreamedETLWithFrontendProcess.FIELD_TRANSACTION_LEVEL, StreamedETLWithFrontendProcess.TRANSACTION_LEVEL_AUTO_COMMIT); + values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, preview); + QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(extract, transform, load, values); + process.setName(name); + process.setLabel(label); + process.setTableName(table); + process.setPermissionRules(BackupsTables.permissionRules()); + return (process); + } + + + + /******************************************************************************* + ** Lists the manifests under the bucket alias and prefix; one record per manifest. + *******************************************************************************/ + public static class ImportExtract extends AbstractExtractStep + { + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + String alias = input.getValueString("bucketAlias"); + if(alias == null || alias.isBlank()) + { + throw (new QUserFacingException("Choose the backup bucket alias to import from.")); + } + BackupStorage storage = BackupsRuntime.storageFor(alias); + for(String key : new CatalogImporter().findManifestKeys(storage, input.getValueString("prefix"))) + { + getRecordPipe().addRecord(new QRecord().withTableName(BackupsTables.RUN).withValue("manifestKey", key).withValue("storageBucketAlias", alias)); + } + } + } + + + + /******************************************************************************* + ** Review: what each manifest is (new, already in the catalog, or damaged). + *******************************************************************************/ + public static class ImportTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface + { + private final ProcessSummaryLine found = RestoreSummaries.line(Status.INFO, 0, "backup was found in storage", "backups were found in storage", "backup was found in storage", "backups were found in storage"); + private final ProcessSummaryLine newRuns = RestoreSummaries.line(Status.OK, 0, "backup will be added to the catalog", "backups will be added to the catalog", "backup was added to the catalog", "backups were added to the catalog"); + private final ProcessSummaryLine existing = RestoreSummaries.line(Status.INFO, 0, "backup is already in the catalog (skipped)", "backups are already in the catalog (skipped)", "backup was already in the catalog", "backups were already in the catalog"); + private final ProcessSummaryLine invalid = RestoreSummaries.line(Status.ERROR, 0, "backup has a manifest or checksum problem and will be skipped", "backups have a manifest or checksum problem and will be skipped", + "backup had a manifest or checksum problem and was skipped", "backups had a manifest or checksum problem and were skipped"); + private final ArrayList problems = new ArrayList<>(); + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + for(QRecord record : input.getRecords()) + { + BackupStorage storage = BackupsRuntime.storageFor(record.getValueString("storageBucketAlias")); + CatalogImporter.Inspection inspection = new CatalogImporter().inspect(storage, record.getValueString("manifestKey")); + found.incrementCount(); + switch(inspection.status()) + { + case CatalogImporter.Inspection.NEW -> + { + newRuns.incrementCount(); + output.getRecords().add(record); + } + case CatalogImporter.Inspection.EXISTS -> existing.incrementCount(); + default -> + { + invalid.incrementCount(); + for(String problem : inspection.problems()) + { + problems.add(inspection.manifestKey() + ": " + problem); + } + } + } + } + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen) + { + ArrayList lines = new ArrayList<>(); + found.addSelfToListIfAnyCount(lines); + newRuns.addSelfToListIfAnyCount(lines); + existing.addSelfToListIfAnyCount(lines); + invalid.setBulletsOfText(problems); + invalid.addSelfToListIfAnyCount(lines); + return (lines); + } + } + + + + /******************************************************************************* + ** Execute: create the run and artifact rows from each new manifest. + *******************************************************************************/ + public static class ImportLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface + { + private final ProcessSummaryLine imported = RestoreSummaries.line(Status.OK, 0, "backup will be added to the catalog", "backups will be added to the catalog", "backup was added to the catalog", "backups were added to the catalog"); + private final ProcessSummaryLine skipped = RestoreSummaries.line(Status.WARNING, 0, "backup was not added", "backups were not added", "backup was not added", "backups were not added"); + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + CatalogImporter importer = new CatalogImporter(); + for(QRecord record : input.getRecords()) + { + String alias = record.getValueString("storageBucketAlias"); + BackupStorage storage = BackupsRuntime.storageFor(alias); + CatalogImporter.Inspection inspection = importer.inspect(storage, record.getValueString("manifestKey")); + if(CatalogImporter.Inspection.NEW.equals(inspection.status())) + { + imported.incrementCountAndAddPrimaryKey(importer.importRun(alias, storage, inspection)); + } + else + { + skipped.incrementCount(); + } + output.getRecords().add(record); + } + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen) + { + ArrayList lines = new ArrayList<>(); + imported.addSelfToListIfAnyCount(lines); + skipped.addSelfToListIfAnyCount(lines); + return (lines); + } + } + + + + /******************************************************************************* + ** One object to link to. + *******************************************************************************/ + record DownloadItem(String auditTable, Integer auditId, BackupStorage storage, String key, String downloadName, boolean encrypted, long size) + { + } + + + + /******************************************************************************* + ** The objects the selected records ask for: for a run, all of its artifacts; for + ** an artifact, itself plus the manifest and checksums of its run. + *******************************************************************************/ + static List items(List records) throws QException + { + BackupCatalog catalog = new BackupCatalog(); + Map items = new LinkedHashMap<>(); + for(QRecord record : records) + { + boolean isArtifact = BackupsTables.ARTIFACT.equals(record.getTableName()) || record.getValue("artifactType") != null; + Integer runId = isArtifact ? record.getValueInteger("backupRunId") : record.getValueInteger("id"); + QRecord run = catalog.get(BackupsTables.RUN, runId).orElseThrow(() -> new QUserFacingException("The backup run was not found.")); + String table = isArtifact ? BackupsTables.ARTIFACT : BackupsTables.RUN; + BackupStorage storage = BackupsRuntime.storageFor(run.getValueString("storageBucketAlias")); + + List artifacts = new ArrayList<>(); + if(isArtifact) + { + artifacts.add(record); + for(QRecord other : catalog.where(BackupsTables.ARTIFACT, "backupRunId", runId)) + { + if("MANIFEST".equals(other.getValueString("artifactType")) || "CHECKSUMS".equals(other.getValueString("artifactType"))) + { + artifacts.add(other); + } + } + } + else + { + artifacts.addAll(catalog.where(BackupsTables.ARTIFACT, "backupRunId", runId)); + } + + for(QRecord artifact : artifacts) + { + if(!"COMPLETE".equals(artifact.getValueString("status"))) + { + continue; + } + String key = run.getValueString("runPrefix") + artifact.getValueString("objectKey"); + String name = run.getValueString("runKey") + "-" + artifact.getValueString("objectKey").replace('/', '-'); + items.putIfAbsent(key, new DownloadItem(table, isArtifact && artifact == record ? artifact.getValueInteger("id") : runId, storage, key, name, + Boolean.TRUE.equals(artifact.getValueBoolean("encrypted")), artifact.getValueLong("bytesStored") == null ? 0 : artifact.getValueLong("bytesStored"))); + } + } + return (new ArrayList<>(items.values())); + } + + + + /******************************************************************************* + ** Review: what will be linked. + *******************************************************************************/ + public static class DownloadTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface + { + private final ProcessSummaryLine objects = RestoreSummaries.line(Status.OK, 0, "file will get a download link", "files will get download links", "file got a download link", "files got download links"); + private final ProcessSummaryLine encrypted = RestoreSummaries.line(Status.INFO, 0, "file is encrypted: you get the ciphertext only, and decrypt it with the command line tool", + "files are encrypted: you get the ciphertext only, and decrypt them with the command line tool", "file is encrypted (ciphertext only)", "files are encrypted (ciphertext only)"); + private long bytes; + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + for(DownloadItem item : items(input.getRecords())) + { + objects.incrementCount(); + bytes += item.size(); + if(item.encrypted()) + { + encrypted.incrementCount(); + } + } + output.getRecords().addAll(input.getRecords()); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen) + { + ArrayList lines = new ArrayList<>(); + objects.addSelfToListIfAnyCount(lines); + encrypted.addSelfToListIfAnyCount(lines); + if(objects.getCount() != null && objects.getCount() > 0) + { + lines.add(RestoreSummaries.message(Status.INFO, "The links last " + LINK_TTL.toMinutes() + " minutes and the download is about " + bytes + " bytes. Each link is recorded in the audit.")); + } + return (lines); + } + } + + + + /******************************************************************************* + ** Execute: issue the presigned links (ciphertext only), each one audited. + *******************************************************************************/ + public static class DownloadLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface + { + private final ProcessSummaryLine issued = RestoreSummaries.line(Status.OK, 0, "download link was issued", "download links were issued", "download link was issued", "download links were issued"); + private final ArrayList links = new ArrayList<>(); + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + String user = QContext.getQSession() == null || QContext.getQSession().getUser() == null ? "system" : QContext.getQSession().getUser().getIdReference(); + for(DownloadItem item : items(input.getRecords())) + { + Instant expires = BackupsRuntime.getClock().instant().plus(LINK_TTL); + String url = item.storage().presignGet(item.key(), LINK_TTL, item.downloadName()); + links.add(item.downloadName() + (item.encrypted() ? " (encrypted)" : "") + ": " + url); + issued.incrementCount(); + LOG.info("Issued a backup download link", logPair("object", item.key()), logPair("user", user), logPair("expires", expires)); + try + { + AuditAction.execute(item.auditTable(), item.auditId(), Map.of(), "Download link issued for " + item.key() + " to " + user + ", expires " + expires); + } + catch(RuntimeException e) + { + LOG.warn("Could not audit a download link", e, logPair("object", item.key())); + } + } + output.getRecords().addAll(input.getRecords()); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen) + { + ArrayList lines = new ArrayList<>(); + issued.setBulletsOfText(links); + issued.addSelfToListIfAnyCount(lines); + if(issued.getCount() != null && issued.getCount() > 0) + { + lines.add(RestoreSummaries.message(Status.INFO, "The links expire in " + LINK_TTL.toMinutes() + " minutes. Encrypted files download as ciphertext: use the command line tool to decrypt them.")); + } + return (lines); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/processes/RestoreProcesses.java b/src/main/java/com/kingsrook/qbits/backups/processes/RestoreProcesses.java new file mode 100644 index 0000000..b8d7ef2 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/processes/RestoreProcesses.java @@ -0,0 +1,269 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.processes; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qbits.backups.model.BackupsTables; +import com.kingsrook.qbits.backups.permissions.BackupsPermissions; +import com.kingsrook.qbits.backups.restore.RestoreEngine; +import com.kingsrook.qbits.backups.restore.RestoreOutcome; +import com.kingsrook.qbits.backups.restore.RestoreRequest; +import com.kingsrook.qbits.backups.restore.RestoreSummaries; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; + + +/******************************************************************************* + ** The restore processes (spec 9.3): Restore... on a backup run, and Restore this + ** table on a backup artifact. Both are the standard streamed ETL. The first + ** screen takes the inputs; Review is a real dry run of the whole restore with the + ** per-table summary lines and a masked sample diff; Execute is the restore, and + ** Result shows what was done. Neither is in the nav. + *******************************************************************************/ +public final class RestoreProcesses +{ + public static final String RESTORE = "backupRestore"; + public static final String RESTORE_TABLE = "backupRestoreTable"; + + public static final String STEP_INPUT = "input"; + + + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private RestoreProcesses() + { + } + + + + /******************************************************************************* + ** The processes defined here. + *******************************************************************************/ + public static List processes() + { + List processes = new ArrayList<>(); + processes.add(BackupsProcesses.permission(restore(RESTORE, "Restore...", BackupsTables.RUN), BackupsPermissions.RESTORE)); + processes.add(BackupsProcesses.permission(restore(RESTORE_TABLE, "Restore this table", BackupsTables.ARTIFACT), BackupsPermissions.RESTORE)); + return (processes); + } + + + + /******************************************************************************* + ** One restore process, sourced from the table whose records it is run on. + *******************************************************************************/ + private static QProcessMetaData restore(String name, String label, String table) + { + Map values = new HashMap<>(); + values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, table); + values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table); + values.put(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION, true); + values.put(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true); + values.put(StreamedETLWithFrontendProcess.FIELD_TRANSACTION_LEVEL, StreamedETLWithFrontendProcess.TRANSACTION_LEVEL_AUTO_COMMIT); + values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, "This is what the restore will do. Nothing is written until you continue."); + + QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(ExtractViaQueryStep.class, RestoreTransform.class, RestoreLoad.class, values); + process.withStep(0, inputStep()); + process.setName(name); + process.setLabel(label); + process.setTableName(table); + process.setPermissionRules(BackupsTables.permissionRules()); + return (process); + } + + + + /******************************************************************************* + ** The first screen: what to restore and how. + *******************************************************************************/ + private static QFrontendStepMetaData inputStep() + { + return (new QFrontendStepMetaData() + .withName(STEP_INPUT) + .withLabel("Restore options") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM)) + .withFormField(new QFieldMetaData("tables", QFieldType.STRING).withLabel("Tables (blank for every table in the backup)")) + .withFormField(new QFieldMetaData("tableFilters", QFieldType.TEXT).withLabel("Only these records (JSON: table name to filter, optional)")) + .withFormField(new QFieldMetaData("targetTable", QFieldType.STRING).withLabel("Restore into another table (optional, one table only)")) + .withFormField(new QFieldMetaData("mode", QFieldType.STRING).withLabel("Mode").withIsRequired(true).withDefaultValue(RestoreRequest.MODE_INSERT_NEW) + .withPossibleValueSourceName(BackupsTables.pvs("restore_mode"))) + .withFormField(new QFieldMetaData("matchOn", QFieldType.STRING).withLabel("Match records on (PRIMARY_KEY, or a unique key's fields)").withDefaultValue(RestoreRequest.MATCH_PRIMARY_KEY)) + .withFormField(new QFieldMetaData("idStrategy", QFieldType.STRING).withLabel("Ids").withIsRequired(true).withDefaultValue(RestoreRequest.IDS_PRESERVE) + .withPossibleValueSourceName(BackupsTables.pvs("id_strategy"))) + .withFormField(new QFieldMetaData("suppressAutomations", QFieldType.BOOLEAN).withLabel("Do not trigger automations").withDefaultValue(true)) + .withFormField(new QFieldMetaData("publishEvents", QFieldType.BOOLEAN).withLabel("Publish change events").withDefaultValue(false)) + .withFormField(new QFieldMetaData("runCustomizers", QFieldType.BOOLEAN).withLabel("Run table customizers").withDefaultValue(false)) + .withFormField(new QFieldMetaData("perRecordAudit", QFieldType.BOOLEAN).withLabel("Audit each record (otherwise one summary per table)").withDefaultValue(false)) + .withFormField(new QFieldMetaData("restoreFiles", QFieldType.BOOLEAN).withLabel("Restore referenced files").withDefaultValue(true)) + .withFormField(new QFieldMetaData("fileTargetBucketAlias", QFieldType.STRING).withLabel("Restore files to another bucket alias (optional)")) + .withFormField(new QFieldMetaData("fileTargetPrefix", QFieldType.STRING).withLabel("Restore files under this prefix (optional)")) + .withFormField(new QFieldMetaData("preRestoreBackup", QFieldType.BOOLEAN).withLabel("Back up the target first").withDefaultValue(true)) + .withFormField(new QFieldMetaData("confirmationText", QFieldType.STRING).withLabel("Type the table name (or REPLACE TABLES) to confirm a REPLACE"))); + } + + + + /******************************************************************************* + ** The backup runs the selected records belong to, with the tables chosen for each + ** (empty means the tables of the form). Artifact records choose their own table. + *******************************************************************************/ + static Map> sources(List records) throws QException + { + Map> sources = new LinkedHashMap<>(); + for(QRecord record : records) + { + boolean artifact = BackupsTables.ARTIFACT.equals(record.getTableName()) || record.getValue("artifactType") != null; + if(artifact) + { + String table = record.getValueString("tableName"); + if(table == null) + { + throw (new QUserFacingException("Only the data of a table can be restored: choose a table data artifact.")); + } + sources.computeIfAbsent(record.getValueInteger("backupRunId"), k -> new LinkedHashSet<>()).add(table); + } + else + { + sources.computeIfAbsent(record.getValueInteger("id"), k -> new LinkedHashSet<>()); + } + } + return (sources); + } + + + + /******************************************************************************* + ** Review (preview, validate) is the dry run; execute only re-checks the rails. + *******************************************************************************/ + public static class RestoreTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface + { + private final List outcomes = new ArrayList<>(); + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + boolean execute = StreamedETLWithFrontendProcess.STEP_NAME_EXECUTE.equals(input.getStepName()); + RestoreEngine engine = new RestoreEngine(); + for(Map.Entry> source : sources(input.getRecords()).entrySet()) + { + RestoreRequest request = RestoreRequest.fromInput(input, source.getKey(), new ArrayList<>(source.getValue())); + if(execute) + { + RestoreEngine.Plan plan = engine.plan(request); + if(!plan.errors().isEmpty()) + { + throw (new QUserFacingException(String.join(" ", plan.errors()))); + } + } + else + { + outcomes.add(engine.review(request)); + } + } + output.getRecords().addAll(input.getRecords()); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen) + { + ArrayList lines = new ArrayList<>(); + for(RestoreOutcome outcome : outcomes) + { + lines.addAll(RestoreSummaries.lines(outcome, false)); + } + return (lines); + } + } + + + + /******************************************************************************* + ** Execute: the restore itself, one backup run at a time. + *******************************************************************************/ + public static class RestoreLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface + { + private final List outcomes = new ArrayList<>(); + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + RestoreEngine engine = new RestoreEngine(); + for(Map.Entry> source : sources(input.getRecords()).entrySet()) + { + RestoreRequest request = RestoreRequest.fromInput(input, source.getKey(), new ArrayList<>(source.getValue())); + outcomes.add(engine.execute(request)); + } + output.getRecords().addAll(input.getRecords()); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen) + { + ArrayList lines = new ArrayList<>(); + for(RestoreOutcome outcome : outcomes) + { + lines.addAll(RestoreSummaries.lines(outcome, true)); + } + return (lines); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/restore/BackupSource.java b/src/main/java/com/kingsrook/qbits/backups/restore/BackupSource.java new file mode 100644 index 0000000..92511b9 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/restore/BackupSource.java @@ -0,0 +1,359 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.restore; + + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.security.DigestInputStream; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.zip.GZIPInputStream; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.kingsrook.qbits.backups.BackupsRuntime; +import com.kingsrook.qbits.backups.codec.ArtifactTransform; +import com.kingsrook.qbits.backups.codec.CsvRowCodec; +import com.kingsrook.qbits.backups.codec.HashingOutputStream; +import com.kingsrook.qbits.backups.codec.JsonlRowCodec; +import com.kingsrook.qbits.backups.manifest.BackupManifest; +import com.kingsrook.qbits.backups.model.BackupsTables; +import com.kingsrook.qbits.backups.run.BackupCatalog; +import com.kingsrook.qbits.backups.storage.BackupStorage; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** A backup run opened for restore: the catalog row, its manifest (checked against + ** the manifest hash in the catalog), its storage, and readers for the rows and the + ** file index. Every part is checked against the stored SHA-256 in the manifest as + ** it is read, so a damaged part fails the restore instead of restoring bad rows. + *******************************************************************************/ +public final class BackupSource +{ + private static final ObjectMapper MAPPER = new ObjectMapper(); + + ///////////////////////////////////////////////////////////////////////////////// + // Supplies the transform that decrypts the parts of an encrypted run. The // + // default knows no keys, so restoring an encrypted run fails with a clear // + // message until the crypto package (QQQ core PR 1107) provider is wired in. // + ///////////////////////////////////////////////////////////////////////////////// + private static volatile Function transformResolver = null; + + private final QRecord run; + private final BackupStorage storage; + private final String runPrefix; + private final BackupManifest manifest; + private final ArtifactTransform transform; + + + + /******************************************************************************* + ** + *******************************************************************************/ + private BackupSource(QRecord run, BackupStorage storage, String runPrefix, BackupManifest manifest, ArtifactTransform transform) + { + this.run = run; + this.storage = storage; + this.runPrefix = runPrefix; + this.manifest = manifest; + this.transform = transform; + } + + + + /******************************************************************************* + ** Install the resolver for the decrypting transform (null removes it). + *******************************************************************************/ + public static void setTransformResolver(Function resolver) + { + transformResolver = resolver; + } + + + + /******************************************************************************* + ** Open a run for restore. + *******************************************************************************/ + public static BackupSource open(Integer runId) throws QException + { + QRecord run = new BackupCatalog().get(BackupsTables.RUN, runId).orElseThrow(() -> new QException("The backup run " + runId + " was not found")); + String status = run.getValueString("status"); + if(!"SUCCEEDED".equals(status) && !"SUCCEEDED_WITH_WARNINGS".equals(status)) + { + throw (new QException("The backup run " + run.getValueString("runKey") + " did not complete (status " + status + "), so it cannot be restored")); + } + String manifestKey = run.getValueString("manifestKey"); + if(manifestKey == null) + { + throw (new QException("The backup run " + run.getValueString("runKey") + " has no manifest")); + } + + BackupStorage storage = BackupsRuntime.storageFor(run.getValueString("storageBucketAlias")); + try + { + byte[] bytes = readAll(storage.openRead(manifestKey)); + String expected = run.getValueString("manifestSha256"); + if(expected != null && !expected.equals(HashingOutputStream.sha256Hex(bytes))) + { + throw (new QException("The manifest of backup run " + run.getValueString("runKey") + " does not match its recorded checksum")); + } + BackupManifest manifest = BackupManifest.read(bytes); + String runPrefix = run.getValueString("runPrefix"); + if(runPrefix == null) + { + runPrefix = manifestKey.substring(0, manifestKey.length() - "manifest.json".length()); + } + return (new BackupSource(run, storage, runPrefix, manifest, transformFor(manifest))); + } + catch(IOException | RuntimeException e) + { + throw (new QException("Could not read the manifest of backup run " + run.getValueString("runKey") + ": " + e.getMessage(), e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static ArtifactTransform transformFor(BackupManifest manifest) throws QException + { + BackupManifest.EncryptionInfo encryption = manifest.encryption(); + if(encryption == null || encryption.mode() == null || "NONE".equals(encryption.mode())) + { + return (ArtifactTransform.IDENTITY); + } + Function resolver = transformResolver; + ArtifactTransform resolved = resolver == null ? null : resolver.apply(encryption); + if(resolved == null) + { + throw (new QException("This backup is encrypted (key " + encryption.keyRef() + "), and no key provider is available to restore it")); + } + return (resolved); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + static byte[] readAll(InputStream in) throws IOException + { + try(in) + { + return (in.readAllBytes()); + } + } + + + + /******************************************************************************* + ** The catalog row of the run. + *******************************************************************************/ + public QRecord getRun() + { + return (run); + } + + + + /******************************************************************************* + ** The manifest. + *******************************************************************************/ + public BackupManifest getManifest() + { + return (manifest); + } + + + + /******************************************************************************* + ** The storage the backup lives in. + *******************************************************************************/ + public BackupStorage getStorage() + { + return (storage); + } + + + + /******************************************************************************* + ** The transform that decrypts the stored parts (identity when not encrypted). + *******************************************************************************/ + public ArtifactTransform getTransform() + { + return (transform); + } + + + + /******************************************************************************* + ** The prefix of the run's objects. + *******************************************************************************/ + public String getRunPrefix() + { + return (runPrefix); + } + + + + /******************************************************************************* + ** Stream the rows of one table, decoding each with the target table's field types. + ** The sink may throw an unchecked exception to stop. + *******************************************************************************/ + public void readRows(BackupManifest.TableEntry entry, QTableMetaData decodeTable, Consumer sink) throws QException + { + List parts = new ArrayList<>(); + for(BackupManifest.Part part : entry.parts()) + { + if("JSONL".equals(part.format())) + { + parts.add(part); + } + } + if(parts.isEmpty()) + { + parts.addAll(entry.parts()); + } + + for(BackupManifest.Part part : parts) + { + try + { + readPart(part, decodeTable, sink); + } + catch(IOException e) + { + throw (new QException("Could not read " + part.key() + " of the backup: " + e.getMessage(), e)); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void readPart(BackupManifest.Part part, QTableMetaData decodeTable, Consumer sink) throws IOException, QException + { + MessageDigest digest = HashingOutputStream.newSha256(); + try(DigestInputStream stored = new DigestInputStream(storage.openRead(runPrefix + part.key()), digest)) + { + InputStream plain = transform.unwrap(stored); + InputStream text = "GZIP".equals(part.compression()) ? new GZIPInputStream(plain) : plain; + + if("CSV".equals(part.format())) + { + String all = new String(text.readAllBytes(), StandardCharsets.UTF_8); + List records = CsvRowCodec.splitRecords(all); + if(!records.isEmpty()) + { + CsvRowCodec codec = new CsvRowCodec(decodeTable, null); + List header = CsvRowCodec.splitCells(records.get(0)); + for(int i = 1; i < records.size(); i++) + { + sink.accept(codec.toRecord(CsvRowCodec.splitCells(records.get(i)), header)); + } + } + } + else + { + JsonlRowCodec codec = new JsonlRowCodec(decodeTable, null); + BufferedReader reader = new BufferedReader(new InputStreamReader(text, StandardCharsets.UTF_8)); + String line; + while((line = reader.readLine()) != null) + { + if(!line.isEmpty()) + { + sink.accept(codec.decode(line)); + } + } + } + + stored.transferTo(OutputStream.nullOutputStream()); + } + + String actual = HashingOutputStream.hex(digest.digest()); + if(part.sha256Stored() != null && !part.sha256Stored().equals(actual)) + { + throw (new IOException("checksum mismatch (the stored part is damaged or was changed)")); + } + } + + + + /******************************************************************************* + ** The file index entries of the given tables, keyed by table, record key and field. + *******************************************************************************/ + public Map fileIndex(Set tables) throws QException + { + Map index = new HashMap<>(); + if(manifest.files() == null || manifest.files().indexKeys() == null) + { + return (index); + } + for(String indexKey : manifest.files().indexKeys()) + { + try(InputStream raw = storage.openRead(runPrefix + indexKey); + BufferedReader reader = new BufferedReader(new InputStreamReader(new GZIPInputStream(transform.unwrap(raw)), StandardCharsets.UTF_8))) + { + String line; + while((line = reader.readLine()) != null) + { + if(line.isEmpty()) + { + continue; + } + JsonNode node = MAPPER.readTree(new ByteArrayInputStream(line.getBytes(StandardCharsets.UTF_8))); + if(tables.contains(node.path("table").asText())) + { + index.put(indexKeyOf(node.path("table").asText(), node.path("recordKey").asText(), node.path("field").asText()), node); + } + } + } + catch(IOException e) + { + throw (new QException("Could not read the file index " + indexKey + ": " + e.getMessage(), e)); + } + } + return (index); + } + + + + /******************************************************************************* + ** The lookup key of a file index entry. + *******************************************************************************/ + public static String indexKeyOf(String table, String recordKey, String field) + { + return (table + "\u0001" + recordKey + "\u0001" + field); + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/restore/CatalogImporter.java b/src/main/java/com/kingsrook/qbits/backups/restore/CatalogImporter.java new file mode 100644 index 0000000..4db1435 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/restore/CatalogImporter.java @@ -0,0 +1,362 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.restore; + + +import java.io.IOException; +import java.io.Serializable; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import com.kingsrook.qbits.backups.codec.HashingOutputStream; +import com.kingsrook.qbits.backups.manifest.BackupManifest; +import com.kingsrook.qbits.backups.manifest.TableFingerprints; +import com.kingsrook.qbits.backups.model.BackupsTables; +import com.kingsrook.qbits.backups.run.BackupCatalog; +import com.kingsrook.qbits.backups.storage.BackupStorage; +import com.kingsrook.qbits.backups.storage.StoredObjectInfo; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; + + +/******************************************************************************* + ** Rebuilds the catalog from storage (spec 9.6): every manifest is + ** self-describing, so after the database with the catalog is lost, the manifests + ** under a bucket alias and prefix are listed, validated, and turned back into + ** backups_run and backups_artifact rows (trigger IMPORTED). The content + ** addressed file store is re-registered in backups_file_object so dedup keeps + ** working. Importing is idempotent: a run already in the catalog is skipped. + ** + ** Validation: the manifest must be readable and of a known version, its run key + ** must match its folder, checksums.sha256 must exist and agree with the manifest + ** for every part, and every part must exist with the recorded size. The checksum + ** of the manifest itself is computed and stored on the run, so a later restore + ** detects a changed manifest. + *******************************************************************************/ +public class CatalogImporter +{ + private static final Pattern PART_NUMBER = Pattern.compile("part-(\\d+)"); + private static final Pattern SHA_KEY = Pattern.compile("^[0-9a-f]{64}$"); + + private final BackupCatalog catalog = new BackupCatalog(); + + + + /******************************************************************************* + ** What was found for one manifest. + *******************************************************************************/ + public record Inspection(String manifestKey, String status, List problems, BackupManifest manifest, String runPrefix, String manifestSha256) + { + public static final String NEW = "NEW"; + public static final String EXISTS = "EXISTS"; + public static final String INVALID = "INVALID"; + } + + + + /******************************************************************************* + ** The keys of every manifest under a prefix. + *******************************************************************************/ + public List findManifestKeys(BackupStorage storage, String prefix) + { + List keys = new ArrayList<>(); + for(StoredObjectInfo info : storage.list(prefix == null ? "" : prefix)) + { + if(info.key().endsWith("/manifest.json")) + { + keys.add(info.key()); + } + } + return (keys); + } + + + + /******************************************************************************* + ** Read and validate one manifest, and say whether the catalog already has it. + *******************************************************************************/ + public Inspection inspect(BackupStorage storage, String manifestKey) throws QException + { + String runPrefix = manifestKey.substring(0, manifestKey.length() - "manifest.json".length()); + byte[] bytes; + BackupManifest manifest; + try + { + bytes = BackupSource.readAll(storage.openRead(manifestKey)); + manifest = BackupManifest.read(bytes); + } + catch(IOException | RuntimeException e) + { + return (invalid(manifestKey, runPrefix, "The manifest cannot be read: " + e.getMessage())); + } + + String sha = HashingOutputStream.sha256Hex(bytes); + List problems = new ArrayList<>(); + if(manifest.manifestVersion() > BackupManifest.CURRENT_VERSION) + { + problems.add("The manifest is version " + manifest.manifestVersion() + ", newer than this qbit understands"); + } + String[] segments = runPrefix.split("/"); + if(manifest.runKey() == null || segments.length == 0 || !manifest.runKey().equals(segments[segments.length - 1])) + { + problems.add("The manifest's run key does not match its folder"); + } + + if(!problems.isEmpty()) + { + return (new Inspection(manifestKey, Inspection.INVALID, problems, manifest, runPrefix, sha)); + } + + if(!catalog.where(BackupsTables.RUN, "runKey", manifest.runKey()).isEmpty()) + { + return (new Inspection(manifestKey, Inspection.EXISTS, List.of(), manifest, runPrefix, sha)); + } + + checkParts(storage, manifest, runPrefix, problems); + return (new Inspection(manifestKey, problems.isEmpty() ? Inspection.NEW : Inspection.INVALID, problems, manifest, runPrefix, sha)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private Inspection invalid(String manifestKey, String runPrefix, String problem) + { + return (new Inspection(manifestKey, Inspection.INVALID, List.of(problem), null, runPrefix, null)); + } + + + + /******************************************************************************* + ** Check checksums.sha256 against the manifest, and that each part exists. + *******************************************************************************/ + private void checkParts(BackupStorage storage, BackupManifest manifest, String runPrefix, List problems) + { + Map checksums = readChecksums(storage, runPrefix); + if(checksums == null) + { + problems.add("checksums.sha256 is missing or unreadable"); + return; + } + + for(BackupManifest.TableEntry table : manifest.tables() == null ? List.of() : manifest.tables()) + { + for(BackupManifest.Part part : table.parts()) + { + String listed = checksums.get(part.key()); + if(listed == null || !listed.equals(part.sha256Stored())) + { + problems.add("The checksum of " + part.key() + " does not match the manifest"); + } + else if(storage.head(runPrefix + part.key()).map(StoredObjectInfo::size).orElse(-1L) != part.bytesStored()) + { + problems.add("The part " + part.key() + " is missing or has the wrong size"); + } + } + } + } + + + + /******************************************************************************* + ** Create the catalog rows of one valid manifest. Returns the new run id. + *******************************************************************************/ + public Integer importRun(String bucketAlias, BackupStorage storage, Inspection inspection) throws QException + { + BackupManifest manifest = inspection.manifest(); + String runPrefix = inspection.runPrefix(); + List tables = manifest.tables() == null ? List.of() : manifest.tables(); + + Instant started = parse(manifest.startedAt()); + Instant ended = parse(manifest.finishedAt()); + Integer planId = null; + if(manifest.plan() != null && manifest.plan().planKey() != null) + { + List plans = catalog.where(BackupsTables.PLAN, "planKey", manifest.plan().planKey()); + planId = plans.isEmpty() ? null : plans.get(0).getValueInteger("id"); + } + + Map row = new LinkedHashMap<>(); + row.put("backupPlanId", planId); + row.put("runKey", manifest.runKey()); + row.put("trigger", "IMPORTED"); + row.put("requestedByUserId", manifest.requestedBy()); + row.put("status", manifest.warnings() == null || manifest.warnings().isEmpty() ? "SUCCEEDED" : "SUCCEEDED_WITH_WARNINGS"); + row.put("startDate", started); + row.put("endDate", ended); + row.put("durationMs", started != null && ended != null ? ended.toEpochMilli() - started.toEpochMilli() : null); + row.put("tablesPlanned", tables.size()); + row.put("tablesDone", tables.size()); + row.put("rowsTotal", manifest.totals() == null ? null : manifest.totals().rows()); + row.put("bytesStored", manifest.totals() == null ? null : manifest.totals().bytesStored()); + row.put("consistencyAchieved", manifest.consistency() == null ? null : manifest.consistency().achieved()); + row.put("storageBucketAlias", bucketAlias); + row.put("runPrefix", runPrefix); + row.put("manifestKey", inspection.manifestKey()); + row.put("manifestSha256", inspection.manifestSha256()); + row.put("securityScope", manifest.securityScope()); + row.put("qqqVersion", manifest.instance() == null ? null : manifest.instance().qqqVersion()); + row.put("appVersion", manifest.instance() == null ? null : manifest.instance().appVersion()); + row.put("instanceName", manifest.instance() == null ? null : manifest.instance().name()); + row.put("schemaFingerprint", TableFingerprints.combine(tables.stream().map(BackupManifest.TableEntry::fingerprint).toList())); + boolean encrypted = manifest.encryption() != null && manifest.encryption().mode() != null && !"NONE".equals(manifest.encryption().mode()); + row.put("encryptionKeyRef", manifest.encryption() == null ? null : manifest.encryption().keyRef()); + row.put("encryptionKeyFingerprint", manifest.encryption() == null ? null : manifest.encryption().keyFingerprint()); + row.put("verificationStatus", "UNVERIFIED"); + row.put("warningCount", manifest.warnings() == null ? 0 : manifest.warnings().size()); + if(manifest.files() != null) + { + row.put("filesReferenced", (int) manifest.files().objects()); + row.put("filesMissing", (int) manifest.files().missing()); + row.put("fileBytesReferenced", manifest.files().bytes()); + } + Integer runId = catalog.insert(BackupsTables.RUN, row); + + Map checksums = readChecksums(storage, runPrefix); + if(checksums == null) + { + checksums = new HashMap<>(); + } + for(BackupManifest.TableEntry table : tables) + { + for(int i = 0; i < table.parts().size(); i++) + { + BackupManifest.Part part = table.parts().get(i); + Matcher matcher = PART_NUMBER.matcher(part.key()); + int number = matcher.find() ? Integer.parseInt(matcher.group(1)) : i + 1; + boolean last = i == table.parts().size() - 1; + catalog.insertArtifact(runId, "TABLE_DATA", table.name(), number, part.key(), part.format(), part.compression(), part.encrypted(), part.rows(), + part.bytesRaw(), part.bytesStored(), part.sha256Stored(), part.sha256Raw(), last ? table.rowsHash() : null, null, null, table.fingerprint()); + } + } + if(manifest.files() != null && manifest.files().indexKeys() != null) + { + int number = 1; + for(String key : manifest.files().indexKeys()) + { + long size = storage.head(runPrefix + key).map(StoredObjectInfo::size).orElse(0L); + catalog.insertArtifact(runId, "FILE_OBJECTS_INDEX", null, number++, key, "JSONL", "GZIP", encrypted, 0, 0, size, checksums.get(key), null, null, null, null, null); + } + } + artifactOfKey(storage, runPrefix, "checksums.sha256", "CHECKSUMS", "TEXT", runId, null); + artifactOfKey(storage, runPrefix, "manifest.json", "MANIFEST", "JSON", runId, inspection.manifestSha256()); + + registerFileStore(bucketAlias, storage, runPrefix, runId, manifest, encrypted); + return (runId); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void artifactOfKey(BackupStorage storage, String runPrefix, String key, String type, String format, Integer runId, String knownSha) throws QException + { + long size = storage.head(runPrefix + key).map(StoredObjectInfo::size).orElse(0L); + String sha = knownSha; + if(sha == null) + { + try + { + sha = HashingOutputStream.sha256Hex(BackupSource.readAll(storage.openRead(runPrefix + key))); + } + catch(IOException | RuntimeException e) + { + sha = null; + } + } + catalog.insertArtifact(runId, type, null, 1, key, format, "NONE", false, 0, size, size, sha, sha, null, null, null, null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private Map readChecksums(BackupStorage storage, String runPrefix) + { + Map checksums = new HashMap<>(); + try + { + for(String line : new String(BackupSource.readAll(storage.openRead(runPrefix + "checksums.sha256")), StandardCharsets.UTF_8).split("\n")) + { + String[] parts = line.split(" ", 2); + if(parts.length == 2) + { + checksums.put(parts[1].trim(), parts[0].trim()); + } + } + } + catch(IOException | RuntimeException e) + { + return (null); + } + return (checksums); + } + + + + /******************************************************************************* + ** Re-register the content-addressed files of the instance, so the next backup + ** deduplicates against them again. + *******************************************************************************/ + private void registerFileStore(String bucketAlias, BackupStorage storage, String runPrefix, Integer runId, BackupManifest manifest, boolean encrypted) + { + String[] segments = runPrefix.split("/"); + if(segments.length < 6) + { + return; + } + String instancePrefix = String.join("/", java.util.Arrays.copyOf(segments, segments.length - 5)) + "/"; + String fingerprint = manifest.encryption() == null ? null : manifest.encryption().keyFingerprint(); + var registry = catalog.fileStoreRegistry(bucketAlias, runId, fingerprint); + for(StoredObjectInfo info : storage.list(instancePrefix + "_files/sha256/")) + { + String name = info.key().substring(info.key().lastIndexOf('/') + 1); + int dot = name.indexOf('.'); + String sha = dot < 0 ? name : name.substring(0, dot); + if(SHA_KEY.matcher(sha).matches() && !registry.isKnown(sha)) + { + registry.register(sha, info.size(), null, info.key(), encrypted); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static Instant parse(String text) + { + try + { + return (text == null ? null : Instant.parse(text)); + } + catch(RuntimeException e) + { + return (null); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/restore/RestoreEngine.java b/src/main/java/com/kingsrook/qbits/backups/restore/RestoreEngine.java new file mode 100644 index 0000000..fcf839f --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/restore/RestoreEngine.java @@ -0,0 +1,651 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.restore; + + +import java.io.IOException; +import java.io.InputStream; +import java.io.Serializable; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qbits.backups.BackupsQBitConfig; +import com.kingsrook.qbits.backups.BackupsRuntime; +import com.kingsrook.qbits.backups.codec.HashingOutputStream; +import com.kingsrook.qbits.backups.manifest.BackupManifest; +import com.kingsrook.qbits.backups.model.BackupsTables; +import com.kingsrook.qbits.backups.permissions.BackupsPermissionChecker; +import com.kingsrook.qbits.backups.permissions.BackupsPermissions; +import com.kingsrook.qbits.backups.run.BackupCatalog; +import com.kingsrook.qbits.backups.run.BackupEngine; +import com.kingsrook.qbits.backups.run.BackupRunFactory; +import com.kingsrook.qbits.backups.run.BackupRunRequest; +import com.kingsrook.qbits.backups.run.BackupRunResult; +import com.kingsrook.qbits.backups.scope.PlanSettings; +import com.kingsrook.qbits.backups.scope.ScopeResolver; +import com.kingsrook.qbits.backups.scope.TableRole; +import com.kingsrook.qbits.backups.scope.TableRoles; +import com.kingsrook.qbits.backups.storage.BackupStorage; +import com.kingsrook.qbits.backups.storage.WriteOptions; +import com.kingsrook.qqq.backend.core.actions.audits.AuditAction; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Restores from a backup run (spec 9.3 to 9.5). Two entry points share the same + ** table pass: review() reads and compares without writing anything, and + ** execute() takes the pre-restore backup, writes in restore order, resyncs key + ** sequences, brings files back (re-hashing each against the index), writes one + ** summary audit per table, and records backups_restore_run and + ** backups_restore_run_table rows. + ** + ** Safety rails (spec 9.4) are enforced here, not only in the screens: the caller + ** needs backups.restore (REPLACE also backups.admin and the typed confirmation), + ** SECRET and BACKUPS tables are never restored, EPHEMERAL tables only when named, + ** and nothing restores while a backup of the same tables is running. + *******************************************************************************/ +public class RestoreEngine +{ + private static final QLogger LOG = QLogger.getLogger(RestoreEngine.class); + + private static final long RUNNING_HEARTBEAT_MINUTES = 10; + + private final BackupCatalog catalog = new BackupCatalog(); + + + + /******************************************************************************* + ** What a request resolves to: the source, the tables in restore order and the + ** table each is written to, with every rail violation as an error. + *******************************************************************************/ + public record Plan(BackupSource source, List order, Map targets, List errors, List warnings, List notes) + { + } + + + + /******************************************************************************* + ** Resolve a request and check every safety rail, without touching any data. + *******************************************************************************/ + public Plan plan(RestoreRequest request) + { + List errors = new ArrayList<>(); + List warnings = new ArrayList<>(); + List notes = new ArrayList<>(); + List order = new ArrayList<>(); + Map targets = new LinkedHashMap<>(); + QInstance instance = QContext.getQInstance(); + + if(!BackupsPermissionChecker.has(BackupsPermissions.RESTORE)) + { + errors.add("Restoring needs the " + BackupsPermissions.RESTORE + " permission."); + } + boolean replace = RestoreRequest.MODE_REPLACE.equals(request.getMode()); + if(!List.of(RestoreRequest.MODE_INSERT_NEW, RestoreRequest.MODE_UPSERT, RestoreRequest.MODE_REPLACE).contains(request.getMode())) + { + errors.add("Unknown restore mode " + request.getMode() + "."); + } + if(!List.of(RestoreRequest.IDS_PRESERVE, RestoreRequest.IDS_REMAP).contains(request.getIdStrategy())) + { + errors.add("Unknown id strategy " + request.getIdStrategy() + "."); + } + if(replace && !BackupsPermissionChecker.has(BackupsPermissions.ADMIN)) + { + errors.add("A REPLACE restore needs the " + BackupsPermissions.ADMIN + " permission as well."); + } + if(replace && RestoreRequest.IDS_REMAP.equals(request.getIdStrategy())) + { + errors.add("REPLACE cannot be combined with new ids: use PRESERVE ids."); + } + + BackupSource source = null; + try + { + source = BackupSource.open(request.getSourceRunId()); + } + catch(QException e) + { + errors.add(e.getMessage()); + return (new Plan(null, order, targets, errors, warnings, notes)); + } + + boolean explicit = !request.getTables().isEmpty(); + List names = new ArrayList<>(); + if(explicit) + { + for(String name : request.getTables()) + { + if(source.getManifest().table(name) == null) + { + errors.add("The table " + name + " is not in this backup."); + } + else + { + names.add(name); + } + } + } + else + { + source.getManifest().tables().forEach(t -> names.add(t.name())); + } + + if(request.getTargetTable() != null && names.size() != 1) + { + errors.add("A target table can only be chosen when exactly one table is restored."); + } + + for(String name : names) + { + String targetName = request.getTargetTable() != null && names.size() == 1 ? request.getTargetTable() : name; + QTableMetaData target = instance.getTable(targetName); + if(target == null) + { + errors.add("The table " + targetName + " does not exist in this instance."); + continue; + } + TableRole role = TableRoles.roleOf(instance, target); + if(role == TableRole.SECRET || role == TableRole.BACKUPS) + { + errors.add("The table " + targetName + " is a " + role + " table, and restores never touch those."); + continue; + } + if(role == TableRole.EPHEMERAL) + { + if(!explicit) + { + notes.add("The table " + targetName + " holds temporary data and is left out unless you choose it."); + continue; + } + warnings.add("The table " + targetName + " holds temporary data; you chose to restore it anyway."); + } + targets.put(name, targetName); + } + + List manifestOrder = source.getManifest().restoreOrder() == null ? List.of() : source.getManifest().restoreOrder(); + for(String name : manifestOrder) + { + if(targets.containsKey(name)) + { + order.add(name); + } + } + for(String name : targets.keySet()) + { + if(!order.contains(name)) + { + order.add(name); + } + } + + if(order.isEmpty() && errors.isEmpty()) + { + errors.add("There are no tables to restore."); + } + + if(replace && errors.isEmpty()) + { + String expected = RestoreRequest.expectedConfirmation(order); + if(request.getConfirmationText() == null || !expected.equals(request.getConfirmationText().trim())) + { + errors.add("To confirm a REPLACE, type " + expected + "."); + } + } + + String running = runningBackupFor(order); + if(running != null) + { + errors.add(running); + } + + return (new Plan(source, order, targets, errors, warnings, notes)); + } + + + + /******************************************************************************* + ** A message when a backup that covers any of these tables is running, else null. + *******************************************************************************/ + private String runningBackupFor(List tables) + { + try + { + Instant cutoff = BackupsRuntime.getClock().instant().minusSeconds(RUNNING_HEARTBEAT_MINUTES * 60); + for(QRecord run : catalog.where(BackupsTables.RUN, "status", "RUNNING")) + { + Instant heartbeat = run.getValueInstant("heartbeatDate"); + if(heartbeat == null || !heartbeat.isAfter(cutoff)) + { + continue; + } + if(coversAny(run, tables)) + { + return ("A backup of these tables is running (" + run.getValueString("runKey") + "). Wait for it to finish, then restore."); + } + } + return (null); + } + catch(QException e) + { + return ("Could not check for running backups: " + e.getMessage()); + } + } + + + + /******************************************************************************* + ** Whether a running backup covers any of the tables (a run without a plan is + ** assumed to). + *******************************************************************************/ + private boolean coversAny(QRecord run, List tables) throws QException + { + Integer planId = run.getValueInteger("backupPlanId"); + if(planId == null) + { + return (true); + } + QRecord plan = catalog.get(BackupsTables.PLAN, planId).orElse(null); + if(plan == null) + { + return (true); + } + Set covered = new HashSet<>(BackupRunFactory.resolveScope(plan).names()); + return (tables.stream().anyMatch(covered::contains)); + } + + + + /******************************************************************************* + ** Review: everything the restore would do, with nothing written. + *******************************************************************************/ + public RestoreOutcome review(RestoreRequest request) throws QException + { + Plan plan = plan(request); + if(!plan.errors().isEmpty()) + { + RestoreOutcome outcome = new RestoreOutcome(); + outcome.getErrors().addAll(plan.errors()); + return (outcome); + } + + RestoreOutcome outcome = new RestorePass(request, plan.source(), true, null, plan.order(), plan.targets()).run(); + outcome.getWarnings().addAll(plan.warnings()); + outcome.getNotes().addAll(plan.notes()); + if(!request.getPreRestoreBackup()) + { + outcome.getWarnings().add("The safety backup before the restore is turned off."); + } + if(RestoreRequest.IDS_PRESERVE.equals(request.getIdStrategy()) && outcome.sum(t -> t.getSequenceNeeded() ? 1 : 0) > 0 && !request.getWriter().canResyncSequences()) + { + outcome.getWarnings().add("Key sequences cannot be resynced with this version of QQQ (it needs the sequence resync hook), so on databases with generated keys the next generated id may collide. Resync them after the restore."); + } + if(!request.getWriter().usesCoreRestoreFlag()) + { + outcome.getNotes().add("This version of QQQ has no restore mode for writes, so rows are written with the standard actions: record security and validation apply, and modified dates may be set to now."); + } + return (outcome); + } + + + + /******************************************************************************* + ** Execute the restore. Throws when a rail refuses it; otherwise returns the + ** outcome (also recorded in the restore run rows). + *******************************************************************************/ + public RestoreOutcome execute(RestoreRequest request) throws QException + { + Plan plan = plan(request); + if(!plan.errors().isEmpty()) + { + throw (new QException(String.join(" ", plan.errors()))); + } + + BackupsQBitConfig config = BackupsRuntime.getConfig(); + Instant start = BackupsRuntime.getClock().instant(); + + Map row = new LinkedHashMap<>(); + row.put("sourceBackupRunId", request.getSourceRunId()); + row.put("requestedByUserId", request.getRequestedBy()); + row.put("startDate", start); + row.put("status", "RUNNING"); + row.put("mode", request.getMode()); + row.put("idStrategy", request.getIdStrategy()); + row.put("matchOn", request.getMatchOn()); + row.put("targetInstanceName", config.getInstanceName() != null ? config.getInstanceName() : "qqq"); + row.put("scopeSummary", scopeSummary(request, plan)); + row.put("automationsSuppressed", request.getSuppressAutomations()); + row.put("esbSuppressed", !request.getPublishEvents()); + row.put("confirmationText", RestoreRequest.MODE_REPLACE.equals(request.getMode()) ? request.getConfirmationText() : null); + Integer restoreRunId = catalog.insert(BackupsTables.RESTORE_RUN, row); + + RestoreOutcome outcome = null; + try + { + Integer preRestoreRunId = null; + if(request.getPreRestoreBackup()) + { + preRestoreRunId = preRestoreBackup(request, plan); + catalog.update(BackupsTables.RESTORE_RUN, restoreRunId, Map.of("preRestoreBackupRunId", preRestoreRunId)); + } + + RestorePass pass = new RestorePass(request, plan.source(), false, restoreRunId, plan.order(), plan.targets()); + outcome = pass.run(); + outcome.setRestoreRunId(restoreRunId); + outcome.setPreRestoreRunId(preRestoreRunId); + outcome.getWarnings().addAll(plan.warnings()); + outcome.getNotes().addAll(plan.notes()); + + resyncSequences(request, plan, outcome); + restoreFiles(request, plan, pass, outcome, restoreRunId); + recordResults(request, plan, outcome, restoreRunId, start); + deleteIdMap(restoreRunId); + return (outcome); + } + catch(QException | RuntimeException e) + { + LOG.warn("Restore failed", e, logPair("restoreRunId", restoreRunId)); + String message = e.getClass().getSimpleName() + ": " + e.getMessage(); + try + { + Map failed = new LinkedHashMap<>(); + failed.put("status", "FAILED"); + failed.put("endDate", BackupsRuntime.getClock().instant()); + failed.put("errorMessage", message.length() > 2000 ? message.substring(0, 2000) : message); + catalog.update(BackupsTables.RESTORE_RUN, restoreRunId, failed); + } + catch(QException inner) + { + LOG.warn("Could not record the failed restore", inner); + } + throw (e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String scopeSummary(RestoreRequest request, Plan plan) + { + StringBuilder sb = new StringBuilder(String.join(", ", plan.order())); + if(!request.getFilters().isEmpty()) + { + sb.append(" | filters on ").append(String.join(", ", request.getFilters().keySet())); + } + if(request.getTargetTable() != null) + { + sb.append(" | into ").append(request.getTargetTable()); + } + if(!request.getPreRestoreBackup()) + { + sb.append(" | no pre-restore backup"); + } + return (sb.toString()); + } + + + + /******************************************************************************* + ** Back up the target scope (with its files) before writing anything. + *******************************************************************************/ + private Integer preRestoreBackup(RestoreRequest request, Plan plan) throws QException + { + BackupSource source = plan.source(); + BackupsQBitConfig config = BackupsRuntime.getConfig(); + String alias = source.getRun().getValueString("storageBucketAlias"); + + PlanSettings settings = PlanSettings.adHoc().with("scopeType", "TABLES").with("planKey", "pre-restore"); + Map original = source.getManifest().plan() == null ? Map.of() : source.getManifest().plan().settings(); + for(String key : List.of("prefix", "format", "compression", "pageSize", "targetPartBytes", "fileObjectPolicy", "heavyFieldPolicy", "consistency")) + { + if(original != null && original.get(key) != null) + { + settings.with(key, original.get(key)); + } + } + boolean encrypts = source.getTransform().encrypts(); + if(encrypts) + { + settings.with("encryptionMode", "CLIENT_AES256_GCM").with("sensitiveFieldPolicy", "INCLUDE_ENCRYPTED_ONLY"); + } + + List rules = new ArrayList<>(); + for(String name : plan.order()) + { + rules.add(new ScopeResolver.ScopeRule(true, "TABLE", plan.targets().get(name), request.filterFor(name), false, null)); + } + + BackupRunRequest backup = new BackupRunRequest() + .withSettings(settings) + .withScope(new ScopeResolver(QContext.getQInstance()).resolve(settings, rules)) + .withTrigger("PRE_RESTORE") + .withRequestedBy(request.getRequestedBy()) + .withBucketAlias(alias) + .withStorage(source.getStorage()) + .withSourceResolver(BackupsRuntime::storageFor) + .withInstanceName(config.getInstanceName() != null ? config.getInstanceName() : "qqq") + .withPlanName("Pre-restore backup") + .withClock(BackupsRuntime.getClock()); + BackupManifest.EncryptionInfo encryption = source.getManifest().encryption(); + backup.withTransform(source.getTransform(), encrypts && encryption != null ? encryption.keyRef() : null, encrypts && encryption != null ? encryption.keyFingerprint() : null); + + BackupRunResult result = new BackupEngine().execute(backup); + if(!result.isUsable()) + { + throw (new QException("The backup taken before the restore failed, so nothing was restored: " + result.getErrorMessage())); + } + return (result.getRunId()); + } + + + + /******************************************************************************* + ** Advance the key sequences of the tables written with their original ids. + *******************************************************************************/ + private void resyncSequences(RestoreRequest request, Plan plan, RestoreOutcome outcome) throws QException + { + List tables = new ArrayList<>(); + for(RestoreOutcome.TableOutcome table : outcome.getTables().values()) + { + if(table.getSequenceNeeded()) + { + tables.add(QContext.getQInstance().getTable(table.getTargetName())); + } + } + if(tables.isEmpty()) + { + return; + } + if(!request.getWriter().canResyncSequences()) + { + outcome.getWarnings().add("Key sequences were not resynced: this version of QQQ has no resync hook."); + return; + } + List resynced = request.getWriter().resyncSequences(tables); + outcome.setSequencesResynced(resynced.size()); + for(RestoreOutcome.TableOutcome table : outcome.getTables().values()) + { + table.setSequenceResynced(resynced.contains(table.getTargetName())); + } + } + + + + /******************************************************************************* + ** Copy files back from the content-addressed store, re-hashing each against the + ** index. A conflicting file (different content at the target) is only replaced + ** by a REPLACE restore. + *******************************************************************************/ + private void restoreFiles(RestoreRequest request, Plan plan, RestorePass pass, RestoreOutcome outcome, Integer restoreRunId) + { + for(RestorePass.FileWork work : pass.getFileWork()) + { + if(work.conflict() && !RestoreRequest.MODE_REPLACE.equals(request.getMode())) + { + continue; + } + RestoreOutcome.TableOutcome table = outcome.getTables().get(work.table()); + try + { + copyFileBack(plan.source(), work, restoreRunId); + table.addFilesRestored(1); + table.addFileBytesRestored(Math.max(0, work.size())); + } + catch(IOException | RuntimeException e) + { + outcome.addFilesFailed(1); + table.getProblems().add("The file " + work.targetKey() + " was not restored: " + e.getMessage()); + } + } + } + + + + /******************************************************************************* + ** Write one file to a temporary key, check its SHA-256 against the index, and + ** only then put it in place. + *******************************************************************************/ + private void copyFileBack(BackupSource source, RestorePass.FileWork work, Integer restoreRunId) throws IOException + { + BackupStorage target = BackupsRuntime.storageFor(work.targetAlias()); + String temp = work.targetKey() + ".qqq-restore-" + restoreRunId; + + String observed; + try(InputStream in = source.getTransform().unwrap(source.getStorage().openRead(work.casKey())); + HashingOutputStream out = new HashingOutputStream(target.openWrite(temp, WriteOptions.create().withMetadata("qqq-restore-run", String.valueOf(restoreRunId))))) + { + in.transferTo(out); + out.close(); + observed = out.getSha256(); + } + + if(!work.sha256().equals(observed)) + { + target.deletePermanent(temp); + throw (new IOException("checksum mismatch after reading the stored copy")); + } + target.copy(temp, work.targetKey()); + target.deletePermanent(temp); + } + + + + /******************************************************************************* + ** One summary audit per table, the per-table rows, and the totals on the run. + *******************************************************************************/ + private void recordResults(RestoreRequest request, Plan plan, RestoreOutcome outcome, Integer restoreRunId, Instant start) throws QException + { + String runKey = plan.source().getRun().getValueString("runKey"); + for(RestoreOutcome.TableOutcome t : outcome.getTables().values()) + { + Map row = new LinkedHashMap<>(); + row.put("backupRestoreRunId", restoreRunId); + row.put("tableName", t.getName()); + row.put("targetTableName", t.getTargetName()); + row.put("rowsInserted", t.getInserted()); + row.put("rowsUpdated", t.getUpdated()); + row.put("rowsUnchanged", t.getUnchanged()); + row.put("rowsDeleted", t.getDeleted()); + row.put("rowsSkipped", t.getSkipped() + t.getConflicts()); + row.put("rowsErrored", t.getErrored()); + row.put("schemaStatus", t.getSchemaStatus()); + row.put("unmappedFields", String.join(", ", t.getUnmappedFields())); + row.put("sequenceResynced", t.getSequenceResynced()); + row.put("status", t.getStatus()); + catalog.insert(BackupsTables.RESTORE_RUN_TABLE, row); + + if(!"SKIPPED".equals(t.getStatus())) + { + summaryAudit(t, runKey, restoreRunId); + } + } + + boolean warnings = outcome.sum(t -> t.getErrored()) > 0 || outcome.getFilesFailed() > 0 || !outcome.getWarnings().isEmpty() + || outcome.getTables().values().stream().anyMatch(t -> "SKIPPED".equals(t.getStatus())); + + Map totals = new LinkedHashMap<>(); + totals.put("status", warnings ? "SUCCEEDED_WITH_WARNINGS" : "SUCCEEDED"); + totals.put("endDate", BackupsRuntime.getClock().instant()); + totals.put("rowsInserted", outcome.sum(RestoreOutcome.TableOutcome::getInserted)); + totals.put("rowsUpdated", outcome.sum(RestoreOutcome.TableOutcome::getUpdated)); + totals.put("rowsUnchanged", outcome.sum(RestoreOutcome.TableOutcome::getUnchanged)); + totals.put("rowsDeleted", outcome.sum(RestoreOutcome.TableOutcome::getDeleted)); + totals.put("rowsSkipped", outcome.sum(t -> t.getSkipped() + t.getConflicts())); + totals.put("rowsErrored", outcome.sum(RestoreOutcome.TableOutcome::getErrored)); + totals.put("filesRestored", (int) outcome.sum(t -> t.getFilesRestored())); + totals.put("filesAlreadyPresent", (int) outcome.sum(t -> t.getFilesPresent())); + totals.put("filesConflicting", (int) outcome.sum(t -> t.getFilesConflicting())); + totals.put("filesMissingFromBackup", (int) outcome.sum(t -> t.getFilesMissing())); + totals.put("fileBytesRestored", outcome.sum(RestoreOutcome.TableOutcome::getFileBytesRestored)); + totals.put("sequencesResynced", outcome.getSequencesResynced()); + catalog.update(BackupsTables.RESTORE_RUN, restoreRunId, totals); + } + + + + /******************************************************************************* + ** The one audit entry for a table in a restore (replaces per-record audit). + *******************************************************************************/ + private void summaryAudit(RestoreOutcome.TableOutcome t, String runKey, Integer restoreRunId) + { + try + { + AuditAction.execute(t.getTargetName(), null, Map.of(), "Restore " + restoreRunId + " from backup " + runKey + ": " + + t.getInserted() + " inserted, " + t.getUpdated() + " updated, " + t.getDeleted() + " deleted, " + t.getUnchanged() + " unchanged"); + } + catch(RuntimeException e) + { + LOG.warn("Could not write the restore summary audit", e, logPair("table", t.getTargetName())); + } + } + + + + /******************************************************************************* + ** The id map rows are only needed while the restore runs. + *******************************************************************************/ + private void deleteIdMap(Integer restoreRunId) + { + try + { + DeleteInput input = new DeleteInput(BackupsTables.ID_MAP); + input.setQueryFilter(new QQueryFilter(new QFilterCriteria("restoreRunId", QCriteriaOperator.EQUALS, restoreRunId))); + input.withPermissionBypass("backups restore id map"); + new DeleteAction().execute(input); + } + catch(QException | RuntimeException e) + { + LOG.warn("Could not clear the id map of a restore", e, logPair("restoreRunId", restoreRunId)); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/restore/RestoreOutcome.java b/src/main/java/com/kingsrook/qbits/backups/restore/RestoreOutcome.java new file mode 100644 index 0000000..45dee40 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/restore/RestoreOutcome.java @@ -0,0 +1,710 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.restore; + + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + + +/******************************************************************************* + ** What a restore pass found (Review) or did (Execute): the per-table counts, the + ** schema comparison, the sample diffs and the file counts. Review and Execute + ** share one algorithm, so the Review numbers are the numbers the restore produces. + ** Nothing here ever holds a sensitive value: sample diffs are masked when built. + *******************************************************************************/ +public class RestoreOutcome +{ + private final Map tables = new LinkedHashMap<>(); + private final List warnings = new ArrayList<>(); + private final List errors = new ArrayList<>(); + private final List notes = new ArrayList<>(); + + private Integer preRestoreRunId; + private Integer restoreRunId; + private int sequencesResynced; + private int filesFailed; + + + + /******************************************************************************* + ** One changed record, as masked "field: old -> new" texts. + *******************************************************************************/ + public record Sample(String key, List changes) + { + } + + + + /******************************************************************************* + ** The result for one table. + *******************************************************************************/ + public static class TableOutcome + { + private final String name; + private final String targetName; + + private String status = "PENDING"; + private String schemaStatus = "MATCH"; + private final List schemaNotes = new ArrayList<>(); + private final List unmappedFields = new ArrayList<>(); + private final List unmappedReferences = new ArrayList<>(); + private final List problems = new ArrayList<>(); + private final List samples = new ArrayList<>(); + private long rows; + private long inserted; + private long updated; + private long unchanged; + private long skipped; + private long deleted; + private long conflicts; + private long orphans; + private long errored; + private boolean sequenceNeeded; + private boolean sequenceResynced; + private long updatedSampleTotal; + private int filesToRestore; + private int filesPresent; + private int filesConflicting; + private int filesMissing; + private int filesRestored; + private long fileBytesRestored; + + + + /******************************************************************************* + ** Constructor. + *******************************************************************************/ + public TableOutcome(String name, String targetName) + { + this.name = name; + this.targetName = targetName; + } + + + + /******************************************************************************* + ** Getter for name (the table in the backup) + *******************************************************************************/ + public String getName() + { + return (name); + } + + + + /******************************************************************************* + ** Getter for targetName (the table written) + *******************************************************************************/ + public String getTargetName() + { + return (targetName); + } + + + + /******************************************************************************* + ** Getter for status + *******************************************************************************/ + public String getStatus() + { + return (status); + } + + + + /******************************************************************************* + ** Setter for status + *******************************************************************************/ + public void setStatus(String status) + { + this.status = status; + } + + + + /******************************************************************************* + ** Getter for schemaStatus + *******************************************************************************/ + public String getSchemaStatus() + { + return (schemaStatus); + } + + + + /******************************************************************************* + ** Setter for schemaStatus + *******************************************************************************/ + public void setSchemaStatus(String schemaStatus) + { + this.schemaStatus = schemaStatus; + } + + + + /******************************************************************************* + ** Getter for schemaNotes + *******************************************************************************/ + public List getSchemaNotes() + { + return (schemaNotes); + } + + + + /******************************************************************************* + ** Getter for unmappedFields (in the backup, not in the target) + *******************************************************************************/ + public List getUnmappedFields() + { + return (unmappedFields); + } + + + + /******************************************************************************* + ** Getter for unmappedReferences (id fields that metadata cannot trace) + *******************************************************************************/ + public List getUnmappedReferences() + { + return (unmappedReferences); + } + + + + /******************************************************************************* + ** Getter for problems (error messages, never row values) + *******************************************************************************/ + public List getProblems() + { + return (problems); + } + + + + /******************************************************************************* + ** Getter for samples + *******************************************************************************/ + public List getSamples() + { + return (samples); + } + + + + /******************************************************************************* + ** Getter for rows + *******************************************************************************/ + public long getRows() + { + return (rows); + } + + + + /******************************************************************************* + ** Add rows read from the backup. + *******************************************************************************/ + public void addRows(long n) + { + rows += n; + } + + + + /******************************************************************************* + ** Getter for inserted + *******************************************************************************/ + public long getInserted() + { + return (inserted); + } + + + + /******************************************************************************* + ** Add inserted + *******************************************************************************/ + public void addInserted(long n) + { + inserted += n; + } + + + + /******************************************************************************* + ** Getter for updated + *******************************************************************************/ + public long getUpdated() + { + return (updated); + } + + + + /******************************************************************************* + ** Add updated + *******************************************************************************/ + public void addUpdated(long n) + { + updated += n; + } + + + + /******************************************************************************* + ** Getter for unchanged + *******************************************************************************/ + public long getUnchanged() + { + return (unchanged); + } + + + + /******************************************************************************* + ** Add unchanged + *******************************************************************************/ + public void addUnchanged(long n) + { + unchanged += n; + } + + + + /******************************************************************************* + ** Getter for skipped (INSERT_NEW rows that exist with different content) + *******************************************************************************/ + public long getSkipped() + { + return (skipped); + } + + + + /******************************************************************************* + ** Add skipped + *******************************************************************************/ + public void addSkipped(long n) + { + skipped += n; + } + + + + /******************************************************************************* + ** Getter for deleted + *******************************************************************************/ + public long getDeleted() + { + return (deleted); + } + + + + /******************************************************************************* + ** Add deleted + *******************************************************************************/ + public void addDeleted(long n) + { + deleted += n; + } + + + + /******************************************************************************* + ** Getter for conflicts (rows that collide on a unique key) + *******************************************************************************/ + public long getConflicts() + { + return (conflicts); + } + + + + /******************************************************************************* + ** Add conflicts + *******************************************************************************/ + public void addConflicts(long n) + { + conflicts += n; + } + + + + /******************************************************************************* + ** Getter for orphans (rows whose parent is in neither the backup nor the target) + *******************************************************************************/ + public long getOrphans() + { + return (orphans); + } + + + + /******************************************************************************* + ** Add orphans + *******************************************************************************/ + public void addOrphans(long n) + { + orphans += n; + } + + + + /******************************************************************************* + ** Getter for errored + *******************************************************************************/ + public long getErrored() + { + return (errored); + } + + + + /******************************************************************************* + ** Add errored + *******************************************************************************/ + public void addErrored(long n) + { + errored += n; + } + + + + /******************************************************************************* + ** Getter for sequenceNeeded + *******************************************************************************/ + public boolean getSequenceNeeded() + { + return (sequenceNeeded); + } + + + + /******************************************************************************* + ** Setter for sequenceNeeded + *******************************************************************************/ + public void setSequenceNeeded(boolean sequenceNeeded) + { + this.sequenceNeeded = sequenceNeeded; + } + + + + /******************************************************************************* + ** Getter for sequenceResynced + *******************************************************************************/ + public boolean getSequenceResynced() + { + return (sequenceResynced); + } + + + + /******************************************************************************* + ** Setter for sequenceResynced + *******************************************************************************/ + public void setSequenceResynced(boolean sequenceResynced) + { + this.sequenceResynced = sequenceResynced; + } + + + + /******************************************************************************* + ** Getter for the number of updated records seen (the sample list is capped) + *******************************************************************************/ + public long getUpdatedSampleTotal() + { + return (updatedSampleTotal); + } + + + + /******************************************************************************* + ** Count one more changed record (sampled or not). + *******************************************************************************/ + public void countChangedRecord() + { + updatedSampleTotal++; + } + + + + /******************************************************************************* + ** Getter for filesToRestore + *******************************************************************************/ + public int getFilesToRestore() + { + return (filesToRestore); + } + + + + /******************************************************************************* + ** Add filesToRestore + *******************************************************************************/ + public void addFilesToRestore(int n) + { + filesToRestore += n; + } + + + + /******************************************************************************* + ** Getter for filesPresent + *******************************************************************************/ + public int getFilesPresent() + { + return (filesPresent); + } + + + + /******************************************************************************* + ** Add filesPresent + *******************************************************************************/ + public void addFilesPresent(int n) + { + filesPresent += n; + } + + + + /******************************************************************************* + ** Getter for filesConflicting + *******************************************************************************/ + public int getFilesConflicting() + { + return (filesConflicting); + } + + + + /******************************************************************************* + ** Add filesConflicting + *******************************************************************************/ + public void addFilesConflicting(int n) + { + filesConflicting += n; + } + + + + /******************************************************************************* + ** Getter for filesMissing (missing from the backup) + *******************************************************************************/ + public int getFilesMissing() + { + return (filesMissing); + } + + + + /******************************************************************************* + ** Add filesMissing + *******************************************************************************/ + public void addFilesMissing(int n) + { + filesMissing += n; + } + + + + /******************************************************************************* + ** Getter for filesRestored + *******************************************************************************/ + public int getFilesRestored() + { + return (filesRestored); + } + + + + /******************************************************************************* + ** Add filesRestored + *******************************************************************************/ + public void addFilesRestored(int n) + { + filesRestored += n; + } + + + + /******************************************************************************* + ** Getter for fileBytesRestored + *******************************************************************************/ + public long getFileBytesRestored() + { + return (fileBytesRestored); + } + + + + /******************************************************************************* + ** Add fileBytesRestored + *******************************************************************************/ + public void addFileBytesRestored(long n) + { + fileBytesRestored += n; + } + } + + + + /******************************************************************************* + ** The outcome of one table, created on first use. + *******************************************************************************/ + public TableOutcome table(String name, String targetName) + { + return (tables.computeIfAbsent(name, k -> new TableOutcome(name, targetName))); + } + + + + /******************************************************************************* + ** The tables, in restore order. + *******************************************************************************/ + public Map getTables() + { + return (tables); + } + + + + /******************************************************************************* + ** Run-level warnings. + *******************************************************************************/ + public List getWarnings() + { + return (warnings); + } + + + + /******************************************************************************* + ** Run-level errors (the restore does not run when there are any). + *******************************************************************************/ + public List getErrors() + { + return (errors); + } + + + + /******************************************************************************* + ** Run-level informational notes. + *******************************************************************************/ + public List getNotes() + { + return (notes); + } + + + + /******************************************************************************* + ** Getter for preRestoreRunId + *******************************************************************************/ + public Integer getPreRestoreRunId() + { + return (preRestoreRunId); + } + + + + /******************************************************************************* + ** Setter for preRestoreRunId + *******************************************************************************/ + public void setPreRestoreRunId(Integer preRestoreRunId) + { + this.preRestoreRunId = preRestoreRunId; + } + + + + /******************************************************************************* + ** Getter for restoreRunId + *******************************************************************************/ + public Integer getRestoreRunId() + { + return (restoreRunId); + } + + + + /******************************************************************************* + ** Setter for restoreRunId + *******************************************************************************/ + public void setRestoreRunId(Integer restoreRunId) + { + this.restoreRunId = restoreRunId; + } + + + + /******************************************************************************* + ** Getter for sequencesResynced + *******************************************************************************/ + public int getSequencesResynced() + { + return (sequencesResynced); + } + + + + /******************************************************************************* + ** Setter for sequencesResynced + *******************************************************************************/ + public void setSequencesResynced(int sequencesResynced) + { + this.sequencesResynced = sequencesResynced; + } + + + + /******************************************************************************* + ** Getter for filesFailed + *******************************************************************************/ + public int getFilesFailed() + { + return (filesFailed); + } + + + + /******************************************************************************* + ** Add filesFailed + *******************************************************************************/ + public void addFilesFailed(int n) + { + filesFailed += n; + } + + + + /******************************************************************************* + ** A total over the tables. + *******************************************************************************/ + public long sum(java.util.function.ToLongFunction getter) + { + return (tables.values().stream().mapToLong(getter).sum()); + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/restore/RestorePass.java b/src/main/java/com/kingsrook/qbits/backups/restore/RestorePass.java new file mode 100644 index 0000000..d76eb72 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/restore/RestorePass.java @@ -0,0 +1,1304 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.restore; + + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.Serializable; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import com.fasterxml.jackson.databind.JsonNode; +import com.kingsrook.qbits.backups.BackupsRuntime; +import com.kingsrook.qbits.backups.codec.FieldValueCodec; +import com.kingsrook.qbits.backups.codec.HashingOutputStream; +import com.kingsrook.qbits.backups.files.FileReference; +import com.kingsrook.qbits.backups.files.FileReferenceProvider; +import com.kingsrook.qbits.backups.manifest.BackupManifest; +import com.kingsrook.qbits.backups.manifest.TableFingerprints; +import com.kingsrook.qbits.backups.model.BackupsTables; +import com.kingsrook.qbits.backups.storage.BackupStorage; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; + + +/******************************************************************************* + ** One pass of a restore over the selected tables, in restore order (spec 9.3). + ** The same algorithm backs Review (dryRun: nothing is written) and Execute, so + ** the Review numbers are exactly what the restore does. + ** + ** For each table the backup rows are streamed in pages and compared to the target + ** rows found by match key. Rows are classified as insert, update, unchanged or + ** skipped; unique-key conflicts and orphaned references are counted; REPLACE also + ** finds the target rows that are not in the backup. When writing, each page is its + ** own transaction (REPLACE uses one per table so it is never left half empty), + ** and the files the rows reference are collected for the file restore. + ** + ** Memory is O(page), plus the key sets of the tables that are REPLACEd or are + ** parents of the restored tables (keys only, never rows). + *******************************************************************************/ +public class RestorePass +{ + private static final int MAX_SAMPLES = 20; + private static final int MAX_TEXT = 60; + private static final int MAX_PROBLEMS = 5; + private static final char SEP = '\u0001'; + + private final RestoreRequest request; + private final BackupSource source; + private final boolean dryRun; + private final Integer restoreRunId; + private final List order; + private final Map targets; + private final QInstance qInstance = QContext.getQInstance(); + + private final RestoreOutcome outcome = new RestoreOutcome(); + private final Map> pkKeySets = new HashMap<>(); + private final Map> idMaps = new HashMap<>(); + private final List fileWork = new ArrayList<>(); + private final Set seenFiles = new HashSet<>(); + private final Map> retry = new LinkedHashMap<>(); + private Map fileIndex = Map.of(); + + + + /******************************************************************************* + ** A row waiting to be inserted, with the primary key it had in the backup. + *******************************************************************************/ + record Pending(QRecord record, String oldPkText) + { + } + + + + /******************************************************************************* + ** A file the restore will copy back from the content-addressed store. + *******************************************************************************/ + public record FileWork(String table, String targetAlias, String targetKey, String sha256, String casKey, long size, boolean conflict) + { + } + + + + /******************************************************************************* + ** Context of the table being restored. + *******************************************************************************/ + private static class TableContext + { + private QTableMetaData target; + private BackupManifest.TableEntry entry; + private RestoreOutcome.TableOutcome outcome; + private String name; + private String pk; + private List matchFields; + private boolean matchIsPk; + private boolean remap; + private Map foreignKeys; + private Set deletePkTexts = new HashSet<>(); + private QBackendTransaction tableTransaction; + } + + + + /******************************************************************************* + ** Wraps a checked failure so it can cross a row sink. + *******************************************************************************/ + private static class Failure extends RuntimeException + { + /******************************************************************************* + ** Constructor. + *******************************************************************************/ + Failure(QException cause) + { + super(cause); + } + } + + + + /******************************************************************************* + ** Constructor. The order is the restore order of the selected tables and the + ** targets map says which table each is written to. The restoreRunId is only + ** needed when writing with REMAP_NEW_IDS (it keys the id map rows). + *******************************************************************************/ + public RestorePass(RestoreRequest request, BackupSource source, boolean dryRun, Integer restoreRunId, List order, Map targets) + { + this.request = request; + this.source = source; + this.dryRun = dryRun; + this.restoreRunId = restoreRunId; + this.order = order; + this.targets = targets; + } + + + + /******************************************************************************* + ** The files collected for restore (valid after run()). + *******************************************************************************/ + public List getFileWork() + { + return (fileWork); + } + + + + /******************************************************************************* + ** Run the pass. + *******************************************************************************/ + public RestoreOutcome run() throws QException + { + if(request.getRestoreFiles() && source.getManifest().files() != null && source.getManifest().files().indexKeys() != null) + { + fileIndex = source.fileIndex(new HashSet<>(order)); + } + + for(String name : order) + { + restoreTable(name); + } + retryPass(); + return (outcome); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void restoreTable(String name) throws QException + { + TableContext c = new TableContext(); + c.name = name; + c.target = qInstance.getTable(targets.get(name)); + c.entry = source.getManifest().table(name); + c.pk = c.target.getPrimaryKeyField(); + c.outcome = outcome.table(name, c.target.getName()); + c.remap = RestoreRequest.IDS_REMAP.equals(request.getIdStrategy()); + analyzeSchema(c); + if("MISMATCH".equals(c.outcome.getSchemaStatus())) + { + c.outcome.setStatus("SKIPPED"); + return; + } + + c.matchFields = matchFields(c); + c.matchIsPk = c.matchFields.equals(List.of(c.pk)); + c.foreignKeys = foreignKeys(c.target); + if(c.remap) + { + c.outcome.getUnmappedReferences().addAll(untraceableReferences(c)); + if(c.matchIsPk) + { + c.outcome.getSchemaNotes().add("REMAP_NEW_IDS matches on the primary key, so every row is inserted as new; match on a unique key to reuse existing rows"); + } + } + + QQueryFilter filter = parseFilter(request.filterFor(name)); + List deletePks = new ArrayList<>(); + if(RestoreRequest.MODE_REPLACE.equals(request.getMode())) + { + findDeletes(c, filter, deletePks); + c.outcome.addDeleted(deletePks.size()); + } + + try + { + if(!dryRun && RestoreRequest.MODE_REPLACE.equals(request.getMode())) + { + c.tableTransaction = QBackendTransaction.openFor(new InsertInput(c.target.getName())); + for(int i = 0; i < deletePks.size(); i += request.getPageSize()) + { + request.getWriter().delete(c.target, deletePks.subList(i, Math.min(deletePks.size(), i + request.getPageSize())), request, c.tableTransaction); + } + } + + List buffer = new ArrayList<>(); + try + { + source.readRows(c.entry, c.target, row -> + { + row.getValues().keySet().removeIf(k -> !c.target.getFields().containsKey(k)); + if(filter != null && !BackendQueryFilterUtils.doesRecordMatch(filter, row)) + { + return; + } + buffer.add(row); + if(buffer.size() >= request.getPageSize()) + { + flush(c, buffer); + } + }); + flush(c, buffer); + } + catch(Failure f) + { + throw ((QException) f.getCause()); + } + + if(c.tableTransaction != null) + { + c.tableTransaction.commit(); + } + } + catch(QException | RuntimeException e) + { + if(c.tableTransaction != null) + { + c.tableTransaction.rollback(); + } + throw (e); + } + finally + { + if(c.tableTransaction != null) + { + c.tableTransaction.close(); + } + } + + boolean keyed = c.target.getField(c.pk).getType() == QFieldType.INTEGER || c.target.getField(c.pk).getType() == QFieldType.LONG; + c.outcome.setSequenceNeeded(keyed && !c.remap && (c.outcome.getInserted() > 0 || RestoreRequest.MODE_REPLACE.equals(request.getMode()))); + c.outcome.setStatus("DONE"); + } + + + + /******************************************************************************* + ** Flush the buffered page (and empty the buffer). + *******************************************************************************/ + private void flush(TableContext c, List buffer) + { + if(buffer.isEmpty()) + { + return; + } + try + { + processPage(c, new ArrayList<>(buffer)); + } + catch(QException e) + { + throw (new Failure(e)); + } + buffer.clear(); + } + + + + /******************************************************************************* + ** Compare the backup schema of the table with the target. + *******************************************************************************/ + @SuppressWarnings("unchecked") + private void analyzeSchema(TableContext c) + { + QInstance instance = QContext.getQInstance(); + String backendType = instance.getBackend(c.target.getBackendName()) == null ? null : instance.getBackend(c.target.getBackendName()).getBackendType(); + RestoreOutcome.TableOutcome o = c.outcome; + + if(c.entry.fingerprint() != null && c.entry.fingerprint().equals(TableFingerprints.of(c.target, backendType))) + { + o.setSchemaStatus("MATCH"); + return; + } + + Map backupFields = c.entry.schema() == null ? Map.of() : (Map) c.entry.schema().getOrDefault("fields", Map.of()); + Map targetFields = (Map) TableFingerprints.descriptor(c.target, backendType).get("fields"); + boolean incompatible = false; + boolean different = false; + + for(Map.Entry field : backupFields.entrySet()) + { + if(!targetFields.containsKey(field.getKey())) + { + o.getUnmappedFields().add(field.getKey()); + o.getSchemaNotes().add("field " + field.getKey() + " is in the backup but not in the target and will be skipped"); + different = true; + continue; + } + Map was = (Map) field.getValue(); + Map now = (Map) targetFields.get(field.getKey()); + if(!String.valueOf(was.get("type")).equals(String.valueOf(now.get("type")))) + { + incompatible = true; + o.getSchemaNotes().add("field " + field.getKey() + " changed type from " + was.get("type") + " to " + now.get("type")); + } + else if(!was.equals(now)) + { + different = true; + o.getSchemaNotes().add("field " + field.getKey() + " changed (length, required, possible values or flags)"); + } + } + + List excluded = c.entry.excludedFields() == null ? List.of() : c.entry.excludedFields().stream().map(BackupManifest.ExcludedField::name).toList(); + for(String field : targetFields.keySet()) + { + if(!backupFields.containsKey(field)) + { + QFieldMetaData now = c.target.getField(field); + different = true; + if(now.getIsRequired() && !excluded.contains(field) && !field.equals(c.target.getPrimaryKeyField())) + { + incompatible = true; + o.getSchemaNotes().add("required field " + field + " is in the target but not in the backup"); + } + else + { + o.getSchemaNotes().add("field " + field + " is in the target but not in the backup and keeps its default"); + } + } + } + + o.setSchemaStatus(incompatible ? "MISMATCH" : different ? "COMPATIBLE" : "MATCH"); + if(incompatible) + { + o.getProblems().add("The table " + c.name + " is not compatible with the backup (see the schema notes), so it is not restored"); + } + } + + + + /******************************************************************************* + ** The fields rows are matched on. + *******************************************************************************/ + private List matchFields(TableContext c) throws QException + { + String matchOn = request.getMatchOn(); + List fields = null; + if(matchOn == null || matchOn.isBlank() || RestoreRequest.MATCH_PRIMARY_KEY.equals(matchOn)) + { + fields = List.of(c.pk); + } + else + { + List wanted = Arrays.stream(matchOn.split("[,+]")).map(String::trim).toList(); + for(UniqueKey key : c.target.getUniqueKeys() == null ? List.of() : c.target.getUniqueKeys()) + { + if(new HashSet<>(key.getFieldNames()).equals(new HashSet<>(wanted)) || matchOn.equals(key.getLabel())) + { + fields = key.getFieldNames(); + break; + } + } + if(fields == null && order.size() > 1) + { + c.outcome.getSchemaNotes().add("it has no unique key " + matchOn + ", so rows are matched on the primary key"); + fields = List.of(c.pk); + } + else if(fields == null) + { + throw (new QException("The table " + c.target.getName() + " has no unique key named " + matchOn)); + } + } + for(String field : fields) + { + if(c.entry.excludedFields() != null && c.entry.excludedFields().stream().anyMatch(f -> f.name().equals(field))) + { + throw (new QException("The match field " + field + " of " + c.name + " was left out of the backup, so rows cannot be matched on it")); + } + } + return (fields); + } + + + + /******************************************************************************* + ** The fields that reference another table: those whose possible value source is a + ** table, and those on the many side of a join. Field name to parent table name. + *******************************************************************************/ + private Map foreignKeys(QTableMetaData table) + { + Map fks = new LinkedHashMap<>(); + for(QFieldMetaData field : table.getFields().values()) + { + if(field.getName().equals(table.getPrimaryKeyField()) || field.getPossibleValueSourceName() == null) + { + continue; + } + QPossibleValueSource pvs = qInstance.getPossibleValueSource(field.getPossibleValueSourceName()); + if(pvs != null && pvs.getType() == QPossibleValueSourceType.TABLE && pvs.getTableName() != null) + { + fks.put(field.getName(), pvs.getTableName()); + } + } + for(QJoinMetaData join : qInstance.getJoins().values()) + { + if(join.getJoinOns() == null || join.getJoinOns().size() != 1) + { + continue; + } + JoinOn on = join.getJoinOns().get(0); + if(join.getType() == JoinType.MANY_TO_ONE && table.getName().equals(join.getLeftTable()) && table.getFields().containsKey(on.getLeftField())) + { + fks.putIfAbsent(on.getLeftField(), join.getRightTable()); + } + else if(join.getType() == JoinType.ONE_TO_MANY && table.getName().equals(join.getRightTable()) && table.getFields().containsKey(on.getRightField())) + { + fks.putIfAbsent(on.getRightField(), join.getLeftTable()); + } + } + return (fks); + } + + + + /******************************************************************************* + ** Id looking fields that metadata cannot trace to a parent table. + *******************************************************************************/ + private List untraceableReferences(TableContext c) + { + List rs = new ArrayList<>(); + for(QFieldMetaData field : c.target.getFields().values()) + { + boolean idLike = field.getName().endsWith("Id") || field.getName().endsWith("ID"); + boolean numeric = field.getType() == QFieldType.INTEGER || field.getType() == QFieldType.LONG; + if(idLike && numeric && !field.getName().equals(c.pk) && !c.foreignKeys.containsKey(field.getName())) + { + rs.add(c.target.getName() + "." + field.getName()); + } + } + return (rs); + } + + + + /******************************************************************************* + ** REPLACE: the target rows (within the filter) that are not in the backup. + *******************************************************************************/ + private void findDeletes(TableContext c, QQueryFilter filter, List deletePks) throws QException + { + Set backupKeys = new HashSet<>(); + try + { + source.readRows(c.entry, c.target, row -> + { + if(filter == null || BackendQueryFilterUtils.doesRecordMatch(filter, row)) + { + backupKeys.add(keyText(c.target, c.matchFields, row)); + } + }); + } + catch(Failure f) + { + throw ((QException) f.getCause()); + } + + Serializable last = null; + while(true) + { + QQueryFilter page = new QQueryFilter(); + if(last != null) + { + page.addCriteria(new QFilterCriteria(c.pk, QCriteriaOperator.GREATER_THAN, List.of(last))); + } + if(filter != null) + { + page.addSubFilter(filter.clone()); + } + page.withOrderBy(new QFilterOrderBy(c.pk, true)); + page.setLimit(request.getPageSize()); + List rows = query(c.target.getName(), page); + for(QRecord row : rows) + { + if(!backupKeys.contains(keyText(c.target, c.matchFields, row))) + { + deletePks.add(row.getValue(c.pk)); + c.deletePkTexts.add(pkText(c, row)); + } + last = row.getValue(c.pk); + } + if(rows.size() < request.getPageSize()) + { + return; + } + } + } + + + + /******************************************************************************* + ** Process one page of backup rows. + *******************************************************************************/ + private void processPage(TableContext c, List page) throws QException + { + RestoreOutcome.TableOutcome o = c.outcome; + o.addRows(page.size()); + + if(request.getRestoreFiles()) + { + collectFiles(c, page); + } + countOrphans(c, page); + + if(c.remap && !dryRun) + { + rewriteReferences(c, page); + } + + Map existing = c.remap && c.matchIsPk ? Map.of() : lookup(c.target, c.matchFields, page); + + List inserts = new ArrayList<>(); + List updates = new ArrayList<>(); + for(QRecord row : page) + { + String oldPk = row.getValue(c.pk) == null ? null : canonical(c.target.getField(c.pk), row.getValue(c.pk)); + QRecord found = existing.get(keyText(c.target, c.matchFields, row)); + if(found == null) + { + inserts.add(new Pending(row, oldPk)); + continue; + } + + if(c.remap && oldPk != null) + { + idMaps.computeIfAbsent(c.name, k -> new HashMap<>()).put(oldPk, found.getValue(c.pk)); + } + + Map changes = diff(c, row, found); + if(changes.isEmpty()) + { + o.addUnchanged(1); + } + else if(RestoreRequest.MODE_INSERT_NEW.equals(request.getMode())) + { + o.addSkipped(1); + } + else + { + recordSample(c, found, changes); + QRecord update = new QRecord(); + update.setTableName(c.target.getName()); + update.setValue(c.pk, found.getValue(c.pk)); + changes.forEach(update::setValue); + updates.add(update); + } + } + + List clean = withoutConflicts(c, inserts); + + if(dryRun) + { + o.addInserted(clean.size()); + o.addUpdated(updates.size()); + return; + } + write(c, clean, updates); + } + + + + /******************************************************************************* + ** The fields that differ between the backup row and the target row. The primary + ** key is not compared unless rows are matched on it. + *******************************************************************************/ + private Map diff(TableContext c, QRecord backupRow, QRecord found) + { + Map changes = new LinkedHashMap<>(); + for(Map.Entry value : backupRow.getValues().entrySet()) + { + QFieldMetaData field = c.target.getFields().get(value.getKey()); + if(field == null || field.getName().equals(c.pk)) + { + continue; + } + if(!same(field, value.getValue(), found.getValue(field.getName()))) + { + changes.put(field.getName(), value.getValue()); + } + } + return (changes); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static boolean same(QFieldMetaData field, Serializable a, Serializable b) + { + if(a == null || b == null) + { + return (a == null && b == null); + } + if(field.getType() == QFieldType.DECIMAL) + { + BigDecimal x = com.kingsrook.qqq.backend.core.utils.ValueUtils.getValueAsBigDecimal(a); + BigDecimal y = com.kingsrook.qqq.backend.core.utils.ValueUtils.getValueAsBigDecimal(b); + return (x.compareTo(y) == 0); + } + return (canonical(field, a).equals(canonical(field, b))); + } + + + + /******************************************************************************* + ** Remember a changed record for the sample diff, masking sensitive values. + *******************************************************************************/ + private void recordSample(TableContext c, QRecord found, Map changes) + { + c.outcome.countChangedRecord(); + if(c.outcome.getSamples().size() >= MAX_SAMPLES) + { + return; + } + List lines = new ArrayList<>(); + for(Map.Entry change : changes.entrySet()) + { + QFieldMetaData field = c.target.getField(change.getKey()); + lines.add(change.getKey() + ": " + show(field, found.getValue(change.getKey())) + " -> " + show(field, change.getValue())); + } + c.outcome.getSamples().add(new RestoreOutcome.Sample(pkText(c, found), lines)); + } + + + + /******************************************************************************* + ** A value as shown in Review: sensitive values are masked, blobs summarized, + ** long text truncated. + *******************************************************************************/ + static String show(QFieldMetaData field, Serializable value) + { + if(field.getIsSensitive() || field.getType() == QFieldType.PASSWORD) + { + return ("********"); + } + if(value == null) + { + return ("(empty)"); + } + if(field.getType() == QFieldType.BLOB) + { + return ("(binary)"); + } + String text = canonical(field, value).replace('\n', ' '); + return (text.length() > MAX_TEXT ? text.substring(0, MAX_TEXT) + "..." : text); + } + + + + /******************************************************************************* + ** Inserts that collide on a unique key (or, when matching on a unique key, on the + ** primary key) with a target row that is staying are conflicts, and are not written. + *******************************************************************************/ + private List withoutConflicts(TableContext c, List inserts) throws QException + { + if(inserts.isEmpty()) + { + return (inserts); + } + List> keys = new ArrayList<>(); + for(UniqueKey key : c.target.getUniqueKeys() == null ? List.of() : c.target.getUniqueKeys()) + { + if(!new HashSet<>(key.getFieldNames()).equals(new HashSet<>(c.matchFields))) + { + keys.add(key.getFieldNames()); + } + } + if(!c.matchIsPk && !c.remap) + { + keys.add(List.of(c.pk)); + } + + Set conflicted = new LinkedHashSet<>(); + for(List fields : keys) + { + List candidates = inserts.stream().map(Pending::record).filter(r -> keyText(c.target, fields, r) != null).toList(); + if(candidates.isEmpty()) + { + continue; + } + Map found = lookup(c.target, fields, candidates); + for(Pending p : inserts) + { + QRecord hit = found.get(keyText(c.target, fields, p.record)); + if(hit != null && !c.deletePkTexts.contains(pkText(c, hit))) + { + conflicted.add(p); + } + } + } + c.outcome.addConflicts(conflicted.size()); + return (inserts.stream().filter(p -> !conflicted.contains(p)).toList()); + } + + + + /******************************************************************************* + ** Count the rows whose reference points to a parent that is in neither the + ** backup (the selected tables) nor the target. + *******************************************************************************/ + private void countOrphans(TableContext c, List page) throws QException + { + Set orphanRows = new HashSet<>(); + for(Map.Entry fk : c.foreignKeys.entrySet()) + { + QFieldMetaData field = c.target.getField(fk.getKey()); + String parent = fk.getValue(); + Set inBackup = order.contains(parent) ? pkKeysOf(parent) : Set.of(); + + Map missing = new LinkedHashMap<>(); + for(int i = 0; i < page.size(); i++) + { + Serializable value = page.get(i).getValue(fk.getKey()); + if(value != null && !inBackup.contains(canonical(field, value))) + { + missing.put(i, canonical(field, value)); + } + } + if(missing.isEmpty()) + { + continue; + } + + Set inTarget = existingKeys(parent, new HashSet<>(missing.values())); + for(Map.Entry m : missing.entrySet()) + { + if(!inTarget.contains(m.getValue())) + { + orphanRows.add(m.getKey()); + } + } + } + c.outcome.addOrphans(orphanRows.size()); + } + + + + /******************************************************************************* + ** The primary keys (as text) of the rows of a selected table in the backup. + *******************************************************************************/ + private Set pkKeysOf(String parent) throws QException + { + if(pkKeySets.containsKey(parent)) + { + return (pkKeySets.get(parent)); + } + Set keys = new HashSet<>(); + QTableMetaData target = qInstance.getTable(targets.get(parent)); + QQueryFilter filter = parseFilter(request.filterFor(parent)); + BackupManifest.TableEntry entry = source.getManifest().table(parent); + if(entry != null) + { + try + { + source.readRows(entry, target, row -> + { + if(filter == null || BackendQueryFilterUtils.doesRecordMatch(filter, row)) + { + keys.add(canonical(target.getField(target.getPrimaryKeyField()), row.getValue(target.getPrimaryKeyField()))); + } + }); + } + catch(Failure f) + { + throw ((QException) f.getCause()); + } + } + pkKeySets.put(parent, keys); + return (keys); + } + + + + /******************************************************************************* + ** Which of these primary keys (as text) exist in the target table. + *******************************************************************************/ + private Set existingKeys(String parent, Set keyTexts) throws QException + { + Set found = new HashSet<>(); + if(keyTexts.isEmpty() || qInstance.getTable(targets.getOrDefault(parent, parent)) == null) + { + return (found); + } + QTableMetaData table = qInstance.getTable(targets.getOrDefault(parent, parent)); + QFieldMetaData pk = table.getField(table.getPrimaryKeyField()); + List values = new ArrayList<>(); + for(String text : keyTexts) + { + values.add(FieldValueCodec.fromCanonicalText(pk.getType(), text)); + } + for(QRecord row : query(table.getName(), new QQueryFilter(new QFilterCriteria(pk.getName(), QCriteriaOperator.IN, values)))) + { + found.add(canonical(pk, row.getValue(pk.getName()))); + } + return (found); + } + + + + /******************************************************************************* + ** REMAP: point references at the new ids of the parents restored earlier. + *******************************************************************************/ + private void rewriteReferences(TableContext c, List page) + { + for(Map.Entry fk : c.foreignKeys.entrySet()) + { + Map map = idMaps.get(fk.getValue()); + if(map == null) + { + continue; + } + QFieldMetaData field = c.target.getField(fk.getKey()); + for(QRecord row : page) + { + Serializable value = row.getValue(fk.getKey()); + if(value != null && map.containsKey(canonical(field, value))) + { + row.setValue(fk.getKey(), map.get(canonical(field, value))); + } + } + } + } + + + + /******************************************************************************* + ** Write a page of inserts and updates in one transaction (or in the table's). + *******************************************************************************/ + private void write(TableContext c, List inserts, List updates) throws QException + { + RestoreOutcome.TableOutcome o = c.outcome; + QBackendTransaction tx = c.tableTransaction; + boolean ownTransaction = tx == null; + if(ownTransaction) + { + tx = QBackendTransaction.openFor(new InsertInput(c.target.getName())); + } + + try + { + if(!updates.isEmpty()) + { + for(QRecord out : request.getWriter().update(c.target, updates, request, tx)) + { + if(hasErrors(out)) + { + o.addErrored(1); + addProblem(o, out); + } + else + { + o.addUpdated(1); + } + } + } + + if(!inserts.isEmpty()) + { + List records = new ArrayList<>(); + for(Pending p : inserts) + { + if(c.remap) + { + p.record.getValues().remove(c.pk); + } + records.add(p.record); + } + List outs = request.getWriter().insert(c.target, records, request, tx); + for(int i = 0; i < outs.size(); i++) + { + QRecord out = outs.get(i); + if(hasErrors(out)) + { + o.addErrored(1); + addProblem(o, out); + retry.computeIfAbsent(c.name, k -> new ArrayList<>()).add(inserts.get(i)); + } + else + { + o.addInserted(1); + if(c.remap && inserts.get(i).oldPkText != null) + { + idMaps.computeIfAbsent(c.name, k -> new HashMap<>()).put(inserts.get(i).oldPkText, out.getValue(c.pk)); + recordIdMap(c, inserts.get(i).oldPkText, out.getValue(c.pk)); + } + } + } + } + + if(ownTransaction) + { + tx.commit(); + } + } + catch(QException | RuntimeException e) + { + if(ownTransaction) + { + tx.rollback(); + } + throw (e); + } + finally + { + if(ownTransaction) + { + tx.close(); + } + } + } + + + + /******************************************************************************* + ** Persist one old-to-new id (visible in backups_id_map until the restore ends). + *******************************************************************************/ + private void recordIdMap(TableContext c, String oldId, Serializable newId) throws QException + { + if(restoreRunId == null) + { + return; + } + QRecord row = new QRecord(); + row.setValue("restoreRunId", restoreRunId); + row.setValue("tableName", c.name); + row.setValue("oldId", oldId); + row.setValue("newId", String.valueOf(newId)); + InsertInput input = new InsertInput(BackupsTables.ID_MAP); + input.setRecords(List.of(row)); + input.withPermissionBypass("backups restore id map"); + new InsertAction().execute(input); + } + + + + /******************************************************************************* + ** One more try, after every other table, for the rows that failed to insert (a + ** database foreign key QQQ does not know about may have needed another table first). + *******************************************************************************/ + private void retryPass() throws QException + { + for(Map.Entry> entry : retry.entrySet()) + { + RestoreOutcome.TableOutcome o = outcome.getTables().get(entry.getKey()); + QTableMetaData target = qInstance.getTable(o.getTargetName()); + for(Pending p : entry.getValue()) + { + try(QBackendTransaction tx = QBackendTransaction.openFor(new InsertInput(target.getName()))) + { + List outs = request.getWriter().insert(target, List.of(p.record), request, tx); + if(hasErrors(outs.get(0))) + { + tx.rollback(); + } + else + { + tx.commit(); + o.addErrored(-1); + o.addInserted(1); + if(p.oldPkText != null && RestoreRequest.IDS_REMAP.equals(request.getIdStrategy())) + { + idMaps.computeIfAbsent(entry.getKey(), k -> new HashMap<>()).put(p.oldPkText, outs.get(0).getValue(target.getPrimaryKeyField())); + } + } + } + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static boolean hasErrors(QRecord record) + { + return (record.getErrors() != null && !record.getErrors().isEmpty()); + } + + + + /******************************************************************************* + ** Keep the first few distinct error messages (never the row values). + *******************************************************************************/ + private static void addProblem(RestoreOutcome.TableOutcome o, QRecord out) + { + String message = out.getErrors().get(0).getMessage(); + if(o.getProblems().size() < MAX_PROBLEMS && !o.getProblems().contains(message)) + { + o.getProblems().add(message); + } + } + + + + /******************************************************************************* + ** Look rows up in the target by a set of fields. Keys are the canonical key texts. + *******************************************************************************/ + private Map lookup(QTableMetaData table, List fields, List rows) throws QException + { + Map found = new HashMap<>(); + QQueryFilter filter = new QQueryFilter(); + boolean any = false; + if(fields.size() == 1) + { + Set values = new LinkedHashSet<>(); + for(QRecord row : rows) + { + if(row.getValue(fields.get(0)) != null) + { + values.add(row.getValue(fields.get(0))); + } + } + any = !values.isEmpty(); + filter.addCriteria(new QFilterCriteria(fields.get(0), QCriteriaOperator.IN, new ArrayList<>(values))); + } + else + { + filter.setBooleanOperator(QQueryFilter.BooleanOperator.OR); + for(QRecord row : rows) + { + if(keyText(table, fields, row) == null) + { + continue; + } + QQueryFilter one = new QQueryFilter(); + for(String field : fields) + { + one.addCriteria(new QFilterCriteria(field, QCriteriaOperator.EQUALS, row.getValue(field))); + } + filter.addSubFilter(one); + any = true; + } + } + if(!any) + { + return (found); + } + for(QRecord row : query(table.getName(), filter)) + { + String key = keyText(table, fields, row); + if(key != null) + { + found.put(key, row); + } + } + return (found); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private List query(String table, QQueryFilter filter) throws QException + { + QueryInput input = new QueryInput(table); + input.setFilter(filter); + input.setShouldGenerateDisplayValues(false); + input.setShouldTranslatePossibleValues(false); + input.setShouldMaskPasswords(false); + input.setShouldOmitHiddenFields(false); + input.setShouldFetchHeavyFields(true); + return (new QueryAction().execute(input).getRecords()); + } + + + + /******************************************************************************* + ** The canonical text of a key made of these fields, or null when any is null. + *******************************************************************************/ + private static String keyText(QTableMetaData table, List fields, QRecord row) + { + StringBuilder sb = new StringBuilder(); + for(String field : fields) + { + Serializable value = row.getValue(field); + if(value == null) + { + return (null); + } + sb.append(canonical(table.getField(field), value)).append(SEP); + } + return (sb.toString()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static String pkText(TableContext c, QRecord row) + { + return (canonical(c.target.getField(c.pk), row.getValue(c.pk))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static String canonical(QFieldMetaData field, Serializable value) + { + return (FieldValueCodec.toCanonicalText(field.getType(), value)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QQueryFilter parseFilter(String json) throws QException + { + try + { + return (json == null || json.isBlank() ? null : JsonUtils.toObject(json, QQueryFilter.class)); + } + catch(IOException e) + { + throw (new QException("Could not read a table filter: " + e.getMessage())); + } + } + + + + /******************************************************************************* + ** The files the rows of this page reference: check them against the target and + ** keep the ones to restore. + *******************************************************************************/ + private void collectFiles(TableContext c, List page) throws QException + { + for(QRecord row : page) + { + for(FileReferenceProvider provider : request.getFileProviders()) + { + for(FileReference ref : provider.referencesIn(c.target, row)) + { + collectFile(c, ref); + } + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void collectFile(TableContext c, FileReference ref) throws QException + { + RestoreOutcome.TableOutcome o = c.outcome; + JsonNode node = fileIndex.get(BackupSource.indexKeyOf(c.name, ref.recordKey(), ref.field())); + String status = node == null ? "MISSING" : node.path("status").asText(); + if(!"COPIED".equals(status) && !"DEDUPLICATED".equals(status)) + { + o.addFilesMissing(1); + return; + } + + String alias = request.getFileTargetBucketAlias() != null ? request.getFileTargetBucketAlias() : ref.bucketAlias(); + String key = (request.getFileTargetPrefix() == null ? "" : request.getFileTargetPrefix()) + ref.key(); + if(!seenFiles.add(alias + SEP + key)) + { + return; + } + + BackupStorage storage; + try + { + storage = BackupsRuntime.storageFor(alias); + } + catch(RuntimeException e) + { + o.addFilesMissing(1); + addFileProblem(o, "No storage is configured for the bucket alias " + alias); + return; + } + + long size = node.path("size").asLong(-1); + String sha256 = node.path("sha256").asText(); + var head = storage.head(key); + if(head.isEmpty()) + { + o.addFilesToRestore(1); + fileWork.add(new FileWork(c.name, alias, key, sha256, node.path("casKey").asText(), size, false)); + } + else if(head.get().size() == size && sha256.equals(hashOf(storage, key))) + { + o.addFilesPresent(1); + } + else + { + o.addFilesConflicting(1); + fileWork.add(new FileWork(c.name, alias, key, sha256, node.path("casKey").asText(), size, true)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void addFileProblem(RestoreOutcome.TableOutcome o, String message) + { + if(o.getProblems().size() < MAX_PROBLEMS && !o.getProblems().contains(message)) + { + o.getProblems().add(message); + } + } + + + + /******************************************************************************* + ** SHA-256 of a stored object (null when it cannot be read). + *******************************************************************************/ + private static String hashOf(BackupStorage storage, String key) + { + try(InputStream in = storage.openRead(key); + HashingOutputStream sink = new HashingOutputStream(OutputStream.nullOutputStream())) + { + in.transferTo(sink); + sink.close(); + return (sink.getSha256()); + } + catch(IOException | RuntimeException e) + { + return (null); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/restore/RestoreRequest.java b/src/main/java/com/kingsrook/qbits/backups/restore/RestoreRequest.java new file mode 100644 index 0000000..e8ea73b --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/restore/RestoreRequest.java @@ -0,0 +1,612 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.restore; + + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.backups.files.FileReferenceProvider; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** Everything a restore needs to know (spec 9.3): the source run, the tables and + ** per-table filters, the target, the mode, how rows are matched, what happens to + ** ids, and the options. Built from the first screen of the process. + *******************************************************************************/ +public class RestoreRequest +{ + public static final String MODE_INSERT_NEW = "INSERT_NEW"; + public static final String MODE_UPSERT = "UPSERT"; + public static final String MODE_REPLACE = "REPLACE"; + + public static final String IDS_PRESERVE = "PRESERVE"; + public static final String IDS_REMAP = "REMAP_NEW_IDS"; + + public static final String MATCH_PRIMARY_KEY = "PRIMARY_KEY"; + + private Integer sourceRunId; + private List tables = new ArrayList<>(); + private Map filters = new LinkedHashMap<>(); + private String targetTable; + private String mode = MODE_INSERT_NEW; + private String matchOn = MATCH_PRIMARY_KEY; + private String idStrategy = IDS_PRESERVE; + private boolean suppressAutomations = true; + private boolean publishEvents = false; + private boolean runCustomizers = false; + private boolean perRecordAudit = false; + private boolean restoreFiles = true; + private boolean preRestoreBackup = true; + private String fileTargetBucketAlias; + private String fileTargetPrefix; + private String confirmationText; + private String requestedBy = "system"; + private int pageSize = 1000; + private List fileProviders = new ArrayList<>(List.of(new FileReferenceProvider.FromTableMetaData())); + private RestoreWriter writer = new StandardRestoreWriter(); + + + + /******************************************************************************* + ** Build from the values of the first screen of the process. The tables value is + ** a comma separated list (blank means every table in the backup); the filters + ** value is a JSON object of table name to query filter. + *******************************************************************************/ + public static RestoreRequest fromInput(RunBackendStepInput input, Integer runId, List forcedTables) throws com.kingsrook.qqq.backend.core.exceptions.QException + { + RestoreRequest request = new RestoreRequest(); + request.sourceRunId = runId; + if(forcedTables != null && !forcedTables.isEmpty()) + { + request.tables = new ArrayList<>(forcedTables); + } + else + { + request.tables = splitList(input.getValueString("tables")); + } + request.filters = parseFilters(input.getValueString("tableFilters")); + request.targetTable = blankToNull(input.getValueString("targetTable")); + request.mode = orDefault(input.getValueString("mode"), MODE_INSERT_NEW); + request.matchOn = orDefault(input.getValueString("matchOn"), MATCH_PRIMARY_KEY); + request.idStrategy = orDefault(input.getValueString("idStrategy"), IDS_PRESERVE); + request.suppressAutomations = bool(input, "suppressAutomations", true); + request.publishEvents = bool(input, "publishEvents", false); + request.runCustomizers = bool(input, "runCustomizers", false); + request.perRecordAudit = bool(input, "perRecordAudit", false); + request.restoreFiles = bool(input, "restoreFiles", true); + request.preRestoreBackup = bool(input, "preRestoreBackup", true); + request.fileTargetBucketAlias = blankToNull(input.getValueString("fileTargetBucketAlias")); + request.fileTargetPrefix = blankToNull(input.getValueString("fileTargetPrefix")); + request.confirmationText = input.getValueString("confirmationText"); + com.kingsrook.qqq.backend.core.model.session.QSession session = com.kingsrook.qqq.backend.core.context.QContext.getQSession(); + request.requestedBy = session == null || session.getUser() == null ? "system" : session.getUser().getIdReference(); + return (request); + } + + + + /******************************************************************************* + ** The text the user must type to confirm a REPLACE of these tables. + *******************************************************************************/ + public static String expectedConfirmation(List tables) + { + return (tables.size() == 1 ? tables.get(0) : "REPLACE " + tables.size() + " TABLES"); + } + + + + /******************************************************************************* + ** The filter json for a table, or null. + *******************************************************************************/ + public String filterFor(String table) + { + return (filters.get(table)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static List splitList(String text) + { + List rs = new ArrayList<>(); + if(StringUtils.hasContent(text)) + { + for(String part : text.split(",")) + { + if(!part.isBlank()) + { + rs.add(part.trim()); + } + } + } + return (rs); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static Map parseFilters(String json) throws com.kingsrook.qqq.backend.core.exceptions.QException + { + Map rs = new LinkedHashMap<>(); + if(!StringUtils.hasContent(json)) + { + return (rs); + } + try + { + com.fasterxml.jackson.databind.JsonNode node = new com.fasterxml.jackson.databind.ObjectMapper().readTree(json); + for(var it = node.fields(); it.hasNext(); ) + { + var entry = it.next(); + rs.put(entry.getKey(), entry.getValue().isTextual() ? entry.getValue().asText() : entry.getValue().toString()); + } + } + catch(java.io.IOException e) + { + throw (new com.kingsrook.qqq.backend.core.exceptions.QException("The per-table filters are not valid JSON: " + e.getMessage())); + } + return (rs); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static boolean bool(RunBackendStepInput input, String name, boolean defaultValue) + { + Boolean value = input.getValueBoolean(name); + return (value == null ? defaultValue : value); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static String orDefault(String value, String defaultValue) + { + return (StringUtils.hasContent(value) ? value : defaultValue); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static String blankToNull(String value) + { + return (StringUtils.hasContent(value) ? value : null); + } + + + + /******************************************************************************* + ** Getter for sourceRunId + *******************************************************************************/ + public Integer getSourceRunId() + { + return (sourceRunId); + } + + + + /******************************************************************************* + ** Fluent setter for sourceRunId + *******************************************************************************/ + public RestoreRequest withSourceRunId(Integer sourceRunId) + { + this.sourceRunId = sourceRunId; + return (this); + } + + + + /******************************************************************************* + ** Getter for tables (empty means every table in the backup) + *******************************************************************************/ + public List getTables() + { + return (tables); + } + + + + /******************************************************************************* + ** Fluent setter for tables + *******************************************************************************/ + public RestoreRequest withTables(List tables) + { + this.tables = new ArrayList<>(tables); + return (this); + } + + + + /******************************************************************************* + ** Getter for filters + *******************************************************************************/ + public Map getFilters() + { + return (filters); + } + + + + /******************************************************************************* + ** Fluent setter for one table filter (json of a QQueryFilter) + *******************************************************************************/ + public RestoreRequest withFilter(String table, String filterJson) + { + this.filters.put(table, filterJson); + return (this); + } + + + + /******************************************************************************* + ** Getter for targetTable + *******************************************************************************/ + public String getTargetTable() + { + return (targetTable); + } + + + + /******************************************************************************* + ** Fluent setter for targetTable + *******************************************************************************/ + public RestoreRequest withTargetTable(String targetTable) + { + this.targetTable = targetTable; + return (this); + } + + + + /******************************************************************************* + ** Getter for mode + *******************************************************************************/ + public String getMode() + { + return (mode); + } + + + + /******************************************************************************* + ** Fluent setter for mode + *******************************************************************************/ + public RestoreRequest withMode(String mode) + { + this.mode = mode; + return (this); + } + + + + /******************************************************************************* + ** Getter for matchOn + *******************************************************************************/ + public String getMatchOn() + { + return (matchOn); + } + + + + /******************************************************************************* + ** Fluent setter for matchOn + *******************************************************************************/ + public RestoreRequest withMatchOn(String matchOn) + { + this.matchOn = matchOn; + return (this); + } + + + + /******************************************************************************* + ** Getter for idStrategy + *******************************************************************************/ + public String getIdStrategy() + { + return (idStrategy); + } + + + + /******************************************************************************* + ** Fluent setter for idStrategy + *******************************************************************************/ + public RestoreRequest withIdStrategy(String idStrategy) + { + this.idStrategy = idStrategy; + return (this); + } + + + + /******************************************************************************* + ** Getter for suppressAutomations + *******************************************************************************/ + public boolean getSuppressAutomations() + { + return (suppressAutomations); + } + + + + /******************************************************************************* + ** Fluent setter for suppressAutomations + *******************************************************************************/ + public RestoreRequest withSuppressAutomations(boolean suppressAutomations) + { + this.suppressAutomations = suppressAutomations; + return (this); + } + + + + /******************************************************************************* + ** Getter for publishEvents + *******************************************************************************/ + public boolean getPublishEvents() + { + return (publishEvents); + } + + + + /******************************************************************************* + ** Fluent setter for publishEvents + *******************************************************************************/ + public RestoreRequest withPublishEvents(boolean publishEvents) + { + this.publishEvents = publishEvents; + return (this); + } + + + + /******************************************************************************* + ** Getter for runCustomizers + *******************************************************************************/ + public boolean getRunCustomizers() + { + return (runCustomizers); + } + + + + /******************************************************************************* + ** Fluent setter for runCustomizers + *******************************************************************************/ + public RestoreRequest withRunCustomizers(boolean runCustomizers) + { + this.runCustomizers = runCustomizers; + return (this); + } + + + + /******************************************************************************* + ** Getter for perRecordAudit + *******************************************************************************/ + public boolean getPerRecordAudit() + { + return (perRecordAudit); + } + + + + /******************************************************************************* + ** Fluent setter for perRecordAudit + *******************************************************************************/ + public RestoreRequest withPerRecordAudit(boolean perRecordAudit) + { + this.perRecordAudit = perRecordAudit; + return (this); + } + + + + /******************************************************************************* + ** Getter for restoreFiles + *******************************************************************************/ + public boolean getRestoreFiles() + { + return (restoreFiles); + } + + + + /******************************************************************************* + ** Fluent setter for restoreFiles + *******************************************************************************/ + public RestoreRequest withRestoreFiles(boolean restoreFiles) + { + this.restoreFiles = restoreFiles; + return (this); + } + + + + /******************************************************************************* + ** Getter for preRestoreBackup + *******************************************************************************/ + public boolean getPreRestoreBackup() + { + return (preRestoreBackup); + } + + + + /******************************************************************************* + ** Fluent setter for preRestoreBackup + *******************************************************************************/ + public RestoreRequest withPreRestoreBackup(boolean preRestoreBackup) + { + this.preRestoreBackup = preRestoreBackup; + return (this); + } + + + + /******************************************************************************* + ** Getter for fileTargetBucketAlias + *******************************************************************************/ + public String getFileTargetBucketAlias() + { + return (fileTargetBucketAlias); + } + + + + /******************************************************************************* + ** Fluent setter for fileTargetBucketAlias + *******************************************************************************/ + public RestoreRequest withFileTargetBucketAlias(String fileTargetBucketAlias) + { + this.fileTargetBucketAlias = fileTargetBucketAlias; + return (this); + } + + + + /******************************************************************************* + ** Getter for fileTargetPrefix + *******************************************************************************/ + public String getFileTargetPrefix() + { + return (fileTargetPrefix); + } + + + + /******************************************************************************* + ** Fluent setter for fileTargetPrefix + *******************************************************************************/ + public RestoreRequest withFileTargetPrefix(String fileTargetPrefix) + { + this.fileTargetPrefix = fileTargetPrefix; + return (this); + } + + + + /******************************************************************************* + ** Getter for confirmationText + *******************************************************************************/ + public String getConfirmationText() + { + return (confirmationText); + } + + + + /******************************************************************************* + ** Fluent setter for confirmationText + *******************************************************************************/ + public RestoreRequest withConfirmationText(String confirmationText) + { + this.confirmationText = confirmationText; + return (this); + } + + + + /******************************************************************************* + ** Getter for requestedBy + *******************************************************************************/ + public String getRequestedBy() + { + return (requestedBy); + } + + + + /******************************************************************************* + ** Fluent setter for requestedBy + *******************************************************************************/ + public RestoreRequest withRequestedBy(String requestedBy) + { + this.requestedBy = requestedBy; + return (this); + } + + + + /******************************************************************************* + ** Getter for pageSize + *******************************************************************************/ + public int getPageSize() + { + return (pageSize); + } + + + + /******************************************************************************* + ** Fluent setter for pageSize + *******************************************************************************/ + public RestoreRequest withPageSize(int pageSize) + { + this.pageSize = pageSize; + return (this); + } + + + + /******************************************************************************* + ** Getter for fileProviders + *******************************************************************************/ + public List getFileProviders() + { + return (fileProviders); + } + + + + /******************************************************************************* + ** Getter for writer + *******************************************************************************/ + public RestoreWriter getWriter() + { + return (writer); + } + + + + /******************************************************************************* + ** Fluent setter for writer (tests, or a backend specific writer) + *******************************************************************************/ + public RestoreRequest withWriter(RestoreWriter writer) + { + this.writer = writer; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/restore/RestoreSummaries.java b/src/main/java/com/kingsrook/qbits/backups/restore/RestoreSummaries.java new file mode 100644 index 0000000..cc2c009 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/restore/RestoreSummaries.java @@ -0,0 +1,222 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.restore; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; + + +/******************************************************************************* + ** Turns a RestoreOutcome into the standard Review and Result summary lines + ** (spec 9.3). One group of lines per table, in restore order. Every line has + ** its future wording (Review) and its past wording (Result); sample diffs are + ** bullets on one line per table, with sensitive values already masked. + *******************************************************************************/ +public final class RestoreSummaries +{ + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private RestoreSummaries() + { + } + + + + /******************************************************************************* + ** A line with all four wordings. + *******************************************************************************/ + public static ProcessSummaryLine line(Status status, long count, String singularFuture, String pluralFuture, String singularPast, String pluralPast) + { + ProcessSummaryLine line = new ProcessSummaryLine(status, (int) Math.min(Integer.MAX_VALUE, count), null); + line.setSingularFutureMessage(singularFuture); + line.setPluralFutureMessage(pluralFuture); + line.setSingularPastMessage(singularPast); + line.setPluralPastMessage(pluralPast); + return (line); + } + + + + /******************************************************************************* + ** A one-off message line (the same text before and after). + *******************************************************************************/ + public static ProcessSummaryLine message(Status status, String text) + { + return (line(status, 1, text, text, text, text)); + } + + + + /******************************************************************************* + ** The lines of an outcome. + *******************************************************************************/ + public static List lines(RestoreOutcome outcome, boolean isResult) + { + List lines = new ArrayList<>(); + + for(String error : outcome.getErrors()) + { + lines.add(message(Status.ERROR, error)); + } + + for(RestoreOutcome.TableOutcome t : outcome.getTables().values()) + { + tableLines(lines, t, isResult); + } + + long sequences = outcome.sum(t -> t.getSequenceNeeded() ? 1 : 0); + if(sequences > 0 && !isResult) + { + lines.add(line(Status.INFO, sequences, "table will have its key sequence resynced", "tables will have their key sequences resynced", + "table had its key sequence resynced", "tables had their key sequences resynced")); + } + if(isResult && outcome.getSequencesResynced() > 0) + { + lines.add(line(Status.OK, outcome.getSequencesResynced(), "key sequence was resynced", "key sequences were resynced", + "key sequence was resynced", "key sequences were resynced")); + } + if(isResult && outcome.getPreRestoreRunId() != null) + { + lines.add(message(Status.INFO, "A backup of the target tables was taken first (backup run " + outcome.getPreRestoreRunId() + "), so this restore can be undone.")); + } + if(isResult && outcome.getFilesFailed() > 0) + { + lines.add(line(Status.ERROR, outcome.getFilesFailed(), "file could not be restored", "files could not be restored", "file could not be restored", "files could not be restored")); + } + + for(String warning : outcome.getWarnings()) + { + lines.add(message(Status.WARNING, warning)); + } + for(String note : outcome.getNotes()) + { + lines.add(message(Status.INFO, note)); + } + return (lines); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void tableLines(List lines, RestoreOutcome.TableOutcome t, boolean isResult) + { + String in = " " + t.getTargetName(); + + if("SKIPPED".equals(t.getStatus())) + { + lines.add(message(Status.ERROR, "The table " + t.getName() + " will not be restored: its schema does not fit the backup")); + } + for(String note : t.getSchemaNotes()) + { + lines.add(message("MISMATCH".equals(t.getSchemaStatus()) ? Status.ERROR : Status.WARNING, "Schema of " + t.getName() + ": " + note)); + } + for(String reference : t.getUnmappedReferences()) + { + lines.add(message(Status.WARNING, "The field " + reference + " looks like a reference but cannot be traced, so it is not rewritten to the new ids")); + } + + add(lines, line(Status.OK, t.getInserted(), "record will be inserted into" + in, "records will be inserted into" + in, + "record was inserted into" + in, "records were inserted into" + in)); + add(lines, line(Status.OK, t.getUpdated(), "record will be updated in" + in, "records will be updated in" + in, + "record was updated in" + in, "records were updated in" + in)); + add(lines, line(Status.INFO, t.getUnchanged(), "record is unchanged in" + in, "records are unchanged in" + in, + "record was already the same in" + in, "records were already the same in" + in)); + add(lines, line(Status.INFO, t.getSkipped(), "record exists with different content and will be left alone in" + in, + "records exist with different content and will be left alone in" + in, + "record existed with different content and was left alone in" + in, "records existed with different content and were left alone in" + in)); + add(lines, line(Status.WARNING, t.getDeleted(), "record will be deleted from" + in + " (REPLACE)", "records will be deleted from" + in + " (REPLACE)", + "record was deleted from" + in + " (REPLACE)", "records were deleted from" + in + " (REPLACE)")); + add(lines, line(Status.WARNING, t.getConflicts(), "record conflicts on a unique key in" + in + " and will be skipped", + "records conflict on a unique key in" + in + " and will be skipped", + "record conflicted on a unique key in" + in + " and was skipped", "records conflicted on a unique key in" + in + " and were skipped")); + add(lines, line(Status.WARNING, t.getOrphans(), "record in" + in + " references a parent that is missing from the backup and the target", + "records in" + in + " reference a parent that is missing from the backup and the target", + "record in" + in + " referenced a parent that is missing from the backup and the target", + "records in" + in + " referenced a parent that is missing from the backup and the target")); + if(isResult) + { + add(lines, line(Status.ERROR, t.getErrored(), "record could not be written to" + in, "records could not be written to" + in, + "record could not be written to" + in, "records could not be written to" + in)); + for(String problem : t.getProblems()) + { + lines.add(message(Status.ERROR, problem)); + } + } + + fileLines(lines, t, isResult); + + if(!t.getSamples().isEmpty() && !isResult) + { + ProcessSummaryLine sample = line(Status.INFO, t.getSamples().size(), + "changed record in" + in + " (field: now -> from the backup; sensitive values are hidden)", + "changed records in" + in + " (field: now -> from the backup; sensitive values are hidden)", "", ""); + ArrayList bullets = new ArrayList<>(); + for(RestoreOutcome.Sample s : t.getSamples()) + { + bullets.add("#" + s.key() + " " + String.join("; ", s.changes())); + } + sample.setBulletsOfText(bullets); + lines.add(sample); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void fileLines(List lines, RestoreOutcome.TableOutcome t, boolean isResult) + { + String in = " for" + " " + t.getTargetName(); + if(isResult) + { + add(lines, line(Status.OK, t.getFilesRestored(), "file was restored" + in, "files were restored" + in, "file was restored" + in, "files were restored" + in)); + } + else + { + add(lines, line(Status.OK, t.getFilesToRestore(), "file will be restored" + in, "files will be restored" + in, "file was restored" + in, "files were restored" + in)); + } + add(lines, line(Status.INFO, t.getFilesPresent(), "file is already there with the same checksum" + in + " and is skipped", + "files are already there with the same checksum" + in + " and are skipped", + "file was already there with the same checksum" + in, "files were already there with the same checksum" + in)); + add(lines, line(Status.WARNING, t.getFilesConflicting(), "file exists with different content" + in + " (kept, unless the mode is REPLACE)", + "files exist with different content" + in + " (kept, unless the mode is REPLACE)", + "file existed with different content" + in, "files existed with different content" + in)); + add(lines, line(Status.WARNING, t.getFilesMissing(), "file is missing from the backup" + in, "files are missing from the backup" + in, + "file was missing from the backup" + in, "files were missing from the backup" + in)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void add(List lines, ProcessSummaryLine line) + { + if(line.getCount() != null && line.getCount() > 0) + { + lines.add(line); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/restore/RestoreWriter.java b/src/main/java/com/kingsrook/qbits/backups/restore/RestoreWriter.java new file mode 100644 index 0000000..d6ffb2f --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/restore/RestoreWriter.java @@ -0,0 +1,81 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.restore; + + +import java.io.Serializable; +import java.util.Collection; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** How a restore writes (spec 5.3 and 5.4). The default implementation uses the + ** plain Insert, Update and Delete actions, and selects the core RestoreWriteFlag + ** (QRun-IO/qqq#1111) and the core key-sequence resync (#1112) reflectively when the + ** running QQQ has them. A backend specific writer can be supplied on the request. + ** + ** Whatever the implementation, record security, field security, unique keys and + ** required-field validation still apply to every write. + *******************************************************************************/ +public interface RestoreWriter +{ + /******************************************************************************* + ** Insert records (restore mode). Returns the output records, with any errors. + *******************************************************************************/ + List insert(QTableMetaData table, List records, RestoreRequest options, QBackendTransaction transaction) throws QException; + + + + /******************************************************************************* + ** Update records (restore mode). Returns the output records, with any errors. + *******************************************************************************/ + List update(QTableMetaData table, List records, RestoreRequest options, QBackendTransaction transaction) throws QException; + + + + /******************************************************************************* + ** Delete records by primary key (REPLACE). Returns the number deleted. + *******************************************************************************/ + long delete(QTableMetaData table, List primaryKeys, RestoreRequest options, QBackendTransaction transaction) throws QException; + + + + /******************************************************************************* + ** Whether the writes honor the core restore flag (so created and modified dates + ** are kept as supplied). + *******************************************************************************/ + boolean usesCoreRestoreFlag(); + + + + /******************************************************************************* + ** Whether key sequences can be resynced on this QQQ (the core hook is present). + *******************************************************************************/ + boolean canResyncSequences(); + + + + /******************************************************************************* + ** Resync key sequences after explicit-id inserts. Returns the names of the tables + ** whose sequence was advanced. + *******************************************************************************/ + List resyncSequences(Collection tables) throws QException; +} diff --git a/src/main/java/com/kingsrook/qbits/backups/restore/StandardRestoreWriter.java b/src/main/java/com/kingsrook/qbits/backups/restore/StandardRestoreWriter.java new file mode 100644 index 0000000..a98603f --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/restore/StandardRestoreWriter.java @@ -0,0 +1,257 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.restore; + + +import java.io.Serializable; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.ActionFlag; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** The default RestoreWriter: the standard Insert, Update and Delete actions. + ** + ** Stand-ins (replace with the core API when QQQ 4.1.0-RC.5 ships): + **
    + **
  • RestoreWriteFlag (QRun-IO/qqq#1111) is created reflectively when the class is + ** present. Without it, the DML audit is omitted (a summary audit is written per + ** table instead), but created and modified dates may be set by the normal defaults.
    • + **
  • resyncKeySequences (QRun-IO/qqq#1112) is called reflectively on the backend + ** module when it exists; otherwise it is a no-op that the Review reports.
    • + **
+ *******************************************************************************/ +public class StandardRestoreWriter implements RestoreWriter +{ + private static final QLogger LOG = QLogger.getLogger(StandardRestoreWriter.class); + + private static final String FLAG_CLASS = "com.kingsrook.qqq.backend.core.model.actions.tables.RestoreWriteFlag"; + + private static final Class FLAG = lookup(FLAG_CLASS); + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static Class lookup(String name) + { + try + { + return (Class.forName(name)); + } + catch(ClassNotFoundException | LinkageError e) + { + return (null); + } + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public List insert(QTableMetaData table, List records, RestoreRequest options, QBackendTransaction transaction) throws QException + { + InsertInput input = new InsertInput(table.getName()); + input.setRecords(records); + input.setOmitDmlAudit(!options.getPerRecordAudit()); + if(transaction != null) + { + input.setTransaction(transaction); + } + addFlag(options, input::withFlag); + return (new InsertAction().execute(input).getRecords()); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public List update(QTableMetaData table, List records, RestoreRequest options, QBackendTransaction transaction) throws QException + { + UpdateInput input = new UpdateInput(table.getName()); + input.setRecords(records); + input.setOmitDmlAudit(!options.getPerRecordAudit()); + if(transaction != null) + { + input.setTransaction(transaction); + } + addFlag(options, input::withFlag); + return (new UpdateAction().execute(input).getRecords()); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public long delete(QTableMetaData table, List primaryKeys, RestoreRequest options, QBackendTransaction transaction) throws QException + { + DeleteInput input = new DeleteInput(table.getName()); + input.setPrimaryKeys(primaryKeys); + input.setOmitDmlAudit(!options.getPerRecordAudit()); + if(transaction != null) + { + input.setTransaction(transaction); + } + addFlag(options, input::withFlag); + DeleteOutput output = new DeleteAction().execute(input); + if(output.getRecordsWithErrors() != null && !output.getRecordsWithErrors().isEmpty()) + { + QRecord first = output.getRecordsWithErrors().get(0); + throw (new QException("Could not delete " + output.getRecordsWithErrors().size() + " records of " + table.getName() + ": " + + (first.getErrors().isEmpty() ? "unknown error" : first.getErrors().get(0).getMessage()))); + } + return (output.getDeletedRecordCount()); + } + + + + /******************************************************************************* + ** Create the core flag (when the running QQQ has it) and hand it to the input. + *******************************************************************************/ + private void addFlag(RestoreRequest options, java.util.function.Consumer sink) throws QException + { + if(FLAG == null) + { + return; + } + try + { + Object flag = FLAG.getDeclaredConstructor().newInstance(); + FLAG.getMethod("withSuppressAutomations", boolean.class).invoke(flag, options.getSuppressAutomations()); + FLAG.getMethod("withRunCustomizers", boolean.class).invoke(flag, options.getRunCustomizers()); + FLAG.getMethod("withPerRecordAudit", boolean.class).invoke(flag, options.getPerRecordAudit()); + FLAG.getMethod("withPublishEvents", boolean.class).invoke(flag, options.getPublishEvents()); + sink.accept((ActionFlag) flag); + } + catch(ReflectiveOperationException | ClassCastException e) + { + throw (new QException("Could not use the core restore flag: " + e.getMessage(), e)); + } + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public boolean usesCoreRestoreFlag() + { + return (FLAG != null); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public boolean canResyncSequences() + { + return (resyncMethod() != null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static Method resyncMethod() + { + try + { + return (QBackendModuleInterface.class.getMethod("resyncKeySequences", QBackendMetaData.class, Collection.class)); + } + catch(NoSuchMethodException | LinkageError e) + { + return (null); + } + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public List resyncSequences(Collection tables) throws QException + { + Method method = resyncMethod(); + if(method == null) + { + return (List.of()); + } + + Map> byBackend = new LinkedHashMap<>(); + for(QTableMetaData table : tables) + { + byBackend.computeIfAbsent(table.getBackendName(), k -> new ArrayList<>()).add(table); + } + + List resynced = new ArrayList<>(); + for(Map.Entry> entry : byBackend.entrySet()) + { + QBackendMetaData backend = QContext.getQInstance().getBackend(entry.getKey()); + try + { + QBackendModuleInterface module = new QBackendModuleDispatcher().getQBackendModule(backend); + for(Object result : (List) method.invoke(module, backend, entry.getValue())) + { + Object status = result.getClass().getMethod("getStatus").invoke(result); + if("RESYNCED".equals(String.valueOf(status))) + { + resynced.add(String.valueOf(result.getClass().getMethod("getTableName").invoke(result))); + } + } + } + catch(ReflectiveOperationException e) + { + LOG.warn("Could not resync key sequences", e, logPair("backend", entry.getKey())); + throw (new QException("Could not resync key sequences on backend " + entry.getKey() + ": " + e.getMessage(), e)); + } + } + return (resynced); + } +} diff --git a/src/test/java/com/kingsrook/qbits/backups/restore/CatalogImportTest.java b/src/test/java/com/kingsrook/qbits/backups/restore/CatalogImportTest.java new file mode 100644 index 0000000..2be7d23 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/backups/restore/CatalogImportTest.java @@ -0,0 +1,252 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.restore; + + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.backups.BackupsTestUtils; +import com.kingsrook.qbits.backups.codec.HashingOutputStream; +import com.kingsrook.qbits.backups.manifest.BackupManifest; +import com.kingsrook.qbits.backups.model.BackupsTables; +import com.kingsrook.qbits.backups.run.BackupCatalog; +import com.kingsrook.qbits.backups.run.BackupRunResult; +import com.kingsrook.qbits.backups.storage.StoredObjectInfo; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** Import backups from storage (spec 9.6): the database with the catalog is gone, + ** a fresh instance imports the catalog from the manifests in storage and restores + ** from it, files included. + *******************************************************************************/ +class CatalogImportTest +{ + private String originalPostsHash; + private BackupManifest originalManifest; + private Map snapshot; + + + + /******************************************************************************* + ** Back up an instance, keep what is in the backup bucket, then start over: a fresh + ** instance (empty tables, empty catalog, empty media) that only has the bucket. + *******************************************************************************/ + @BeforeEach + void setUp() throws Exception + { + BackupsTestUtils.init(); + BackupsTestUtils.loadBlog(30); + BackupRunResult run = RestoreTestSupport.backup(Map.of()); + originalPostsHash = RestoreTestSupport.contentHash("blogPost"); + originalManifest = RestoreTestSupport.manifest(run); + + snapshot = new LinkedHashMap<>(); + for(StoredObjectInfo info : BackupsTestUtils.BACKUPS.list("")) + { + snapshot.put(info.key(), BackupsTestUtils.BACKUPS.getBytes(info.key())); + } + + BackupsTestUtils.init(); + snapshot.forEach(BackupsTestUtils.BACKUPS::putBytes); + assertThat(new BackupCatalog().query(BackupsTables.RUN, new QQueryFilter())).as("a fresh catalog").isEmpty(); + assertThat(RestoreTestSupport.rows("blogPost")).isEmpty(); + } + + + + /******************************************************************************* + ** The manifests are found, validated and imported (trigger IMPORTED), the content + ** addressed files are registered again, and importing twice changes nothing. + *******************************************************************************/ + @Test + void testImportRebuildsTheCatalog() throws Exception + { + CatalogImporter importer = new CatalogImporter(); + List keys = importer.findManifestKeys(BackupsTestUtils.BACKUPS, "qqq-backups/"); + assertThat(keys).hasSize(1); + + CatalogImporter.Inspection inspection = importer.inspect(BackupsTestUtils.BACKUPS, keys.get(0)); + assertThat(inspection.status()).isEqualTo(CatalogImporter.Inspection.NEW); + assertThat(inspection.manifestSha256()).isEqualTo(HashingOutputStream.sha256Hex(BackupsTestUtils.BACKUPS.getBytes(keys.get(0)))); + + Integer runId = importer.importRun("backups", BackupsTestUtils.BACKUPS, inspection); + QRecord run = new BackupCatalog().get(BackupsTables.RUN, runId).orElseThrow(); + assertThat(run.getValueString("trigger")).isEqualTo("IMPORTED"); + assertThat(run.getValueString("runKey")).isEqualTo(originalManifest.runKey()); + assertThat(run.getValueString("status")).isEqualTo("SUCCEEDED_WITH_WARNINGS"); + assertThat(run.getValueString("manifestKey")).isEqualTo(keys.get(0)); + assertThat(run.getValueString("manifestSha256")).isEqualTo(inspection.manifestSha256()); + assertThat(run.getValueString("storageBucketAlias")).isEqualTo("backups"); + assertThat(run.getValueString("instanceName")).isEqualTo("testapp"); + assertThat(run.getValueLong("rowsTotal")).isEqualTo(originalManifest.totals().rows()); + assertThat(run.getValueInteger("tablesPlanned")).isEqualTo(originalManifest.tables().size()); + assertThat(run.getValueString("verificationStatus")).isEqualTo("UNVERIFIED"); + + List artifacts = new BackupCatalog().where(BackupsTables.ARTIFACT, "backupRunId", runId); + long parts = originalManifest.tables().stream().mapToLong(t -> t.parts().size()).sum(); + assertThat(artifacts).hasSize((int) parts + 3); + assertThat(artifacts).extracting(a -> a.getValueString("artifactType")).contains("MANIFEST", "CHECKSUMS", "FILE_OBJECTS_INDEX", "TABLE_DATA"); + QRecord posts = artifacts.stream().filter(a -> "blogPost".equals(a.getValueString("tableName"))).findFirst().orElseThrow(); + assertThat(posts.getValueString("sha256Stored")).isEqualTo(originalManifest.table("blogPost").parts().get(0).sha256Stored()); + assertThat(posts.getValueString("rowsHash")).isEqualTo(originalManifest.table("blogPost").rowsHash()); + + assertThat(new BackupCatalog().query(BackupsTables.FILE_OBJECT, new QQueryFilter())).as("the file store is registered again").hasSize(3); + + assertThat(importer.inspect(BackupsTestUtils.BACKUPS, keys.get(0)).status()).isEqualTo(CatalogImporter.Inspection.EXISTS); + assertThat(new BackupCatalog().query(BackupsTables.RUN, new QQueryFilter())).hasSize(1); + } + + + + /******************************************************************************* + ** The point of the catalog import: restore from the imported run into the fresh + ** instance and get the original rows (same hash) and the files (same checksum). + *******************************************************************************/ + @Test + void testRestoreFromTheImportedCatalog() throws Exception + { + CatalogImporter importer = new CatalogImporter(); + CatalogImporter.Inspection inspection = importer.inspect(BackupsTestUtils.BACKUPS, importer.findManifestKeys(BackupsTestUtils.BACKUPS, "qqq-backups/").get(0)); + Integer runId = importer.importRun("backups", BackupsTestUtils.BACKUPS, inspection); + + RestoreRequest request = new RestoreRequest().withSourceRunId(runId).withMode(RestoreRequest.MODE_INSERT_NEW).withRequestedBy("tester"); + RestoreOutcome review = new RestoreEngine().review(request); + assertThat(review.getErrors()).isEmpty(); + assertThat(review.getTables().get("author").getInserted()).isEqualTo(3); + assertThat(review.getTables().get("blogPost").getInserted()).isEqualTo(30); + assertThat(review.getTables().get("blogPost").getFilesToRestore()).isEqualTo(3); + + RestoreOutcome outcome = new RestoreEngine().execute(request); + assertThat(outcome.getTables().get("blogPost").getInserted()).isEqualTo(30); + assertThat(outcome.getTables().get("blogPost").getFilesRestored()).isEqualTo(3); + assertThat(outcome.getFilesFailed()).isZero(); + + assertThat(RestoreTestSupport.rows("author")).hasSize(3); + assertThat(RestoreTestSupport.contentHash("blogPost")).isEqualTo(originalPostsHash); + for(int i = 0; i < 3; i++) + { + assertThat(new String(BackupsTestUtils.MEDIA.getBytes("media/photo-" + i + ".jpg"))).isEqualTo("photo bytes " + i); + } + assertThat(new BackupCatalog().where(BackupsTables.RESTORE_RUN, "sourceBackupRunId", runId)).hasSize(1); + } + + + + /******************************************************************************* + ** A manifest whose parts are damaged is reported, not imported. + *******************************************************************************/ + @Test + void testDamagedBackupsAreNotImported() throws Exception + { + CatalogImporter importer = new CatalogImporter(); + String key = importer.findManifestKeys(BackupsTestUtils.BACKUPS, "qqq-backups/").get(0); + String partKey = key.substring(0, key.length() - "manifest.json".length()) + originalManifest.table("blogPost").parts().get(0).key(); + BackupsTestUtils.BACKUPS.corrupt(partKey, new byte[] { 1, 2, 3 }); + + CatalogImporter.Inspection inspection = importer.inspect(BackupsTestUtils.BACKUPS, key); + assertThat(inspection.status()).isEqualTo(CatalogImporter.Inspection.INVALID); + assertThat(inspection.problems()).anyMatch(p -> p.contains("wrong size")); + assertThat(new BackupCatalog().query(BackupsTables.RUN, new QQueryFilter())).isEmpty(); + + BackupsTestUtils.BACKUPS.putBytes(key, "not a manifest".getBytes()); + assertThat(importer.inspect(BackupsTestUtils.BACKUPS, key).problems()).anyMatch(p -> p.contains("cannot be read")); + } + + + + /******************************************************************************* + ** Restore refuses a part that does not match its stored checksum. + *******************************************************************************/ + @Test + void testRestoreRefusesADamagedPart() throws Exception + { + CatalogImporter importer = new CatalogImporter(); + String key = importer.findManifestKeys(BackupsTestUtils.BACKUPS, "qqq-backups/").get(0); + Integer runId = importer.importRun("backups", BackupsTestUtils.BACKUPS, importer.inspect(BackupsTestUtils.BACKUPS, key)); + + String partKey = key.substring(0, key.length() - "manifest.json".length()) + originalManifest.table("author").parts().get(0).key(); + byte[] same = BackupsTestUtils.BACKUPS.getBytes(partKey).clone(); + same[same.length / 2] ^= 0x55; + BackupsTestUtils.BACKUPS.corrupt(partKey, same); + + RestoreRequest request = new RestoreRequest().withSourceRunId(runId).withTables(List.of("author")).withMode(RestoreRequest.MODE_INSERT_NEW).withPreRestoreBackup(false); + assertThatThrownBy(() -> new RestoreEngine().execute(request)).hasMessageContaining("Could not read"); + assertThat(RestoreTestSupport.rows("author")).isEmpty(); + } + + + + /******************************************************************************* + ** The import process: Review lists what will be added, Result says what was. + *******************************************************************************/ + @Test + void testImportProcess() throws Exception + { + RunProcessInput input = new RunProcessInput(); + input.setProcessName("backupImportCatalog"); + input.addValue("bucketAlias", "backups"); + input.addValue("prefix", "qqq-backups/"); + + RunProcessOutput output = new RunProcessAction().execute(input); + assertThat(output.getException()).isEmpty(); + assertThat(output.getProcessState().getNextStepName()).isPresent().get().isEqualTo("input"); + input.addValue("processUUID", output.getProcessUUID()); + + input.setStartAfterStep("input"); + output = new RunProcessAction().execute(input); + assertThat(output.getException()).isEmpty(); + List review = RestoreProcessTest.lines(output, StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY); + assertThat(RestoreTestSupport.line(review, false, "backup will be added to the catalog").getCount()).isEqualTo(1); + assertThat(new BackupCatalog().query(BackupsTables.RUN, new QQueryFilter())).as("Review writes nothing").isEmpty(); + + input.setStartAfterStep("review"); + output = new RunProcessAction().execute(input); + assertThat(output.getException()).isEmpty(); + List result = RestoreProcessTest.lines(output, StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY); + ProcessSummaryLine imported = RestoreTestSupport.line(result, true, "backup was added to the catalog"); + assertThat(imported.getCount()).isEqualTo(1); + assertThat(imported.getPrimaryKeys()).hasSize(1); + assertThat(new BackupCatalog().query(BackupsTables.RUN, new QQueryFilter())).hasSize(1); + + RunProcessInput again = new RunProcessInput(); + again.setProcessName("backupImportCatalog"); + again.addValue("bucketAlias", "backups"); + again.addValue("prefix", "qqq-backups/"); + output = new RunProcessAction().execute(again); + again.addValue("processUUID", output.getProcessUUID()); + again.setStartAfterStep("input"); + output = new RunProcessAction().execute(again); + List second = RestoreProcessTest.lines(output, StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY); + assertThat(RestoreTestSupport.line(second, false, "already in the catalog").getCount()).isEqualTo(1); + assertThat(RestoreTestSupport.line(second, false, "will be added")).isNull(); + assertThat(second.stream().map(l -> ((ProcessSummaryLine) l).getStatus())).doesNotContain(Status.ERROR); + } +} diff --git a/src/test/java/com/kingsrook/qbits/backups/restore/RestoreModesTest.java b/src/test/java/com/kingsrook/qbits/backups/restore/RestoreModesTest.java new file mode 100644 index 0000000..fe05f87 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/backups/restore/RestoreModesTest.java @@ -0,0 +1,515 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.restore; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.backups.BackupsTestUtils; +import com.kingsrook.qbits.backups.codec.HashingOutputStream; +import com.kingsrook.qbits.backups.model.BackupsTables; +import com.kingsrook.qbits.backups.run.BackupCatalog; +import com.kingsrook.qbits.backups.run.BackupRunResult; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** End to end tests of restore (spec 13): back up 30 posts, then delete 10, edit + ** 10 and add 10; Review each mode and check the summary lines; restore and check + ** counts, hashes and files. + *******************************************************************************/ +class RestoreModesTest +{ + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void setUp() throws Exception + { + BackupsTestUtils.init(); + BackupsTestUtils.loadBlog(30); + } + + + + /******************************************************************************* + ** Review of each mode reports the counts the restore will produce, in the standard + ** summary lines, and writes nothing. + *******************************************************************************/ + @Test + void testReviewEachMode() throws Exception + { + BackupRunResult run = RestoreTestSupport.backup(Map.of()); + RestoreTestSupport.mutate(); + assertThat(RestoreTestSupport.rows("blogPost")).hasSize(30); + + RestoreOutcome insertNew = new RestoreEngine().review(RestoreTestSupport.request(run, RestoreRequest.MODE_INSERT_NEW, "blogPost")); + RestoreOutcome.TableOutcome t = insertNew.getTables().get("blogPost"); + assertThat(insertNew.getErrors()).isEmpty(); + assertThat(t.getRows()).isEqualTo(30); + assertThat(t.getInserted()).isEqualTo(10); + assertThat(t.getUpdated()).isZero(); + assertThat(t.getUnchanged()).isEqualTo(10); + assertThat(t.getSkipped()).isEqualTo(10); + assertThat(t.getDeleted()).isZero(); + assertThat(t.getSchemaStatus()).isEqualTo("MATCH"); + List lines = RestoreSummaries.lines(insertNew, false); + assertThat(RestoreTestSupport.line(lines, false, "records will be inserted into blogPost").getCount()).isEqualTo(10); + assertThat(RestoreTestSupport.line(lines, false, "records are unchanged in blogPost").getCount()).isEqualTo(10); + assertThat(RestoreTestSupport.line(lines, false, "left alone in blogPost").getCount()).isEqualTo(10); + assertThat(RestoreTestSupport.line(lines, false, "key sequence")).isNotNull(); + + RestoreOutcome upsert = new RestoreEngine().review(RestoreTestSupport.request(run, RestoreRequest.MODE_UPSERT, "blogPost")); + t = upsert.getTables().get("blogPost"); + assertThat(t.getInserted()).isEqualTo(10); + assertThat(t.getUpdated()).isEqualTo(10); + assertThat(t.getUnchanged()).isEqualTo(10); + assertThat(t.getSkipped()).isZero(); + assertThat(t.getDeleted()).isZero(); + lines = RestoreSummaries.lines(upsert, false); + assertThat(RestoreTestSupport.line(lines, false, "records will be updated in blogPost").getCount()).isEqualTo(10); + assertThat(t.getSamples()).hasSize(10); + assertThat(t.getSamples().get(0).changes()).contains("title: Edited 11 -> Post 11"); + + RestoreOutcome replace = new RestoreEngine().review(RestoreTestSupport.request(run, RestoreRequest.MODE_REPLACE, "blogPost").withConfirmationText("blogPost")); + assertThat(replace.getErrors()).isEmpty(); + t = replace.getTables().get("blogPost"); + assertThat(t.getDeleted()).isEqualTo(10); + assertThat(t.getInserted()).isEqualTo(10); + assertThat(t.getUpdated()).isEqualTo(10); + assertThat(t.getUnchanged()).isEqualTo(10); + ProcessSummaryLine deleted = RestoreTestSupport.line(RestoreSummaries.lines(replace, false), false, "will be deleted from blogPost (REPLACE)"); + assertThat(deleted.getCount()).isEqualTo(10); + + assertThat(RestoreTestSupport.rows("blogPost")).as("Review writes nothing").hasSize(30); + assertThat(new BackupCatalog().query(BackupsTables.RESTORE_RUN, new com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter())).isEmpty(); + } + + + + /******************************************************************************* + ** INSERT_NEW brings back the deleted rows and leaves edits and additions alone. + *******************************************************************************/ + @Test + void testExecuteInsertNew() throws Exception + { + BackupRunResult run = RestoreTestSupport.backup(Map.of()); + RestoreTestSupport.mutate(); + + RestoreOutcome outcome = new RestoreEngine().execute(RestoreTestSupport.request(run, RestoreRequest.MODE_INSERT_NEW, "blogPost")); + assertThat(outcome.getTables().get("blogPost").getInserted()).isEqualTo(10); + + List rows = RestoreTestSupport.rows("blogPost"); + assertThat(rows).hasSize(40); + assertThat(rows.get(0).getValueString("title")).isEqualTo("Post 1"); + assertThat(rows.stream().filter(r -> r.getValueInteger("id") == 15).findFirst().orElseThrow().getValueString("title")).isEqualTo("Edited 15"); + } + + + + /******************************************************************************* + ** UPSERT rolls edits back and restores deleted rows, keeps additions; the file that + ** was removed comes back with the right checksum, and a file with different content + ** is kept (a conflict). + *******************************************************************************/ + @Test + void testExecuteUpsertWithFiles() throws Exception + { + BackupRunResult run = RestoreTestSupport.backup(Map.of()); + RestoreTestSupport.mutate(); + String original1 = "photo bytes 1"; + BackupsTestUtils.MEDIA.removeIgnoringLock("media/photo-1.jpg"); + BackupsTestUtils.MEDIA.putBytes("media/photo-2.jpg", "tampered".getBytes()); + + RestoreOutcome review = new RestoreEngine().review(RestoreTestSupport.request(run, RestoreRequest.MODE_UPSERT, "blogPost")); + RestoreOutcome.TableOutcome rt = review.getTables().get("blogPost"); + assertThat(rt.getFilesToRestore()).isEqualTo(1); + assertThat(rt.getFilesPresent()).isEqualTo(1); + assertThat(rt.getFilesConflicting()).isEqualTo(1); + assertThat(rt.getFilesMissing()).isZero(); + assertThat(RestoreTestSupport.line(RestoreSummaries.lines(review, false), false, "different content for blogPost")).isNotNull(); + + RestoreOutcome outcome = new RestoreEngine().execute(RestoreTestSupport.request(run, RestoreRequest.MODE_UPSERT, "blogPost")); + RestoreOutcome.TableOutcome t = outcome.getTables().get("blogPost"); + assertThat(t.getInserted()).isEqualTo(10); + assertThat(t.getUpdated()).isEqualTo(10); + assertThat(t.getFilesRestored()).isEqualTo(1); + assertThat(outcome.getFilesFailed()).isZero(); + + List rows = RestoreTestSupport.rows("blogPost"); + assertThat(rows).hasSize(40); + assertThat(rows.stream().filter(r -> r.getValueInteger("id") == 15).findFirst().orElseThrow().getValueString("title")).isEqualTo("Post 15"); + + byte[] restored = BackupsTestUtils.MEDIA.getBytes("media/photo-1.jpg"); + assertThat(new String(restored)).isEqualTo(original1); + assertThat(HashingOutputStream.sha256Hex(restored)).isEqualTo(HashingOutputStream.sha256Hex(original1.getBytes())); + assertThat(new String(BackupsTestUtils.MEDIA.getBytes("media/photo-2.jpg"))).as("a conflicting file is kept").isEqualTo("tampered"); + assertThat(BackupsTestUtils.MEDIA.list("media/").stream().filter(o -> o.key().contains("qqq-restore"))).as("no temporary object is left").isEmpty(); + } + + + + /******************************************************************************* + ** REPLACE needs the typed confirmation, deletes what is not in the backup, takes a + ** pre-restore backup first, brings the table back to exactly the backed up rows + ** (same hash), overwrites a conflicting file, and records the run rows. + *******************************************************************************/ + @Test + void testExecuteReplace() throws Exception + { + BackupRunResult run = RestoreTestSupport.backup(Map.of()); + String originalHash = RestoreTestSupport.contentHash("blogPost"); + RestoreTestSupport.mutate(); + assertThat(RestoreTestSupport.contentHash("blogPost")).isNotEqualTo(originalHash); + BackupsTestUtils.MEDIA.putBytes("media/photo-2.jpg", "tampered".getBytes()); + + RestoreRequest request = RestoreTestSupport.request(run, RestoreRequest.MODE_REPLACE, "blogPost"); + assertThatThrownBy(() -> new RestoreEngine().execute(request)).hasMessageContaining("type blogPost"); + assertThat(RestoreTestSupport.rows("blogPost")).hasSize(30); + + RestoreOutcome outcome = new RestoreEngine().execute(request.withConfirmationText("blogPost")); + RestoreOutcome.TableOutcome t = outcome.getTables().get("blogPost"); + assertThat(t.getDeleted()).isEqualTo(10); + assertThat(t.getInserted()).isEqualTo(10); + assertThat(t.getUpdated()).isEqualTo(10); + + assertThat(RestoreTestSupport.rows("blogPost")).hasSize(30); + assertThat(RestoreTestSupport.contentHash("blogPost")).as("the rows are exactly the backed up rows again").isEqualTo(originalHash); + assertThat(new String(BackupsTestUtils.MEDIA.getBytes("media/photo-2.jpg"))).as("REPLACE overwrites a conflicting file").isEqualTo("photo bytes 2"); + + QRecord restoreRun = new BackupCatalog().get(BackupsTables.RESTORE_RUN, outcome.getRestoreRunId()).orElseThrow(); + assertThat(restoreRun.getValueString("status")).isIn("SUCCEEDED", "SUCCEEDED_WITH_WARNINGS"); + assertThat(restoreRun.getValueString("mode")).isEqualTo("REPLACE"); + assertThat(restoreRun.getValueLong("rowsDeleted")).isEqualTo(10L); + assertThat(restoreRun.getValueLong("rowsInserted")).isEqualTo(10L); + assertThat(restoreRun.getValueInteger("filesRestored")).isEqualTo(1); + assertThat(restoreRun.getValueString("confirmationText")).isEqualTo("blogPost"); + assertThat(restoreRun.getValueInteger("preRestoreBackupRunId")).isNotNull(); + QRecord pre = new BackupCatalog().get(BackupsTables.RUN, restoreRun.getValueInteger("preRestoreBackupRunId")).orElseThrow(); + assertThat(pre.getValueString("trigger")).isEqualTo("PRE_RESTORE"); + assertThat(pre.getValueString("status")).startsWith("SUCCEEDED"); + List tableRows = new BackupCatalog().where(BackupsTables.RESTORE_RUN_TABLE, "backupRestoreRunId", outcome.getRestoreRunId()); + assertThat(tableRows).hasSize(1); + assertThat(tableRows.get(0).getValueString("tableName")).isEqualTo("blogPost"); + assertThat(tableRows.get(0).getValueLong("rowsDeleted")).isEqualTo(10L); + assertThat(tableRows.get(0).getValueString("schemaStatus")).isEqualTo("MATCH"); + } + + + + /******************************************************************************* + ** Turning the pre-restore backup off is allowed and is recorded on the restore run. + *******************************************************************************/ + @Test + void testPreRestoreBackupCanBeTurnedOff() throws Exception + { + BackupRunResult run = RestoreTestSupport.backup(Map.of()); + RestoreTestSupport.mutate(); + RestoreOutcome outcome = new RestoreEngine().execute(RestoreTestSupport.request(run, RestoreRequest.MODE_INSERT_NEW, "blogPost").withPreRestoreBackup(false)); + QRecord restoreRun = new BackupCatalog().get(BackupsTables.RESTORE_RUN, outcome.getRestoreRunId()).orElseThrow(); + assertThat(restoreRun.getValueInteger("preRestoreBackupRunId")).isNull(); + assertThat(restoreRun.getValueString("scopeSummary")).contains("no pre-restore backup"); + } + + + + /******************************************************************************* + ** REMAP_NEW_IDS lets the target assign ids and rewrites the references of later + ** tables through the map; the id map is cleared when the restore ends. + *******************************************************************************/ + @Test + void testRemapNewIds() throws Exception + { + BackupRunResult run = RestoreTestSupport.backup(Map.of()); + + RestoreRequest request = RestoreTestSupport.request(run, RestoreRequest.MODE_INSERT_NEW, "author", "blogPost").withIdStrategy(RestoreRequest.IDS_REMAP); + RestoreOutcome review = new RestoreEngine().review(request); + assertThat(review.getTables().get("author").getInserted()).isEqualTo(3); + assertThat(review.getTables().get("blogPost").getInserted()).isEqualTo(30); + assertThat(review.getTables().get("blogPost").getOrphans()).as("the parents are in the backup").isZero(); + + RestoreOutcome outcome = new RestoreEngine().execute(request); + assertThat(outcome.getTables().get("author").getInserted()).isEqualTo(3); + + List authors = RestoreTestSupport.rows("author"); + assertThat(authors).hasSize(6); + List posts = RestoreTestSupport.rows("blogPost"); + assertThat(posts).hasSize(60); + QRecord newPost1 = posts.stream().filter(p -> p.getValueInteger("id") > 30 && "Post 1".equals(p.getValueString("title"))).findFirst().orElseThrow(); + QRecord newAuthor = authors.stream().filter(a -> a.getValueInteger("id") == newPost1.getValueInteger("authorId")).findFirst().orElseThrow(); + assertThat(newAuthor.getValueInteger("id")).as("the post points at a new author").isGreaterThan(3); + assertThat(newAuthor.getValueString("name")).as("old author 2 was Grace").isEqualTo("Grace"); + assertThat(new BackupCatalog().query(BackupsTables.ID_MAP, new com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter())).isEmpty(); + assertThat(new BackupCatalog().get(BackupsTables.RESTORE_RUN, outcome.getRestoreRunId()).orElseThrow().getValueString("idStrategy")).isEqualTo("REMAP_NEW_IDS"); + } + + + + /******************************************************************************* + ** REMAP with a unique key reuses rows that already exist (the target ids differ), + ** and the later tables point at the existing parents. + *******************************************************************************/ + @Test + void testRemapMatchingOnAUniqueKey() throws Exception + { + RestoreTestSupport.uniqueKey("author", "name"); + BackupRunResult run = RestoreTestSupport.backup(Map.of()); + + com.kingsrook.qqq.backend.core.actions.tables.DeleteAction delete = new com.kingsrook.qqq.backend.core.actions.tables.DeleteAction(); + for(String table : List.of("blogPost", "author")) + { + com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput input = new com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput(table); + List ids = new ArrayList<>(); + for(QRecord row : RestoreTestSupport.rows(table)) + { + ids.add(row.getValue("id")); + } + input.setPrimaryKeys(ids); + delete.execute(input); + } + BackupsTestUtils.insert("author", List.of(Map.of("id", 10, "name", "Linus"), Map.of("id", 20, "name", "Ada", "bio", "other bio"), Map.of("id", 30, "name", "Grace", "bio", ""))); + + RestoreRequest request = RestoreTestSupport.request(run, RestoreRequest.MODE_UPSERT, "author", "blogPost").withIdStrategy(RestoreRequest.IDS_REMAP).withMatchOn("name"); + RestoreOutcome review = new RestoreEngine().review(request); + assertThat(review.getTables().get("author").getInserted()).isZero(); + assertThat(review.getTables().get("author").getUpdated()).as("only Ada's bio differs").isEqualTo(1); + assertThat(review.getTables().get("author").getUnchanged()).isEqualTo(2); + + new RestoreEngine().execute(request); + assertThat(RestoreTestSupport.rows("author")).hasSize(3); + assertThat(RestoreTestSupport.rows("author").stream().filter(a -> a.getValueInteger("id") == 20).findFirst().orElseThrow().getValueString("bio")) + .startsWith("Line one"); + List posts = RestoreTestSupport.rows("blogPost"); + assertThat(posts).hasSize(30); + QRecord post1 = posts.stream().filter(p -> "Post 1".equals(p.getValueString("title"))).findFirst().orElseThrow(); + assertThat(post1.getValueInteger("authorId")).as("Post 1 belonged to Grace, who is now id 30").isEqualTo(30); + } + + + + /******************************************************************************* + ** Unique-key conflicts and orphaned references are counted in Review. + *******************************************************************************/ + @Test + void testConflictsAndOrphans() throws Exception + { + RestoreTestSupport.uniqueKey("author", "name"); + BackupRunResult run = RestoreTestSupport.backup(Map.of()); + + com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput input = new com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput("author"); + input.setPrimaryKeys(List.of(1, 2)); + new com.kingsrook.qqq.backend.core.actions.tables.DeleteAction().execute(input); + BackupsTestUtils.insert("author", List.of(Map.of("id", 5, "name", "Ada"))); + + RestoreOutcome authors = new RestoreEngine().review(RestoreTestSupport.request(run, RestoreRequest.MODE_INSERT_NEW, "author")); + RestoreOutcome.TableOutcome a = authors.getTables().get("author"); + assertThat(a.getConflicts()).as("Ada id 1 collides with the existing Ada id 5").isEqualTo(1); + assertThat(a.getInserted()).as("Grace id 2 can come back").isEqualTo(1); + assertThat(a.getUnchanged()).isEqualTo(1); + assertThat(RestoreTestSupport.line(RestoreSummaries.lines(authors, false), false, "conflicts on a unique key")).isNotNull(); + + RestoreOutcome posts = new RestoreEngine().review(RestoreTestSupport.request(run, RestoreRequest.MODE_INSERT_NEW, "blogPost")); + assertThat(posts.getTables().get("blogPost").getOrphans()).as("posts of author 1 and 2: those ids are in neither the backup selection nor the target").isEqualTo(20); + assertThat(RestoreTestSupport.line(RestoreSummaries.lines(posts, false), false, "reference a parent that is missing")).isNotNull(); + } + + + + /******************************************************************************* + ** The schema is compared with the backup: extra and removed fields are compatible + ** (the removed ones are skipped); a changed type blocks that table. + *******************************************************************************/ + @Test + void testSchemaComparison() throws Exception + { + BackupRunResult run = RestoreTestSupport.backup(Map.of()); + RestoreTestSupport.mutate(); + + com.kingsrook.qqq.backend.core.context.QContext.getQInstance().getTable("author").getFields().remove("bio"); + com.kingsrook.qqq.backend.core.context.QContext.getQInstance().getTable("blogPost").getField("price").setType(QFieldType.STRING); + + RestoreOutcome review = new RestoreEngine().review(RestoreTestSupport.request(run, RestoreRequest.MODE_UPSERT, "author", "blogPost")); + RestoreOutcome.TableOutcome author = review.getTables().get("author"); + assertThat(author.getSchemaStatus()).isEqualTo("COMPATIBLE"); + assertThat(author.getUnmappedFields()).containsExactly("bio"); + assertThat(RestoreTestSupport.line(RestoreSummaries.lines(review, false), false, "field bio is in the backup but not in the target and will be skipped")).isNotNull(); + + RestoreOutcome.TableOutcome posts = review.getTables().get("blogPost"); + assertThat(posts.getSchemaStatus()).isEqualTo("MISMATCH"); + assertThat(posts.getStatus()).isEqualTo("SKIPPED"); + assertThat(posts.getInserted()).isZero(); + assertThat(RestoreTestSupport.line(RestoreSummaries.lines(review, false), false, "changed type from DECIMAL to STRING").getStatus()) + .isEqualTo(com.kingsrook.qqq.backend.core.model.actions.processes.Status.ERROR); + + RestoreOutcome outcome = new RestoreEngine().execute(RestoreTestSupport.request(run, RestoreRequest.MODE_UPSERT, "author", "blogPost")); + assertThat(outcome.getTables().get("blogPost").getStatus()).isEqualTo("SKIPPED"); + assertThat(RestoreTestSupport.rows("blogPost")).as("the incompatible table is left alone").hasSize(30); + List tableRows = new BackupCatalog().where(BackupsTables.RESTORE_RUN_TABLE, "backupRestoreRunId", outcome.getRestoreRunId()); + assertThat(tableRows).extracting(r -> r.getValueString("schemaStatus")).containsExactlyInAnyOrder("COMPATIBLE", "MISMATCH"); + } + + + + /******************************************************************************* + ** Sensitive values are masked in the sample diff, even when they are in the backup. + *******************************************************************************/ + @Test + void testSensitiveValuesAreNeverShown() throws Exception + { + BackupRunResult run = RestoreTestSupport.backup(Map.of("sensitiveFieldPolicy", "INCLUDE")); + UpdateInput update = new UpdateInput("blogPost"); + update.setRecords(List.of(new QRecord().withValue("id", 1).withValue("secretNote", "brand-new-secret"))); + new UpdateAction().execute(update); + + RestoreOutcome review = new RestoreEngine().review(RestoreTestSupport.request(run, RestoreRequest.MODE_UPSERT, "blogPost")); + RestoreOutcome.TableOutcome t = review.getTables().get("blogPost"); + assertThat(t.getUpdated()).isEqualTo(1); + assertThat(t.getSamples()).hasSize(1); + assertThat(t.getSamples().get(0).changes()).containsExactly("secretNote: ******** -> ********"); + + String everything = RestoreSummaries.lines(review, false).stream().map(l -> String.valueOf(((ProcessSummaryLine) l).getBulletsOfText())).reduce("", String::concat); + assertThat(everything).doesNotContain("brand-new-secret").doesNotContain("do-not-leak"); + } + + + + /******************************************************************************* + ** Per table filters restrict the restore to selected records. + *******************************************************************************/ + @Test + void testFilteredRestore() throws Exception + { + BackupRunResult run = RestoreTestSupport.backup(Map.of()); + RestoreTestSupport.mutate(); + + RestoreRequest request = RestoreTestSupport.request(run, RestoreRequest.MODE_UPSERT, "blogPost") + .withFilter("blogPost", "{\"criteria\":[{\"fieldName\":\"id\",\"operator\":\"IN\",\"values\":[2,3,12]}]}"); + RestoreOutcome outcome = new RestoreEngine().execute(request); + RestoreOutcome.TableOutcome t = outcome.getTables().get("blogPost"); + assertThat(t.getRows()).isEqualTo(3); + assertThat(t.getInserted()).isEqualTo(2); + assertThat(t.getUpdated()).isEqualTo(1); + assertThat(RestoreTestSupport.rows("blogPost")).hasSize(32); + } + + + + /******************************************************************************* + ** Writes go through the RestoreWriter in page transactions. + *******************************************************************************/ + @Test + void testWritesUseThePluggableWriterInPageTransactions() throws Exception + { + BackupRunResult run = RestoreTestSupport.backup(Map.of()); + RestoreTestSupport.mutate(); + + List calls = new ArrayList<>(); + StandardRestoreWriter standard = new StandardRestoreWriter(); + RestoreWriter recording = new RestoreWriter() + { + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public List insert(com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData table, List records, RestoreRequest options, QBackendTransaction tx) + throws com.kingsrook.qqq.backend.core.exceptions.QException + { + calls.add("insert:" + records.size() + ":" + (tx != null)); + return (standard.insert(table, records, options, tx)); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public List update(com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData table, List records, RestoreRequest options, QBackendTransaction tx) + throws com.kingsrook.qqq.backend.core.exceptions.QException + { + calls.add("update:" + records.size() + ":" + (tx != null)); + return (standard.update(table, records, options, tx)); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public long delete(com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData table, List primaryKeys, RestoreRequest options, QBackendTransaction tx) + throws com.kingsrook.qqq.backend.core.exceptions.QException + { + return (standard.delete(table, primaryKeys, options, tx)); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public boolean usesCoreRestoreFlag() + { + return (standard.usesCoreRestoreFlag()); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public boolean canResyncSequences() + { + return (true); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public List resyncSequences(java.util.Collection tables) + throws com.kingsrook.qqq.backend.core.exceptions.QException + { + tables.forEach(t -> calls.add("resync:" + t.getName())); + return (tables.stream().map(com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData::getName).toList()); + } + }; + + RestoreOutcome outcome = new RestoreEngine().execute(RestoreTestSupport.request(run, RestoreRequest.MODE_UPSERT, "blogPost").withWriter(recording).withPageSize(8)); + assertThat(calls).contains("insert:8:true", "insert:2:true", "update:6:true", "update:4:true"); + assertThat(calls).filteredOn(c -> !c.startsWith("resync")).allMatch(c -> c.endsWith(":true")); + assertThat(calls).as("the key sequence of the table that got its original ids is resynced, after the writes").endsWith("resync:blogPost"); + assertThat(outcome.getSequencesResynced()).isEqualTo(1); + assertThat(new BackupCatalog().get(BackupsTables.RESTORE_RUN, outcome.getRestoreRunId()).orElseThrow().getValueInteger("sequencesResynced")).isEqualTo(1); + } +} diff --git a/src/test/java/com/kingsrook/qbits/backups/restore/RestoreProcessTest.java b/src/test/java/com/kingsrook/qbits/backups/restore/RestoreProcessTest.java new file mode 100644 index 0000000..30bd91e --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/backups/restore/RestoreProcessTest.java @@ -0,0 +1,249 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.restore; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.backups.BackupsTestUtils; +import com.kingsrook.qbits.backups.model.BackupsTables; +import com.kingsrook.qbits.backups.run.BackupCatalog; +import com.kingsrook.qbits.backups.run.BackupRunResult; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** The restore and download processes, run the way the frontend runs them: the + ** first screen, Review (the validation summary), Execute and Result. + *******************************************************************************/ +class RestoreProcessTest +{ + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void setUp() throws Exception + { + BackupsTestUtils.init(); + BackupsTestUtils.loadBlog(30); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + static List lines(RunProcessOutput output, String field) + { + return (new ArrayList<>((List) output.getValues().get(field))); + } + + + + /******************************************************************************* + ** Restore... on a backup run: Review shows the per-table lines and a sample diff, + ** Execute restores, Result shows what was done. + *******************************************************************************/ + @Test + void testRestoreProcessEndToEnd() throws Exception + { + BackupRunResult run = RestoreTestSupport.backup(Map.of()); + RestoreTestSupport.mutate(); + + RunProcessInput input = new RunProcessInput(); + input.setProcessName("backupRestore"); + input.addValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, run.getRunId()))); + input.addValue("tables", "blogPost"); + input.addValue("mode", "UPSERT"); + + RunProcessOutput output = new RunProcessAction().execute(input); + assertThat(output.getException()).isEmpty(); + assertThat(output.getProcessState().getNextStepName()).as("the first screen asks for the options").isPresent().get().isEqualTo("input"); + input.addValue("processUUID", output.getProcessUUID()); + + input.setStartAfterStep("input"); + output = new RunProcessAction().execute(input); + assertThat(output.getException()).isEmpty(); + assertThat(output.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review"); + + List review = lines(output, StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY); + assertThat(RestoreTestSupport.line(review, false, "records will be inserted into blogPost").getCount()).isEqualTo(10); + assertThat(RestoreTestSupport.line(review, false, "records will be updated in blogPost").getCount()).isEqualTo(10); + assertThat(RestoreTestSupport.line(review, false, "records are unchanged in blogPost").getCount()).isEqualTo(10); + ProcessSummaryLine sample = RestoreTestSupport.line(review, false, "changed records in blogPost"); + assertThat(sample.getBulletsOfText()).hasSize(10); + assertThat(sample.getBulletsOfText().get(0)).contains("title: Edited 11 -> Post 11"); + assertThat(RestoreTestSupport.rows("blogPost")).as("Review writes nothing").hasSize(30); + + input.setStartAfterStep("review"); + output = new RunProcessAction().execute(input); + assertThat(output.getException()).isEmpty(); + assertThat(output.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result"); + + List result = lines(output, StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY); + assertThat(RestoreTestSupport.line(result, true, "records were inserted into blogPost").getCount()).isEqualTo(10); + assertThat(RestoreTestSupport.line(result, true, "records were updated in blogPost").getCount()).isEqualTo(10); + assertThat(RestoreTestSupport.line(result, true, "A backup of the target tables was taken first")).isNotNull(); + assertThat(result.stream().map(l -> ((ProcessSummaryLine) l).getStatus())).doesNotContain(Status.ERROR); + + assertThat(RestoreTestSupport.rows("blogPost")).hasSize(40); + assertThat(new BackupCatalog().where(BackupsTables.RESTORE_RUN, "sourceBackupRunId", run.getRunId())).hasSize(1); + } + + + + /******************************************************************************* + ** Restore this table, on a table's data artifact, restores just that table. + *******************************************************************************/ + @Test + void testRestoreThisTableOnAnArtifact() throws Exception + { + BackupRunResult run = RestoreTestSupport.backup(Map.of()); + RestoreTestSupport.mutate(); + QRecord artifact = new BackupCatalog().where(BackupsTables.ARTIFACT, "backupRunId", run.getRunId()).stream() + .filter(a -> "blogPost".equals(a.getValueString("tableName"))).findFirst().orElseThrow(); + + RunProcessInput input = new RunProcessInput(); + input.setProcessName("backupRestoreTable"); + input.addValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, artifact.getValueInteger("id")))); + input.addValue("mode", "INSERT_NEW"); + + RunProcessOutput output = new RunProcessAction().execute(input); + input.addValue("processUUID", output.getProcessUUID()); + input.setStartAfterStep("input"); + output = new RunProcessAction().execute(input); + assertThat(output.getException()).isEmpty(); + assertThat(RestoreTestSupport.line(lines(output, StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY), false, "records will be inserted into blogPost").getCount()).isEqualTo(10); + assertThat(RestoreTestSupport.line(lines(output, StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY), false, "author")).as("only that table").isNull(); + + input.setStartAfterStep("review"); + new RunProcessAction().execute(input); + assertThat(RestoreTestSupport.rows("blogPost")).hasSize(40); + } + + + + /******************************************************************************* + ** The Review of a REPLACE without the typed confirmation is an error line, and + ** Execute refuses. + *******************************************************************************/ + @Test + void testReplaceWithoutConfirmationIsRefused() throws Exception + { + BackupRunResult run = RestoreTestSupport.backup(Map.of()); + RestoreTestSupport.mutate(); + + RunProcessInput input = new RunProcessInput(); + input.setProcessName("backupRestore"); + input.addValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, run.getRunId()))); + input.addValue("tables", "blogPost"); + input.addValue("mode", "REPLACE"); + + RunProcessOutput output = new RunProcessAction().execute(input); + input.addValue("processUUID", output.getProcessUUID()); + input.setStartAfterStep("input"); + output = new RunProcessAction().execute(input); + ProcessSummaryLine error = RestoreTestSupport.line(lines(output, StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY), false, "To confirm a REPLACE, type blogPost"); + assertThat(error).isNotNull(); + assertThat(error.getStatus()).isEqualTo(Status.ERROR); + + input.setStartAfterStep("review"); + assertThatThrownBy(() -> new RunProcessAction().execute(input)).hasMessageContaining("To confirm a REPLACE, type blogPost"); + assertThat(RestoreTestSupport.rows("blogPost")).hasSize(30); + } + + + + /******************************************************************************* + ** Download: presigned links for the run's objects (ciphertext as stored), each one + ** a separate link, with the manifest and checksums among them. + *******************************************************************************/ + @Test + void testDownloadIssuesLinks() throws Exception + { + BackupRunResult run = RestoreTestSupport.backup(Map.of()); + int artifacts = new BackupCatalog().where(BackupsTables.ARTIFACT, "backupRunId", run.getRunId()).size(); + + RunProcessInput input = new RunProcessInput(); + input.setProcessName("backupDownload"); + input.addValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, run.getRunId()))); + + RunProcessOutput output = new RunProcessAction().execute(input); + assertThat(output.getException()).isEmpty(); + input.addValue("processUUID", output.getProcessUUID()); + if(output.getProcessState().getNextStepName().orElse("").equals("review")) + { + input.setStartAfterStep("review"); + } + else + { + input.setStartAfterStep(output.getProcessState().getNextStepName().orElse(null)); + } + + List review = lines(output, StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY); + assertThat(RestoreTestSupport.line(review, false, "files will get download links").getCount()).isEqualTo(artifacts); + + output = new RunProcessAction().execute(input); + assertThat(output.getException()).isEmpty(); + ProcessSummaryLine issued = RestoreTestSupport.line(lines(output, StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY), true, "download links were issued"); + assertThat(issued.getCount()).isEqualTo(artifacts); + assertThat(issued.getBulletsOfText()).hasSize(artifacts); + assertThat(issued.getBulletsOfText()).anyMatch(b -> b.contains("manifest.json") && b.contains("memory://") && b.contains("ttl=900")); + assertThat(issued.getBulletsOfText()).anyMatch(b -> b.contains("checksums.sha256")); + } + + + + /******************************************************************************* + ** Download on one artifact links that artifact plus the manifest and checksums. + *******************************************************************************/ + @Test + void testDownloadOfOneArtifact() throws Exception + { + BackupRunResult run = RestoreTestSupport.backup(Map.of()); + QRecord artifact = new BackupCatalog().where(BackupsTables.ARTIFACT, "backupRunId", run.getRunId()).stream() + .filter(a -> "blogPost".equals(a.getValueString("tableName"))).findFirst().orElseThrow(); + + RunProcessInput input = new RunProcessInput(); + input.setProcessName("backupDownloadArtifact"); + input.addValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, artifact.getValueInteger("id")))); + RunProcessOutput output = new RunProcessAction().execute(input); + input.addValue("processUUID", output.getProcessUUID()); + input.setStartAfterStep("review"); + output = new RunProcessAction().execute(input); + + ProcessSummaryLine issued = RestoreTestSupport.line(lines(output, StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY), true, "download links were issued"); + assertThat(issued.getCount()).isEqualTo(3); + assertThat(issued.getBulletsOfText()).anyMatch(b -> b.contains("blogPost-part-00001")); + } +} diff --git a/src/test/java/com/kingsrook/qbits/backups/restore/RestoreSafetyTest.java b/src/test/java/com/kingsrook/qbits/backups/restore/RestoreSafetyTest.java new file mode 100644 index 0000000..acf465c --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/backups/restore/RestoreSafetyTest.java @@ -0,0 +1,190 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.restore; + + +import java.io.Serializable; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.backups.BackupsRuntime; +import com.kingsrook.qbits.backups.BackupsTestUtils; +import com.kingsrook.qbits.backups.model.BackupsTables; +import com.kingsrook.qbits.backups.run.BackupCatalog; +import com.kingsrook.qbits.backups.run.BackupRunResult; +import com.kingsrook.qbits.backups.scope.BackupTableMetaData; +import com.kingsrook.qbits.backups.scope.TableRole; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** The safety rails of restore (spec 9.4) and who may run it (spec 11). + *******************************************************************************/ +class RestoreSafetyTest +{ + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void setUp() throws Exception + { + BackupsTestUtils.init(); + BackupsTestUtils.loadBlog(12); + } + + + + /******************************************************************************* + ** A restore is refused while a backup that covers the same tables is running. + *******************************************************************************/ + @Test + void testRefusesWhileABackupOfTheSameTablesIsRunning() throws Exception + { + BackupRunResult run = RestoreTestSupport.backup(Map.of()); + Integer planId = new BackupCatalog().where(BackupsTables.PLAN, "planKey", "nightly").get(0).getValueInteger("id"); + + Map live = new LinkedHashMap<>(); + live.put("backupPlanId", planId); + live.put("runKey", "live-one"); + live.put("status", "RUNNING"); + live.put("startDate", BackupsRuntime.getClock().instant()); + live.put("heartbeatDate", BackupsRuntime.getClock().instant()); + Integer liveId = new BackupCatalog().insert(BackupsTables.RUN, live); + + RestoreRequest request = RestoreTestSupport.request(run, RestoreRequest.MODE_INSERT_NEW, "blogPost"); + assertThat(new RestoreEngine().plan(request).errors()).anyMatch(e -> e.contains("is running")); + assertThatThrownBy(() -> new RestoreEngine().execute(request)).hasMessageContaining("is running"); + assertThat(new BackupCatalog().where(BackupsTables.RESTORE_RUN, "sourceBackupRunId", run.getRunId())).as("nothing was started").isEmpty(); + + new BackupCatalog().update(BackupsTables.RUN, liveId, Map.of("heartbeatDate", BackupsRuntime.getClock().instant().minusSeconds(3600))); + assertThat(new RestoreEngine().plan(request).errors()).as("a run with an old heartbeat is not running").isEmpty(); + } + + + + /******************************************************************************* + ** REPLACE needs the typed confirmation: the table name, or REPLACE n TABLES. + *******************************************************************************/ + @Test + void testReplaceNeedsTheTypedConfirmation() throws Exception + { + BackupRunResult run = RestoreTestSupport.backup(Map.of()); + + RestoreRequest one = RestoreTestSupport.request(run, RestoreRequest.MODE_REPLACE, "blogPost"); + assertThat(new RestoreEngine().plan(one).errors()).anyMatch(e -> e.contains("type blogPost")); + assertThat(new RestoreEngine().plan(one.withConfirmationText("blogpost")).errors()).as("case matters").isNotEmpty(); + assertThat(new RestoreEngine().plan(one.withConfirmationText("blogPost")).errors()).isEmpty(); + + RestoreRequest two = RestoreTestSupport.request(run, RestoreRequest.MODE_REPLACE, "author", "blogPost"); + assertThat(new RestoreEngine().plan(two).errors()).anyMatch(e -> e.contains("REPLACE 2 TABLES")); + assertThat(new RestoreEngine().plan(two.withConfirmationText("REPLACE 2 TABLES")).errors()).isEmpty(); + + assertThat(new RestoreEngine().plan(RestoreTestSupport.request(run, RestoreRequest.MODE_REPLACE, "blogPost").withConfirmationText("blogPost") + .withIdStrategy(RestoreRequest.IDS_REMAP)).errors()).anyMatch(e -> e.contains("REPLACE cannot be combined with new ids")); + } + + + + /******************************************************************************* + ** REPLACE also needs backups.admin, not just backups.restore. + *******************************************************************************/ + @Test + void testReplaceNeedsAdmin() throws Exception + { + BackupRunResult run = RestoreTestSupport.backup(Map.of()); + BackupsRuntime.getConfig().withOpenWhenAnonymous(false); + RestoreRequest replace = RestoreTestSupport.request(run, RestoreRequest.MODE_REPLACE, "blogPost").withConfirmationText("blogPost"); + + QContext.setQSession(new QSession().withPermissions("backups.view", "backups.restore")); + assertThat(new RestoreEngine().plan(RestoreTestSupport.request(run, RestoreRequest.MODE_UPSERT, "blogPost")).errors()).isEmpty(); + assertThat(new RestoreEngine().plan(replace).errors()).anyMatch(e -> e.contains("backups.admin")); + + QContext.setQSession(new QSession().withPermissions("backups.view", "backups.restore", "backups.admin")); + assertThat(new RestoreEngine().plan(replace).errors()).isEmpty(); + } + + + + /******************************************************************************* + ** Restores never touch SECRET or backups tables, and temporary (EPHEMERAL) tables + ** only when they are named. + *******************************************************************************/ + @Test + void testNeverRestoresSecretBackupsOrUnnamedEphemeralTables() throws Exception + { + BackupRunResult run = RestoreTestSupport.backup(Map.of("includeRoles", "DATA,AUDIT,EPHEMERAL")); + assertThat(RestoreTestSupport.manifest(run).table("userSession")).isNotNull(); + + RestoreEngine.Plan all = new RestoreEngine().plan(new RestoreRequest().withSourceRunId(run.getRunId())); + assertThat(all.errors()).isEmpty(); + assertThat(all.order()).containsExactly("author", "blogPost"); + assertThat(all.notes()).anyMatch(n -> n.contains("userSession") && n.contains("temporary")); + + RestoreEngine.Plan named = new RestoreEngine().plan(new RestoreRequest().withSourceRunId(run.getRunId()).withTables(List.of("userSession"))); + assertThat(named.order()).containsExactly("userSession"); + assertThat(named.warnings()).anyMatch(w -> w.contains("temporary data")); + + assertThat(new RestoreEngine().plan(new RestoreRequest().withSourceRunId(run.getRunId()).withTables(List.of("backups_plan"))).errors()) + .anyMatch(e -> e.contains("not in this backup")); + + BackupTableMetaData.ofOrWithNew(QContext.getQInstance().getTable("author")).withRole(TableRole.SECRET); + RestoreEngine.Plan secret = new RestoreEngine().plan(new RestoreRequest().withSourceRunId(run.getRunId()).withTables(List.of("author"))); + assertThat(secret.errors()).anyMatch(e -> e.contains("SECRET") && e.contains("never touch")); + } + + + + /******************************************************************************* + ** Security: without backups.restore nobody can run a restore, through the process + ** (the core permission check) or by calling the engine. With the permission they can. + *******************************************************************************/ + @Test + void testRestoreNeedsThePermission() throws Exception + { + BackupRunResult run = RestoreTestSupport.backup(Map.of()); + RestoreTestSupport.mutate(); + int rowsBefore = RestoreTestSupport.rows("blogPost").size(); + + BackupsRuntime.getConfig().withOpenWhenAnonymous(false); + QContext.getQInstance().setEnforcePermissionsInCoreActions(true); + QContext.setQSession(new QSession().withPermissions("backups.view", "backups.run")); + + RunProcessInput input = new RunProcessInput(); + input.setProcessName("backupRestore"); + input.addValue("tables", "blogPost"); + input.addValue("mode", "UPSERT"); + assertThatThrownBy(() -> new RunProcessAction().execute(input)).isInstanceOf(QPermissionDeniedException.class).hasMessageContaining("backups.restore"); + + RestoreRequest request = RestoreTestSupport.request(run, RestoreRequest.MODE_UPSERT, "blogPost"); + assertThatThrownBy(() -> new RestoreEngine().execute(request)).hasMessageContaining("backups.restore"); + assertThat(new RestoreEngine().review(request).getErrors()).anyMatch(e -> e.contains("backups.restore")); + + QContext.setQSession(new QSession()); + assertThat(RestoreTestSupport.rows("blogPost")).as("nothing was written").hasSize(rowsBefore); + + QContext.setQSession(new QSession().withPermissions("backups.view", "backups.restore")); + assertThat(new RestoreEngine().plan(request).errors()).isEmpty(); + } +} diff --git a/src/test/java/com/kingsrook/qbits/backups/restore/RestoreTestSupport.java b/src/test/java/com/kingsrook/qbits/backups/restore/RestoreTestSupport.java new file mode 100644 index 0000000..f9dbc5e --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/backups/restore/RestoreTestSupport.java @@ -0,0 +1,220 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.restore; + + +import java.io.Serializable; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.backups.BackupsTestUtils; +import com.kingsrook.qbits.backups.codec.HashingOutputStream; +import com.kingsrook.qbits.backups.codec.JsonlRowCodec; +import com.kingsrook.qbits.backups.manifest.BackupManifest; +import com.kingsrook.qbits.backups.model.BackupsTables; +import com.kingsrook.qbits.backups.run.BackupCatalog; +import com.kingsrook.qbits.backups.run.BackupEngine; +import com.kingsrook.qbits.backups.run.BackupRunFactory; +import com.kingsrook.qbits.backups.run.BackupRunResult; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Helpers shared by the restore tests: take a backup, mutate the data the way + ** the spec's acceptance does (delete 10, edit 10, add 10), and read tables back. + *******************************************************************************/ +final class RestoreTestSupport +{ + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private RestoreTestSupport() + { + } + + + + /******************************************************************************* + ** Create the nightly plan (with overrides) and run it. + *******************************************************************************/ + static BackupRunResult backup(Map overrides) throws Exception + { + Map values = new LinkedHashMap<>(Map.of("name", "Nightly", "planKey", "nightly", "isActive", true, + "scopeType", "INSTANCE", "storageBucketAlias", "backups", "slaHours", 26)); + values.putAll(overrides); + BackupCatalog catalog = new BackupCatalog(); + Integer planId = catalog.insert(BackupsTables.PLAN, values); + BackupRunResult result = new BackupEngine().execute(BackupRunFactory.forPlan(catalog.get(BackupsTables.PLAN, planId).orElseThrow(), "MANUAL")); + assertThat(result.isUsable()).as(result.getErrorMessage()).isTrue(); + return (result); + } + + + + /******************************************************************************* + ** Back up again with the existing nightly plan. + *******************************************************************************/ + static BackupRunResult backupAgain() throws Exception + { + QRecord plan = new BackupCatalog().where(BackupsTables.PLAN, "planKey", "nightly").get(0); + BackupRunResult result = new BackupEngine().execute(BackupRunFactory.forPlan(plan, "MANUAL")); + assertThat(result.isUsable()).as(result.getErrorMessage()).isTrue(); + return (result); + } + + + + /******************************************************************************* + ** The manifest of a run. + *******************************************************************************/ + static BackupManifest manifest(BackupRunResult result) throws Exception + { + return (BackupManifest.read(BackupsTestUtils.BACKUPS.getBytes(result.getManifestKey()))); + } + + + + /******************************************************************************* + ** Delete posts 1..10, edit posts 11..20 (title), add posts 31..40. + *******************************************************************************/ + static void mutate() throws Exception + { + DeleteInput delete = new DeleteInput("blogPost"); + List ids = new ArrayList<>(); + for(int i = 1; i <= 10; i++) + { + ids.add(i); + } + delete.setPrimaryKeys(ids); + new DeleteAction().execute(delete); + + List edits = new ArrayList<>(); + for(int i = 11; i <= 20; i++) + { + edits.add(new QRecord().withValue("id", i).withValue("title", "Edited " + i)); + } + UpdateInput update = new UpdateInput("blogPost"); + update.setRecords(edits); + new UpdateAction().execute(update); + + List> added = new ArrayList<>(); + for(int i = 31; i <= 40; i++) + { + Map row = new LinkedHashMap<>(); + row.put("id", i); + row.put("authorId", 1); + row.put("title", "Added " + i); + row.put("createDate", Instant.parse("2026-10-13T00:00:00Z")); + added.add(row); + } + BackupsTestUtils.insert("blogPost", added); + } + + + + /******************************************************************************* + ** All the rows of a table, by id. + *******************************************************************************/ + static List rows(String table) throws QException + { + QueryInput input = new QueryInput(table); + input.setFilter(new QQueryFilter().withOrderBy(new QFilterOrderBy("id", true))); + return (new QueryAction().execute(input).getRecords()); + } + + + + /******************************************************************************* + ** A SHA-256 over the canonical rows of a table (by id), leaving out the sensitive + ** fields (they are not backed up) and, until the core restore flag ships, the created + ** and modified dates (without the flag the standard actions set them to now). + *******************************************************************************/ + static String contentHash(String table) throws Exception + { + QTableMetaData meta = QContext.getQInstance().getTable(table); + List left = new ArrayList<>(); + meta.getFields().values().stream().filter(f -> f.getIsSensitive()).forEach(f -> left.add(f.getName())); + if(!new StandardRestoreWriter().usesCoreRestoreFlag()) + { + left.addAll(List.of("createDate", "modifyDate")); + } + JsonlRowCodec codec = new JsonlRowCodec(meta, left); + StringBuilder sb = new StringBuilder(); + for(QRecord row : rows(table)) + { + sb.append(codec.encode(row)).append('\n'); + } + return (HashingOutputStream.sha256Hex(sb.toString().getBytes(StandardCharsets.UTF_8))); + } + + + + /******************************************************************************* + ** A request over some tables, restoring from a run. + *******************************************************************************/ + static RestoreRequest request(BackupRunResult run, String mode, String... tables) + { + return (new RestoreRequest().withSourceRunId(run.getRunId()).withMode(mode).withTables(List.of(tables)).withRequestedBy("tester")); + } + + + + /******************************************************************************* + ** The summary line whose message contains the text. + *******************************************************************************/ + static ProcessSummaryLine line(List lines, boolean result, String text) + { + for(ProcessSummaryLineInterface line : lines) + { + ProcessSummaryLine candidate = (ProcessSummaryLine) line; + candidate.pickMessage(result); + if(candidate.getMessage() != null && candidate.getMessage().contains(text)) + { + return (candidate); + } + } + return (null); + } + + + + /******************************************************************************* + ** Declare a unique key on a table of the fixture. + *******************************************************************************/ + static void uniqueKey(String table, String... fields) + { + QContext.getQInstance().getTable(table).withUniqueKey(new UniqueKey(fields)); + } +} -- 2.54.0