From 7ef0849a0223026e273c77812d24a3431e22b4ec Mon Sep 17 00:00:00 2001 From: James Maes Date: Mon, 12 Oct 2026 15:59:28 -0500 Subject: [PATCH 1/2] ci: publish only GA tags (vX.Y.Z), never RC tags An RC tag matched /v.*/ and the orb republished it as the next RC. Co-Authored-By: Claude Opus 5.5 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 65ab99c..2b9734e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -57,7 +57,7 @@ workflows: branches: ignore: /.*/ tags: - only: [/v.*/] + only: [/^v\d+\.\d+\.\d+$/] publish_hotfix_release: jobs: -- 2.54.0 From eea9f638788e66f6bc065b188a921102cd8d1f6f Mon Sep 17 00:00:00 2001 From: James Maes Date: Mon, 12 Oct 2026 16:14:32 -0500 Subject: [PATCH 2/2] feat: backups foundation - tables, scope, codecs, manifest, backup engine, file copy, home Tables (spec 4), permissions, scope resolver and table roles, canonical JSONL and CSV codecs, per-part SHA-256 and rowsHash, manifest written last, the backup engine with referenced-file copy into the deduplicated store, a narrow storage interface with an in-memory implementation, Run backup now (streamed ETL), and the Backups app home. Stand-ins for unreleased core hooks are marked in the code. Co-Authored-By: Claude Sonnet 5.5 --- .../qbits/backups/BackupsQBitConfig.java | 232 +++++- .../qbits/backups/BackupsQBitProducer.java | 50 ++ .../qbits/backups/BackupsRuntime.java | 195 +++++ .../qbits/backups/alerts/BackupAlertSink.java | 65 ++ .../backups/alerts/NoOpBackupAlertSink.java | 56 ++ .../qbits/backups/app/BackupsApp.java | 152 ++++ .../qbits/backups/app/BackupsHomeData.java | 255 ++++++ .../qbits/backups/app/BackupsHomeView.java | 396 +++++++++ .../backups/codec/ArtifactTransform.java | 86 ++ .../qbits/backups/codec/CsvRowCodec.java | 277 +++++++ .../qbits/backups/codec/FieldValueCodec.java | 172 ++++ .../backups/codec/HashingOutputStream.java | 147 ++++ .../qbits/backups/codec/JsonlRowCodec.java | 200 +++++ .../qbits/backups/codec/RowCodec.java | 53 ++ .../backups/codec/TableArtifactWriter.java | 274 +++++++ .../qbits/backups/files/FileCopier.java | 223 +++++ .../qbits/backups/files/FileReference.java | 33 + .../backups/files/FileReferenceProvider.java | 72 ++ .../backups/manifest/BackupManifest.java | 219 +++++ .../backups/manifest/TableFingerprints.java | 184 +++++ .../qbits/backups/model/BackupsTables.java | 572 +++++++++++++ .../permissions/BackupsPermissionChecker.java | 103 +++ .../permissions/BackupsPermissions.java | 42 + .../backups/processes/BackupsProcesses.java | 263 ++++++ .../qbits/backups/run/BackupCatalog.java | 254 ++++++ .../qbits/backups/run/BackupEngine.java | 768 ++++++++++++++++++ .../qbits/backups/run/BackupRunFactory.java | 98 +++ .../qbits/backups/run/BackupRunRequest.java | 413 ++++++++++ .../qbits/backups/run/BackupRunResult.java | 407 ++++++++++ .../qbits/backups/run/TableReader.java | 128 +++ .../backups/scope/BackupTableMetaData.java | 147 ++++ .../qbits/backups/scope/PlanSettings.java | 229 ++++++ .../qbits/backups/scope/ScopeResolver.java | 448 ++++++++++ .../qbits/backups/scope/TableRole.java | 40 + .../qbits/backups/scope/TableRoles.java | 130 +++ .../qbits/backups/storage/BackupStorage.java | 107 +++ .../storage/BackupStorageException.java | 69 ++ .../storage/InMemoryBackupStorage.java | 300 +++++++ .../backups/storage/StoredObjectInfo.java | 35 + .../qbits/backups/storage/WriteOptions.java | 118 +++ .../qbits/backups/BackupsTestUtils.java | 168 ++++ .../qbits/backups/run/BackupEngineTest.java | 282 +++++++ 42 files changed, 8459 insertions(+), 3 deletions(-) create mode 100644 src/main/java/com/kingsrook/qbits/backups/BackupsRuntime.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/alerts/BackupAlertSink.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/alerts/NoOpBackupAlertSink.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/app/BackupsApp.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/app/BackupsHomeData.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/app/BackupsHomeView.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/codec/ArtifactTransform.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/codec/CsvRowCodec.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/codec/FieldValueCodec.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/codec/HashingOutputStream.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/codec/JsonlRowCodec.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/codec/RowCodec.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/codec/TableArtifactWriter.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/files/FileCopier.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/files/FileReference.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/files/FileReferenceProvider.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/manifest/BackupManifest.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/manifest/TableFingerprints.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/model/BackupsTables.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/permissions/BackupsPermissionChecker.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/permissions/BackupsPermissions.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/processes/BackupsProcesses.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/run/BackupCatalog.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/run/BackupEngine.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/run/BackupRunFactory.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/run/BackupRunRequest.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/run/BackupRunResult.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/run/TableReader.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/scope/BackupTableMetaData.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/scope/PlanSettings.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/scope/ScopeResolver.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/scope/TableRole.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/scope/TableRoles.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/storage/BackupStorage.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/storage/BackupStorageException.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/storage/InMemoryBackupStorage.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/storage/StoredObjectInfo.java create mode 100644 src/main/java/com/kingsrook/qbits/backups/storage/WriteOptions.java create mode 100644 src/test/java/com/kingsrook/qbits/backups/BackupsTestUtils.java create mode 100644 src/test/java/com/kingsrook/qbits/backups/run/BackupEngineTest.java diff --git a/src/main/java/com/kingsrook/qbits/backups/BackupsQBitConfig.java b/src/main/java/com/kingsrook/qbits/backups/BackupsQBitConfig.java index 02041ae..a67d580 100644 --- a/src/main/java/com/kingsrook/qbits/backups/BackupsQBitConfig.java +++ b/src/main/java/com/kingsrook/qbits/backups/BackupsQBitConfig.java @@ -18,16 +18,20 @@ package com.kingsrook.qbits.backups; import java.util.List; +import java.util.function.Function; +import com.kingsrook.qbits.backups.storage.BackupStorage; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig; import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* - ** Host-facing configuration for the Backups QBit. + ** Configuration of the Backups qbit. ** - ** Tables are named backups_ (lowercase snake_case, backend name equal - ** to the table name); permissions are backups.. + ** Storage is resolved through a function from a bucket alias (the value of + ** backups_plan.storageBucketAlias) to a BackupStorage. The production resolver + ** is backed by qbit-object-storage; tests and local development use the + ** in-memory implementation. No credential is ever held here or in a table. *******************************************************************************/ public class BackupsQBitConfig implements QBitConfig { @@ -35,6 +39,18 @@ public class BackupsQBitConfig implements QBitConfig public static final String PERMISSION_BASE_NAME = "backups"; private String backendName; + private String appName = "backups"; + private String appLabel = "Backups"; + private String parentAppName; + private String schedulerName; + private String instanceName; + + private Integer maxConcurrentRuns = 1; + private Integer slaCheckIntervalSeconds = 300; + private Boolean openWhenAnonymous = false; + private Boolean prometheusEndpoint = false; + + private Function storageResolver; @@ -89,4 +105,214 @@ public class BackupsQBitConfig implements QBitConfig return (this); } + + + /******************************************************************************* + ** Getter for appName + *******************************************************************************/ + public String getAppName() + { + return (appName); + } + + + + /******************************************************************************* + ** Fluent setter for appName + *******************************************************************************/ + public BackupsQBitConfig withAppName(String appName) + { + this.appName = appName; + return (this); + } + + + + /******************************************************************************* + ** Getter for appLabel + *******************************************************************************/ + public String getAppLabel() + { + return (appLabel); + } + + + + /******************************************************************************* + ** Fluent setter for appLabel + *******************************************************************************/ + public BackupsQBitConfig withAppLabel(String appLabel) + { + this.appLabel = appLabel; + return (this); + } + + + + /******************************************************************************* + ** Getter for parentAppName (for example "admin", so this is Admin, Backups). + *******************************************************************************/ + public String getParentAppName() + { + return (parentAppName); + } + + + + /******************************************************************************* + ** Fluent setter for parentAppName + *******************************************************************************/ + public BackupsQBitConfig withParentAppName(String parentAppName) + { + this.parentAppName = parentAppName; + return (this); + } + + + + /******************************************************************************* + ** Getter for schedulerName (the QQQ scheduler used for plan jobs and the SLA checker) + *******************************************************************************/ + public String getSchedulerName() + { + return (schedulerName); + } + + + + /******************************************************************************* + ** Fluent setter for schedulerName + *******************************************************************************/ + public BackupsQBitConfig withSchedulerName(String schedulerName) + { + this.schedulerName = schedulerName; + return (this); + } + + + + /******************************************************************************* + ** Getter for instanceName (used in object keys); defaults to the QInstance name + *******************************************************************************/ + public String getInstanceName() + { + return (instanceName); + } + + + + /******************************************************************************* + ** Fluent setter for instanceName + *******************************************************************************/ + public BackupsQBitConfig withInstanceName(String instanceName) + { + this.instanceName = instanceName; + return (this); + } + + + + /******************************************************************************* + ** Getter for maxConcurrentRuns + *******************************************************************************/ + public Integer getMaxConcurrentRuns() + { + return (maxConcurrentRuns); + } + + + + /******************************************************************************* + ** Fluent setter for maxConcurrentRuns + *******************************************************************************/ + public BackupsQBitConfig withMaxConcurrentRuns(Integer maxConcurrentRuns) + { + this.maxConcurrentRuns = maxConcurrentRuns; + return (this); + } + + + + /******************************************************************************* + ** Getter for slaCheckIntervalSeconds + *******************************************************************************/ + public Integer getSlaCheckIntervalSeconds() + { + return (slaCheckIntervalSeconds); + } + + + + /******************************************************************************* + ** Fluent setter for slaCheckIntervalSeconds + *******************************************************************************/ + public BackupsQBitConfig withSlaCheckIntervalSeconds(Integer slaCheckIntervalSeconds) + { + this.slaCheckIntervalSeconds = slaCheckIntervalSeconds; + return (this); + } + + + + /******************************************************************************* + ** Getter for openWhenAnonymous (local development only) + *******************************************************************************/ + public Boolean getOpenWhenAnonymous() + { + return (openWhenAnonymous); + } + + + + /******************************************************************************* + ** Fluent setter for openWhenAnonymous + *******************************************************************************/ + public BackupsQBitConfig withOpenWhenAnonymous(Boolean openWhenAnonymous) + { + this.openWhenAnonymous = openWhenAnonymous; + return (this); + } + + + + /******************************************************************************* + ** Getter for prometheusEndpoint + *******************************************************************************/ + public Boolean getPrometheusEndpoint() + { + return (prometheusEndpoint); + } + + + + /******************************************************************************* + ** Fluent setter for prometheusEndpoint + *******************************************************************************/ + public BackupsQBitConfig withPrometheusEndpoint(Boolean prometheusEndpoint) + { + this.prometheusEndpoint = prometheusEndpoint; + return (this); + } + + + + /******************************************************************************* + ** Getter for storageResolver + *******************************************************************************/ + public Function getStorageResolver() + { + return (storageResolver); + } + + + + /******************************************************************************* + ** Fluent setter for storageResolver: bucket alias to storage + *******************************************************************************/ + public BackupsQBitConfig withStorageResolver(Function storageResolver) + { + this.storageResolver = storageResolver; + return (this); + } + } diff --git a/src/main/java/com/kingsrook/qbits/backups/BackupsQBitProducer.java b/src/main/java/com/kingsrook/qbits/backups/BackupsQBitProducer.java index 1b6fc7f..7d45d87 100644 --- a/src/main/java/com/kingsrook/qbits/backups/BackupsQBitProducer.java +++ b/src/main/java/com/kingsrook/qbits/backups/BackupsQBitProducer.java @@ -100,4 +100,54 @@ public class BackupsQBitProducer implements QBitMetaDataProducer errors = new java.util.ArrayList<>(); + qBitConfig.validate(qInstance, errors); + if(!errors.isEmpty()) + { + throw (new com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfigValidationException(qBitConfig, errors)); + } + QBitMetaData qBit = getQBitMetaData(); + qInstance.addQBit(qBit); + BackupsRuntime.install(qBitConfig); + + com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput output = new com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput(); + for(var possibleValueSource : com.kingsrook.qbits.backups.model.BackupsTables.possibleValueSources()) + { + output.add(possibleValueSource); + } + for(var join : com.kingsrook.qbits.backups.model.BackupsTables.joins()) + { + output.add(join); + } + java.util.List tables = new java.util.ArrayList<>(); + for(var table : com.kingsrook.qbits.backups.model.BackupsTables.tables(qBitConfig)) + { + var customized = qBitConfig.getTableMetaDataCustomizer() == null ? table : qBitConfig.getTableMetaDataCustomizer().customizeMetaData(qInstance, table); + customized.setSourceQBitName(qBit.getName()); + tables.add(customized); + output.add(customized); + } + for(var process : com.kingsrook.qbits.backups.processes.BackupsProcesses.processes()) + { + process.setSourceQBitName(qBit.getName()); + output.add(process); + } + for(var widget : com.kingsrook.qbits.backups.app.BackupsApp.widgets()) + { + output.add(widget); + } + output.add(com.kingsrook.qbits.backups.app.BackupsApp.app(qBitConfig, tables)); + output.addSelfToInstance(qInstance); + return (new com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput()); + } + } diff --git a/src/main/java/com/kingsrook/qbits/backups/BackupsRuntime.java b/src/main/java/com/kingsrook/qbits/backups/BackupsRuntime.java new file mode 100644 index 0000000..6c629e7 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/BackupsRuntime.java @@ -0,0 +1,195 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups; + + +import java.time.Clock; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; +import com.kingsrook.qbits.backups.alerts.BackupAlertSink; +import com.kingsrook.qbits.backups.alerts.NoOpBackupAlertSink; +import com.kingsrook.qbits.backups.storage.BackupStorage; +import com.kingsrook.qbits.backups.storage.BackupStorageException; + + +/******************************************************************************* + ** Process-wide state of the installed qbit: the config, the clock, the alert + ** sink and the per-process permission map. Static because QQQ instantiates + ** steps and permission checkers by class reference. + *******************************************************************************/ +public final class BackupsRuntime +{ + private static final Map PROCESS_PERMISSIONS = new ConcurrentHashMap<>(); + + private static volatile BackupsQBitConfig config; + private static volatile Clock clock = Clock.systemUTC(); + private static volatile BackupAlertSink alertSink = new NoOpBackupAlertSink(); + private static volatile Function environment = System::getenv; + + + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private BackupsRuntime() + { + } + + + + /******************************************************************************* + ** Install the config (called by the producer). + *******************************************************************************/ + public static void install(BackupsQBitConfig newConfig) + { + config = newConfig; + } + + + + /******************************************************************************* + ** Forget everything (tests). + *******************************************************************************/ + public static void reset() + { + config = null; + clock = Clock.systemUTC(); + alertSink = new NoOpBackupAlertSink(); + environment = System::getenv; + PROCESS_PERMISSIONS.clear(); + } + + + + /******************************************************************************* + ** The installed config, or null. + *******************************************************************************/ + public static BackupsQBitConfig getConfig() + { + return (config); + } + + + + /******************************************************************************* + ** Whether the qbit is installed. + *******************************************************************************/ + public static boolean isInstalled() + { + return (config != null); + } + + + + /******************************************************************************* + ** Whether permissions are open on fully anonymous instances (local development). + *******************************************************************************/ + public static boolean isOpenWhenAnonymous() + { + return (config != null && Boolean.TRUE.equals(config.getOpenWhenAnonymous())); + } + + + + /******************************************************************************* + ** The clock (replaceable in tests). + *******************************************************************************/ + public static Clock getClock() + { + return (clock); + } + + + + /******************************************************************************* + ** Replace the clock. + *******************************************************************************/ + public static void setClock(Clock newClock) + { + clock = newClock; + } + + + + /******************************************************************************* + ** The alert sink. + *******************************************************************************/ + public static BackupAlertSink getAlertSink() + { + return (alertSink); + } + + + + /******************************************************************************* + ** Bind an alert sink (the Alerts qbit adapter, or a test double). + *******************************************************************************/ + public static void setAlertSink(BackupAlertSink newSink) + { + alertSink = newSink == null ? new NoOpBackupAlertSink() : newSink; + } + + + + /******************************************************************************* + ** Environment lookup (replaceable in tests). + *******************************************************************************/ + public static Function getEnvironment() + { + return (environment); + } + + + + /******************************************************************************* + ** Replace the environment lookup. + *******************************************************************************/ + public static void setEnvironment(Function newEnvironment) + { + environment = newEnvironment; + } + + + + /******************************************************************************* + ** The permission each qbit process needs, by process name. + *******************************************************************************/ + public static Map processPermissions() + { + return (PROCESS_PERMISSIONS); + } + + + + /******************************************************************************* + ** Resolve the storage for a bucket alias. + *******************************************************************************/ + public static BackupStorage storageFor(String bucketAlias) + { + if(config == null || config.getStorageResolver() == null) + { + throw (new BackupStorageException(BackupStorageException.Reason.UNSUPPORTED, "No storage resolver is configured for the Backups qbit")); + } + BackupStorage storage = config.getStorageResolver().apply(bucketAlias); + if(storage == null) + { + throw (new BackupStorageException(BackupStorageException.Reason.NOT_FOUND, "No storage bucket is configured with the alias: " + bucketAlias)); + } + return (storage); + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/alerts/BackupAlertSink.java b/src/main/java/com/kingsrook/qbits/backups/alerts/BackupAlertSink.java new file mode 100644 index 0000000..2dd4bf5 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/alerts/BackupAlertSink.java @@ -0,0 +1,65 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.alerts; + + +import java.util.Map; + + +/******************************************************************************* + ** Where backups raises and resolves alerts. The adapter for the Alerts qbit + ** producer SPI binds here when that qbit is installed; otherwise the no-op + ** sink is used and every condition is still visible on the Backups home and in + ** the tables. An alert has a stable dedupe key; counts and links only, never + ** values, keys or presigned URLs. + *******************************************************************************/ +public interface BackupAlertSink +{ + + /*************************************************************************** + ** Raise (or refresh) an alert. + ** + ** @param type alert type, e.g. backups.plan.missed + ** @param dedupeKey stable key, type plus plan or run + ** @param severity severity + ** @param title short title + ** @param detail counts and context only + ***************************************************************************/ + void raise(String type, String dedupeKey, Severity severity, String title, Map detail); + + + /*************************************************************************** + ** Resolve an alert that was raised under the dedupe key. + ***************************************************************************/ + void resolve(String type, String dedupeKey); + + + /*************************************************************************** + ** Check in a heartbeat monitor: the plan ran successfully. + ***************************************************************************/ + void heartbeat(String monitorKey, long expectedEverySeconds, long graceSeconds); + + + + /*************************************************************************** + ** Severities. + ***************************************************************************/ + enum Severity + { + INFO, WARNING, ERROR, CRITICAL + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/alerts/NoOpBackupAlertSink.java b/src/main/java/com/kingsrook/qbits/backups/alerts/NoOpBackupAlertSink.java new file mode 100644 index 0000000..90c718c --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/alerts/NoOpBackupAlertSink.java @@ -0,0 +1,56 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.alerts; + + +import java.util.Map; + + +/******************************************************************************* + ** The sink used when the Alerts qbit is not installed. + *******************************************************************************/ +public class NoOpBackupAlertSink implements BackupAlertSink +{ + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public void raise(String type, String dedupeKey, Severity severity, String title, Map detail) + { + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public void resolve(String type, String dedupeKey) + { + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public void heartbeat(String monitorKey, long expectedEverySeconds, long graceSeconds) + { + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/app/BackupsApp.java b/src/main/java/com/kingsrook/qbits/backups/app/BackupsApp.java new file mode 100644 index 0000000..41d534e --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/app/BackupsApp.java @@ -0,0 +1,152 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.app; + + +import java.util.List; +import com.kingsrook.qbits.backups.BackupsQBitConfig; +import com.kingsrook.qbits.backups.model.BackupsTables; +import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** The Backups app (a sub-app under Admin when configured) and its widgets. The + ** home is the single owner of backup data: every card is built from the qbit + ** tables, and each has an honest empty state. The nav lists the main tables; + ** sub-tables appear as child lists on their parents. + *******************************************************************************/ +public final class BackupsApp +{ + public static final String HOME_WIDGET = "backupsHome"; + public static final String HEALTH_CARD_WIDGET = "backupsHealthCard"; + + + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private BackupsApp() + { + } + + + + /******************************************************************************* + ** The widgets. + *******************************************************************************/ + public static List widgets() + { + return (List.of( + new QWidgetMetaData().withName(HOME_WIDGET).withLabel("Backups").withType(WidgetType.HTML.getType()) + .withCodeReference(new QCodeReference(HomeRenderer.class)).withGridColumns(12).withIsCard(false).withPermissionRules(BackupsTables.permissionRules()), + new QWidgetMetaData().withName(HEALTH_CARD_WIDGET).withLabel("Backups").withType(WidgetType.HTML.getType()) + .withCodeReference(new QCodeReference(HealthCardRenderer.class)).withGridColumns(12).withIsCard(true).withPermissionRules(BackupsTables.permissionRules()))); + } + + + + /******************************************************************************* + ** The app: the main tables only; sub-tables are relations on their parents. + *******************************************************************************/ + public static QAppMetaData app(BackupsQBitConfig config, List tables) + { + QAppMetaData app = new QAppMetaData() + .withName(config.getAppName()) + .withLabel(config.getAppLabel()) + .withIcon(new QIcon("backup")) + .withPermissionRules(BackupsTables.permissionRules()) + .withWidgets(List.of(HOME_WIDGET)) + .withSection(new QAppSection().withName("backups").withLabel("Backups").withIcon(new QIcon("backup")) + .withTables(List.of(BackupsTables.PLAN, BackupsTables.RUN))) + .withSection(new QAppSection().withName("recovery").withLabel("Restore and verify").withIcon(new QIcon("restore")) + .withTables(List.of(BackupsTables.RESTORE_RUN, BackupsTables.VERIFICATION))) + .withSection(new QAppSection().withName("files").withLabel("Files").withIcon(new QIcon("attach_file")) + .withTables(List.of(BackupsTables.FILE_OBJECT))); + if(config.getParentAppName() != null) + { + app.setParentAppName(config.getParentAppName()); + } + for(QTableMetaData table : tables) + { + if(BackupsTables.NAV_TABLES.contains(table.getName())) + { + app.withChild(table); + } + } + return (app); + } + + + + /******************************************************************************* + ** The home. + *******************************************************************************/ + public static class HomeRenderer extends AbstractWidgetRenderer + { + /******************************************************************************* + ** Render. + *******************************************************************************/ + @Override + public RenderWidgetOutput render(RenderWidgetInput input) throws QException + { + return (new RenderWidgetOutput(new RawHTML(null, BackupsHomeView.render(BackupsHomeData.load(), pathPrefix())))); + } + } + + + + /******************************************************************************* + ** The status card for Admin, Health. + *******************************************************************************/ + public static class HealthCardRenderer extends AbstractWidgetRenderer + { + /******************************************************************************* + ** Render. + *******************************************************************************/ + @Override + public RenderWidgetOutput render(RenderWidgetInput input) throws QException + { + return (new RenderWidgetOutput(new RawHTML("Backups", BackupsHomeView.renderHealthCard(BackupsHomeData.load(), pathPrefix())))); + } + } + + + + /******************************************************************************* + ** The path prefix of table and record links. + *******************************************************************************/ + static String pathPrefix() + { + var config = com.kingsrook.qbits.backups.BackupsRuntime.getConfig(); + if(config == null) + { + return ("/backups"); + } + return (config.getParentAppName() != null ? "/" + config.getParentAppName() + "/" + config.getAppName() : "/" + config.getAppName()); + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/app/BackupsHomeData.java b/src/main/java/com/kingsrook/qbits/backups/app/BackupsHomeData.java new file mode 100644 index 0000000..b2d3dc5 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/app/BackupsHomeData.java @@ -0,0 +1,255 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.app; + + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.backups.BackupsRuntime; +import com.kingsrook.qbits.backups.model.BackupsTables; +import com.kingsrook.qbits.backups.run.BackupCatalog; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; + + +/******************************************************************************* + ** The data behind the Backups home: plan status, what needs attention, recent + ** activity and the size trend. Everything is read from the qbit tables; nothing + ** is invented. Home cards read the denormalized plan fields (slaStatus, last + ** success) that the processes and the SLA checker maintain. + *******************************************************************************/ +public class BackupsHomeData +{ + /** Plan statuses that need attention (the plan or its backups have a problem). */ + public static final List ATTENTION = List.of("LATE", "FAILING", "MISSING_IN_STORAGE", "UNVERIFIED", "ABANDONED", "OWNER_INVALID"); + + private final List plans; + private final List recentRuns; + private final List failedRuns; + private final List verifications; + private final List restores; + private final Instant now; + + + + /******************************************************************************* + ** Constructor. + *******************************************************************************/ + BackupsHomeData(List plans, List recentRuns, List failedRuns, List verifications, List restores, Instant now) + { + this.plans = plans; + this.recentRuns = recentRuns; + this.failedRuns = failedRuns; + this.verifications = verifications; + this.restores = restores; + this.now = now; + } + + + + /******************************************************************************* + ** Load from the tables. + *******************************************************************************/ + public static BackupsHomeData load() throws QException + { + BackupCatalog catalog = new BackupCatalog(); + Instant now = BackupsRuntime.getClock().instant(); + List plans = catalog.query(BackupsTables.PLAN, new QQueryFilter().withOrderBy(new QFilterOrderBy("name", true))); + + QQueryFilter recent = new QQueryFilter().withOrderBy(new QFilterOrderBy("id", false)); + recent.setLimit(200); + List runs = catalog.query(BackupsTables.RUN, recent); + List failed = new ArrayList<>(); + Instant weekAgo = now.minus(Duration.ofDays(7)); + for(QRecord run : runs) + { + Instant start = run.getValueInstant("startDate"); + if(List.of("FAILED", "TIMED_OUT", "ABANDONED").contains(run.getValueString("status")) && (start == null || start.isAfter(weekAgo))) + { + failed.add(run); + } + } + QQueryFilter verificationFilter = new QQueryFilter().withOrderBy(new QFilterOrderBy("id", false)); + verificationFilter.setLimit(100); + QQueryFilter restoreFilter = new QQueryFilter().withOrderBy(new QFilterOrderBy("id", false)); + restoreFilter.setLimit(20); + return (new BackupsHomeData(plans, runs, failed, catalog.query(BackupsTables.VERIFICATION, verificationFilter), + catalog.query(BackupsTables.RESTORE_RUN, restoreFilter), now)); + } + + + + /******************************************************************************* + ** Plans. + *******************************************************************************/ + public List plans() + { + return (plans); + } + + + + /******************************************************************************* + ** Recent runs (newest first). + *******************************************************************************/ + public List recentRuns() + { + return (recentRuns); + } + + + + /******************************************************************************* + ** Runs that failed in the last 7 days. + *******************************************************************************/ + public List failedRuns() + { + return (failedRuns); + } + + + + /******************************************************************************* + ** Verifications (newest first). + *******************************************************************************/ + public List verifications() + { + return (verifications); + } + + + + /******************************************************************************* + ** Restores (newest first). + *******************************************************************************/ + public List restores() + { + return (restores); + } + + + + /******************************************************************************* + ** The time the data was loaded. + *******************************************************************************/ + public Instant now() + { + return (now); + } + + + + /******************************************************************************* + ** Plans that need attention. + *******************************************************************************/ + public List plansNeedingAttention() + { + return (plans.stream().filter(p -> Boolean.TRUE.equals(p.getValueBoolean("isActive")) && ATTENTION.contains(p.getValueString("slaStatus"))).toList()); + } + + + + /******************************************************************************* + ** The overall state: NO_PLANS, ATTENTION or OK. + *******************************************************************************/ + public String overallState() + { + if(plans.isEmpty()) + { + return ("NO_PLANS"); + } + return (plansNeedingAttention().isEmpty() && failedRuns.isEmpty() ? "OK" : "ATTENTION"); + } + + + + /******************************************************************************* + ** Hours since the last successful backup of a plan, or null when it has none. + *******************************************************************************/ + public Long ageHours(QRecord plan) + { + Instant last = plan.getValueInstant("lastSuccessDate"); + return (last == null ? null : Duration.between(last, now).toHours()); + } + + + + /******************************************************************************* + ** The latest test restore of a plan, or null. + *******************************************************************************/ + public QRecord latestTestRestore(Integer planId) + { + for(QRecord verification : verifications) + { + if("TEST_RESTORE".equals(verification.getValueString("level"))) + { + Integer runId = verification.getValueInteger("backupRunId"); + for(QRecord run : recentRuns) + { + if(runId != null && runId.equals(run.getValueInteger("id")) && planId.equals(run.getValueInteger("backupPlanId"))) + { + return (verification); + } + } + } + } + return (null); + } + + + + /******************************************************************************* + ** Successful runs of a plan, oldest first, for the trend (bytes stored, rows, duration). + *******************************************************************************/ + public List successfulRuns(Integer planId) + { + List rs = new ArrayList<>(); + for(QRecord run : recentRuns) + { + if(planId.equals(run.getValueInteger("backupPlanId")) && run.getValueString("status") != null && run.getValueString("status").startsWith("SUCCEEDED")) + { + rs.add(0, run); + } + } + return (rs); + } + + + + /******************************************************************************* + ** Runs kept per retention class for a plan (as classified by the last prune). + *******************************************************************************/ + public Map retentionCounts(Integer planId) + { + Map counts = new LinkedHashMap<>(); + for(QRecord run : recentRuns) + { + if(planId.equals(run.getValueInteger("backupPlanId")) && run.getValueString("status") != null && run.getValueString("status").startsWith("SUCCEEDED") + && run.getValue("prunedDate") == null) + { + counts.merge(run.getValueString("retentionClass") == null ? "UNCLASSIFIED" : run.getValueString("retentionClass"), 1, Integer::sum); + } + } + return (counts); + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/app/BackupsHomeView.java b/src/main/java/com/kingsrook/qbits/backups/app/BackupsHomeView.java new file mode 100644 index 0000000..e4af3cd --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/app/BackupsHomeView.java @@ -0,0 +1,396 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.app; + + +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.backups.model.BackupsTables; +import com.kingsrook.qqq.backend.core.model.data.QRecord; + + +/******************************************************************************* + ** Renders the Backups home: one card per concern, each with an honest empty + ** state. Cards stack to one column on narrow screens, long names wrap, and + ** there are no tables and no inner horizontal scroll (definition lists and + ** lists only), so it reads at every width in light and dark mode (colors are + ** inherited, not set). + *******************************************************************************/ +public final class BackupsHomeView +{ + private static final DateTimeFormatter WHEN = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm 'UTC'").withZone(ZoneOffset.UTC); + + private static final String GRID = "display:grid;grid-template-columns:repeat(auto-fit,minmax(min(100%,22rem),1fr));gap:1rem;"; + private static final String CARD = "border:1px solid rgba(127,127,127,.3);border-radius:8px;padding:1rem;min-width:0;overflow-wrap:anywhere;"; + private static final String LIST = "display:grid;grid-template-columns:minmax(7rem,auto) 1fr;gap:.25rem .75rem;margin:0 0 .75rem;font-size:.9rem;"; + private static final String ITEMS = "margin:0 0 .75rem;padding-left:1.25rem;font-size:.9rem;"; + private static final String MUTED = "opacity:.75;"; + + + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private BackupsHomeView() + { + } + + + + /******************************************************************************* + ** The whole home; pathPrefix is the prefix of table and record links. + *******************************************************************************/ + public static String render(BackupsHomeData data, String pathPrefix) + { + StringBuilder html = new StringBuilder("
"); + planStatus(html, data, pathPrefix); + needsAttention(html, data, pathPrefix); + latestTestRestore(html, data, pathPrefix); + trend(html, data); + recentActivity(html, data, pathPrefix); + retention(html, data); + return (html.append("
").toString()); + } + + + + /******************************************************************************* + ** The one status card for Admin, Health. + *******************************************************************************/ + public static String renderHealthCard(BackupsHomeData data, String pathPrefix) + { + String text = switch(data.overallState()) + { + case "NO_PLANS" -> "No backup plans yet."; + case "ATTENTION" -> "Needs attention: see the Backups home."; + default -> "All backups are within their SLA."; + }; + return ("
Backups

" + esc(text) + "

Open the Backups home
"); + } + + + + /******************************************************************************* + ** Plan status: last successful backup, age against the SLA, last run, verification. + *******************************************************************************/ + private static void planStatus(StringBuilder html, BackupsHomeData data, String prefix) + { + open(html, "Plan status"); + if(data.plans().isEmpty()) + { + empty(html, "No backup plans yet. Create one from Backup plans, New."); + } + for(QRecord plan : data.plans()) + { + Long age = data.ageHours(plan); + Integer sla = plan.getValueInteger("slaHours"); + html.append("
"); + html.append("
Plan
") + .append(esc(plan.getValueString("name"))).append("").append(Boolean.TRUE.equals(plan.getValueBoolean("isActive")) ? "" : " (not active)").append("
"); + row(html, "Status", label(plan.getValueString("slaStatus"))); + row(html, "Last backup", plan.getValueInstant("lastSuccessDate") == null ? "No successful backup yet" : WHEN.format(plan.getValueInstant("lastSuccessDate"))); + if(age != null && sla != null) + { + row(html, "Age against SLA", age + " h of " + sla + " h" + (age > sla ? " - late" : "")); + } + row(html, "Last verified", plan.getValueInstant("lastVerifiedDate") == null ? "Not verified yet" : WHEN.format(plan.getValueInstant("lastVerifiedDate"))); + row(html, "Schedule", plan.getValueString("cronExpression") == null ? "Manual only" : plan.getValueString("cronExpression")); + if(plan.getValueString("objectLockMode") != null && !"NONE".equals(plan.getValueString("objectLockMode"))) + { + row(html, "Object lock", plan.getValueString("objectLockMode") + ", " + plan.getValueInteger("objectLockDays") + " days"); + } + html.append("
"); + } + link(html, prefix, BackupsTables.PLAN, "Backup plans"); + close(html); + } + + + + /******************************************************************************* + ** Needs attention. + *******************************************************************************/ + private static void needsAttention(StringBuilder html, BackupsHomeData data, String prefix) + { + open(html, "Needs attention"); + List plans = data.plansNeedingAttention(); + if(data.plans().isEmpty()) + { + empty(html, "Nothing to check yet: no plans."); + } + else if(plans.isEmpty() && data.failedRuns().isEmpty()) + { + empty(html, "Nothing needs attention."); + } + else + { + html.append(""); + } + link(html, prefix, BackupsTables.RUN, "All runs"); + close(html); + } + + + + /******************************************************************************* + ** Latest test restore per plan. + *******************************************************************************/ + private static void latestTestRestore(StringBuilder html, BackupsHomeData data, String prefix) + { + open(html, "Latest test restore"); + boolean any = false; + for(QRecord plan : data.plans()) + { + QRecord v = data.latestTestRestore(plan.getValueInteger("id")); + if(v == null) + { + continue; + } + any = true; + html.append("
"); + row(html, "Plan", plan.getValueString("name")); + row(html, "Result", label(v.getValueString("status"))); + row(html, "Compared", v.getValueInteger("tablesChecked") + " tables, " + v.getValueLong("rowsChecked") + " rows, " + v.getValueInteger("mismatches") + " mismatches"); + row(html, "Scratch target", v.getValueString("scratchTarget") == null ? "Not recorded" : v.getValueString("scratchTarget")); + html.append("
"); + } + if(!any) + { + empty(html, "No test restore yet. Run Verify backup, Test restore on a run, or set a verify schedule on the plan."); + } + link(html, prefix, BackupsTables.VERIFICATION, "All verifications"); + close(html); + } + + + + /******************************************************************************* + ** Size and duration trend per plan, as numbers next to each other. + *******************************************************************************/ + private static void trend(StringBuilder html, BackupsHomeData data) + { + open(html, "Size and duration trend"); + boolean any = false; + for(QRecord plan : data.plans()) + { + List runs = data.successfulRuns(plan.getValueInteger("id")); + if(runs.isEmpty()) + { + continue; + } + any = true; + QRecord first = runs.get(0); + QRecord last = runs.get(runs.size() - 1); + html.append("
"); + row(html, "Plan", plan.getValueString("name")); + row(html, "Successful runs", String.valueOf(runs.size())); + row(html, "Stored (first, latest)", bytes(first.getValueLong("bytesStored")) + ", " + bytes(last.getValueLong("bytesStored"))); + row(html, "Rows (first, latest)", number(first.getValueLong("rowsTotal")) + ", " + number(last.getValueLong("rowsTotal"))); + row(html, "Duration (latest)", last.getValueLong("durationMs") == null ? "Not recorded" : last.getValueLong("durationMs") / 1000 + " s"); + html.append("
"); + } + if(!any) + { + empty(html, "Trends appear after the first successful run."); + } + close(html); + } + + + + /******************************************************************************* + ** Recent activity: the last runs and restores. + *******************************************************************************/ + private static void recentActivity(StringBuilder html, BackupsHomeData data, String prefix) + { + open(html, "Recent activity"); + if(data.recentRuns().isEmpty() && data.restores().isEmpty() && data.verifications().isEmpty()) + { + empty(html, "Nothing has run yet."); + } + else + { + html.append(""); + } + link(html, prefix, BackupsTables.RUN, "All runs"); + close(html); + } + + + + /******************************************************************************* + ** Retention: runs kept per class and the oldest restorable point. + *******************************************************************************/ + private static void retention(StringBuilder html, BackupsHomeData data) + { + open(html, "Retention"); + boolean any = false; + for(QRecord plan : data.plans()) + { + Map counts = data.retentionCounts(plan.getValueInteger("id")); + if(counts.isEmpty()) + { + continue; + } + any = true; + html.append("
"); + row(html, "Plan", plan.getValueString("name")); + row(html, "Kept", counts.entrySet().stream().map(e -> e.getValue() + " " + label(e.getKey()).toLowerCase()).reduce((a, b) -> a + ", " + b).orElse("")); + List runs = data.successfulRuns(plan.getValueInteger("id")); + Instant oldest = runs.isEmpty() ? null : runs.get(0).getValueInstant("startDate"); + row(html, "Oldest restorable", oldest == null ? "Not known" : WHEN.format(oldest)); + html.append("
"); + } + if(!any) + { + empty(html, "Retention starts once runs exist."); + } + close(html); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void open(StringBuilder html, String title) + { + html.append("

").append(esc(title)).append("

"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void close(StringBuilder html) + { + html.append("
"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void empty(StringBuilder html, String text) + { + html.append("

").append(esc(text)).append("

"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void row(StringBuilder html, String name, String value) + { + html.append("
").append(esc(name)).append("
").append(esc(value == null ? "" : value)).append("
"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void link(StringBuilder html, String prefix, String table, String text) + { + html.append("").append(esc(text)).append(""); + } + + + + /******************************************************************************* + ** SOME_STATUS to Some status. + *******************************************************************************/ + static String label(String status) + { + if(status == null || status.isEmpty()) + { + return ("Unknown"); + } + String s = status.replace('_', ' ').toLowerCase(); + return (Character.toUpperCase(s.charAt(0)) + s.substring(1)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + static String bytes(Long value) + { + if(value == null) + { + return ("Not recorded"); + } + double v = value; + String[] units = { "B", "KB", "MB", "GB", "TB" }; + int i = 0; + while(v >= 1024 && i < units.length - 1) + { + v /= 1024; + i++; + } + return (i == 0 ? String.format("%d B", value) : String.format("%.1f %s", v, units[i])); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + static String number(Long value) + { + return (value == null ? "Not recorded" : String.format("%,d", value)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + static String esc(String s) + { + return (s == null ? "" : s.replace("&", "&").replace("<", "<").replace(">", ">").replace("\"", """)); + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/codec/ArtifactTransform.java b/src/main/java/com/kingsrook/qbits/backups/codec/ArtifactTransform.java new file mode 100644 index 0000000..9a9d764 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/codec/ArtifactTransform.java @@ -0,0 +1,86 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.codec; + + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + + +/******************************************************************************* + ** A reversible transform applied to the bytes of an artifact after compression, + ** before they reach storage. The only implementation is client-side encryption + ** (the streaming cipher of the QQQ crypto package); IDENTITY is the default. + *******************************************************************************/ +public interface ArtifactTransform +{ + /** No transform. */ + ArtifactTransform IDENTITY = new ArtifactTransform() + { + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public OutputStream wrap(OutputStream out) + { + return (out); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public InputStream unwrap(InputStream in) + { + return (in); + } + }; + + + + /*************************************************************************** + ** Wrap the stream going to storage. + ***************************************************************************/ + OutputStream wrap(OutputStream out) throws IOException; + + + /*************************************************************************** + ** Wrap a stream read from storage, to undo the transform. + ***************************************************************************/ + InputStream unwrap(InputStream in) throws IOException; + + + /*************************************************************************** + ** Whether this transform encrypts. + ***************************************************************************/ + default boolean encrypts() + { + return (false); + } + + + /*************************************************************************** + ** File name suffix added to artifact keys, e.g. ".tink". + ***************************************************************************/ + default String keySuffix() + { + return (""); + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/codec/CsvRowCodec.java b/src/main/java/com/kingsrook/qbits/backups/codec/CsvRowCodec.java new file mode 100644 index 0000000..2da162d --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/codec/CsvRowCodec.java @@ -0,0 +1,277 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.codec; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeSet; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** The CSV row format (spec 8.3): RFC 4180, UTF-8 without a BOM, a header of + ** field names, and the same canonical value text as JSONL. Null is written as + ** \N (unquoted) so it can be told from the empty string; a string that is + ** literally \N is quoted. A multi-line value is one record in a quoted field, + ** so the codec works on records (a record may span lines): encode() returns a + ** text that may contain embedded newlines, and the part writer appends one + ** newline after each. + *******************************************************************************/ +public class CsvRowCodec implements RowCodec +{ + public static final String NULL_TOKEN = "\\N"; + + private final QTableMetaData table; + private final List fieldNames; + + + + /******************************************************************************* + ** Constructor. + *******************************************************************************/ + public CsvRowCodec(QTableMetaData table, java.util.Collection excludedFields) + { + this.table = table; + TreeSet names = new TreeSet<>(table.getFields().keySet()); + if(excludedFields != null) + { + names.removeAll(excludedFields); + } + this.fieldNames = new ArrayList<>(names); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public List getFieldNames() + { + return (fieldNames); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public String header() + { + return (String.join(",", fieldNames.stream().map(CsvRowCodec::quoteIfNeeded).toList())); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public String encode(QRecord record) + { + StringBuilder sb = new StringBuilder(); + for(int i = 0; i < fieldNames.size(); i++) + { + if(i > 0) + { + sb.append(','); + } + String name = fieldNames.get(i); + Serializable value = record.getValue(name); + if(value == null) + { + sb.append(NULL_TOKEN); + } + else + { + String text = FieldValueCodec.toCanonicalText(table.getField(name).getType(), value); + sb.append(NULL_TOKEN.equals(text) ? "\"" + text + "\"" : quoteIfNeeded(text)); + } + } + return (sb.toString()); + } + + + + /******************************************************************************* + ** Parse one CSV record (as returned by CsvRecordReader, header excluded) to a record. + ** The line may contain embedded newlines inside quoted fields. + *******************************************************************************/ + @Override + public QRecord decode(String line) throws QException + { + List cells = splitCells(line); + return (toRecord(cells, fieldNames)); + } + + + + /******************************************************************************* + ** Build a record from already-split cells and the header names. + *******************************************************************************/ + public QRecord toRecord(List cells, List header) throws QException + { + if(cells.size() != header.size()) + { + throw (new QException("CSV row has " + cells.size() + " cells but the header has " + header.size() + " for table " + table.getName())); + } + Map values = new LinkedHashMap<>(); + for(int i = 0; i < header.size(); i++) + { + String cell = cells.get(i); + QFieldMetaData field = table.getFields().get(header.get(i)); + if(cell == null) + { + values.put(header.get(i), null); + } + else if(field == null) + { + values.put(header.get(i), cell); + } + else + { + values.put(header.get(i), FieldValueCodec.fromCanonicalText(field.getType(), cell)); + } + } + QRecord record = new QRecord(); + record.setTableName(table.getName()); + record.setValues(values); + return (record); + } + + + + /******************************************************************************* + ** Quote a cell when it contains a comma, quote, CR or LF (or is empty after + ** being a header, which never happens), doubling embedded quotes. + *******************************************************************************/ + static String quoteIfNeeded(String s) + { + if(s.indexOf(',') >= 0 || s.indexOf('"') >= 0 || s.indexOf('\n') >= 0 || s.indexOf('\r') >= 0) + { + return ("\"" + s.replace("\"", "\"\"") + "\""); + } + return (s); + } + + + + /******************************************************************************* + ** Split one CSV record into cells. Returns null for an unquoted \N cell (a null). + *******************************************************************************/ + public static List splitCells(String line) + { + List cells = new ArrayList<>(); + StringBuilder cell = new StringBuilder(); + boolean quoted = false; + boolean wasQuoted = false; + for(int i = 0; i < line.length(); i++) + { + char c = line.charAt(i); + if(quoted) + { + if(c == '"') + { + if(i + 1 < line.length() && line.charAt(i + 1) == '"') + { + cell.append('"'); + i++; + } + else + { + quoted = false; + } + } + else + { + cell.append(c); + } + } + else if(c == '"') + { + quoted = true; + wasQuoted = true; + } + else if(c == ',') + { + cells.add(finishCell(cell, wasQuoted)); + cell = new StringBuilder(); + wasQuoted = false; + } + else + { + cell.append(c); + } + } + cells.add(finishCell(cell, wasQuoted)); + return (cells); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static String finishCell(StringBuilder cell, boolean wasQuoted) + { + String s = cell.toString(); + return (!wasQuoted && NULL_TOKEN.equals(s) ? null : s); + } + + + + /******************************************************************************* + ** Split a whole CSV text into records, honoring newlines inside quoted fields. + *******************************************************************************/ + public static List splitRecords(String text) + { + List records = new ArrayList<>(); + StringBuilder current = new StringBuilder(); + boolean quoted = false; + for(int i = 0; i < text.length(); i++) + { + char c = text.charAt(i); + if(c == '"') + { + quoted = !quoted; + current.append(c); + } + else if(c == '\n' && !quoted) + { + records.add(current.toString()); + current = new StringBuilder(); + } + else + { + current.append(c); + } + } + if(current.length() > 0) + { + records.add(current.toString()); + } + return (records); + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/codec/FieldValueCodec.java b/src/main/java/com/kingsrook/qbits/backups/codec/FieldValueCodec.java new file mode 100644 index 0000000..3f032af --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/codec/FieldValueCodec.java @@ -0,0 +1,172 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.codec; + + +import java.io.Serializable; +import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.Base64; +import com.fasterxml.jackson.databind.JsonNode; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** The canonical, lossless text encoding of a field value, by QFieldType + ** (spec 8.2). The same canonical text is used by JSONL and CSV, so a value + ** means the same thing in both, and by the content hash. + ** + ** STRING, TEXT, HTML, PASSWORD: string + ** INTEGER, LONG: number + ** DECIMAL: string (BigDecimal.toPlainString), so there is no float rounding + ** BOOLEAN: true or false + ** DATE: yyyy-MM-dd + ** TIME: HH:mm:ss, with .SSSSSS only when there are fractional seconds + ** DATE_TIME: an ISO-8601 instant in UTC, with .SSSSSS only when there are fractional seconds + ** BLOB: base64 + *******************************************************************************/ +public final class FieldValueCodec +{ + private static final DateTimeFormatter TIME_FORMAT = DateTimeFormatter.ofPattern("HH:mm:ss"); + private static final DateTimeFormatter TIME_FORMAT_MICROS = DateTimeFormatter.ofPattern("HH:mm:ss.SSSSSS"); + private static final DateTimeFormatter INSTANT_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'").withZone(ZoneOffset.UTC); + private static final DateTimeFormatter INSTANT_FORMAT_MICRO = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'").withZone(ZoneOffset.UTC); + + + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private FieldValueCodec() + { + } + + + + /******************************************************************************* + ** Whether the canonical JSON form of a type is a bare number or boolean (not quoted). + *******************************************************************************/ + public static boolean isBareJson(QFieldType type) + { + return (type == QFieldType.INTEGER || type == QFieldType.LONG || type == QFieldType.BOOLEAN); + } + + + + /******************************************************************************* + ** The canonical text of a non-null value. + *******************************************************************************/ + public static String toCanonicalText(QFieldType type, Serializable value) + { + switch(type) + { + case INTEGER, LONG: + return (String.valueOf(ValueUtils.getValueAsLong(value))); + case DECIMAL: + return (ValueUtils.getValueAsBigDecimal(value).toPlainString()); + case BOOLEAN: + return (String.valueOf(ValueUtils.getValueAsBoolean(value))); + case DATE: + return (ValueUtils.getValueAsLocalDate(value).toString()); + case TIME: + { + LocalTime time = ValueUtils.getValueAsLocalTime(value); + return (time.getNano() == 0 ? TIME_FORMAT.format(time) : TIME_FORMAT_MICROS.format(time)); + } + case DATE_TIME: + { + Instant instant = ValueUtils.getValueAsInstant(value); + return (instant.getNano() == 0 ? INSTANT_FORMAT.format(instant) : INSTANT_FORMAT_MICRO.format(instant)); + } + case BLOB: + return (Base64.getEncoder().encodeToString(blobBytes(value))); + default: + return (String.valueOf(value)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static byte[] blobBytes(Serializable value) + { + if(value instanceof byte[] bytes) + { + return (bytes); + } + return (String.valueOf(value).getBytes(java.nio.charset.StandardCharsets.UTF_8)); + } + + + + /******************************************************************************* + ** Parse canonical text back to a Java value. + *******************************************************************************/ + public static Serializable fromCanonicalText(QFieldType type, String text) + { + if(text == null) + { + return (null); + } + switch(type) + { + case INTEGER: + return (Integer.valueOf(text)); + case LONG: + return (Long.valueOf(text)); + case DECIMAL: + return (new BigDecimal(text)); + case BOOLEAN: + return (Boolean.valueOf(text)); + case DATE: + return (LocalDate.parse(text)); + case TIME: + return (LocalTime.parse(text)); + case DATE_TIME: + return (Instant.parse(text)); + case BLOB: + return (Base64.getDecoder().decode(text)); + default: + return (text); + } + } + + + + /******************************************************************************* + ** Convert a parsed JSON value to a Java value. + *******************************************************************************/ + public static Serializable fromJson(QFieldType type, JsonNode node) + { + if(node == null || node.isNull()) + { + return (null); + } + if(type == QFieldType.BOOLEAN) + { + return (node.asBoolean()); + } + return (fromCanonicalText(type, node.isNumber() ? node.asText() : node.textValue())); + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/codec/HashingOutputStream.java b/src/main/java/com/kingsrook/qbits/backups/codec/HashingOutputStream.java new file mode 100644 index 0000000..993c8ef --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/codec/HashingOutputStream.java @@ -0,0 +1,147 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.codec; + + +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; + + +/******************************************************************************* + ** An output stream that counts and SHA-256 hashes the bytes passing through. + ** Closing it closes the stream underneath. The hash is available after close() + ** (or any time, as the hash of the bytes so far). + *******************************************************************************/ +public class HashingOutputStream extends FilterOutputStream +{ + private final MessageDigest digest; + private long count = 0; + private String hex; + + + + /******************************************************************************* + ** Constructor. + *******************************************************************************/ + public HashingOutputStream(OutputStream out) + { + super(out); + this.digest = newSha256(); + } + + + + /******************************************************************************* + ** A new SHA-256 digest. + *******************************************************************************/ + public static MessageDigest newSha256() + { + try + { + return (MessageDigest.getInstance("SHA-256")); + } + catch(NoSuchAlgorithmException e) + { + throw (new IllegalStateException("SHA-256 is not available", e)); + } + } + + + + /******************************************************************************* + ** Hex of a digest result. + *******************************************************************************/ + public static String hex(byte[] bytes) + { + return (HexFormat.of().formatHex(bytes)); + } + + + + /******************************************************************************* + ** SHA-256 of a byte array, as hex. + *******************************************************************************/ + public static String sha256Hex(byte[] bytes) + { + return (hex(newSha256().digest(bytes))); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public void write(int b) throws IOException + { + out.write(b); + digest.update((byte) b); + count++; + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public void write(byte[] b, int off, int len) throws IOException + { + out.write(b, off, len); + digest.update(b, off, len); + count += len; + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public void close() throws IOException + { + super.close(); + if(hex == null) + { + hex = hex(digest.digest()); + } + } + + + + /******************************************************************************* + ** The SHA-256 (hex) of everything written; valid after close(). + *******************************************************************************/ + public String getSha256() + { + return (hex); + } + + + + /******************************************************************************* + ** The number of bytes written. + *******************************************************************************/ + public long getCount() + { + return (count); + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/codec/JsonlRowCodec.java b/src/main/java/com/kingsrook/qbits/backups/codec/JsonlRowCodec.java new file mode 100644 index 0000000..55a4a8e --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/codec/JsonlRowCodec.java @@ -0,0 +1,200 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.codec; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeSet; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** The JSON Lines row format (spec 8.2): one JSON object per line, keys are field + ** names (sorted), values are the raw canonical encodings, never display values. + ** The text of a line is canonical, so the same rows always give the same bytes + ** (which is what the rowsHash is computed over). + *******************************************************************************/ +public class JsonlRowCodec implements RowCodec +{ + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private final QTableMetaData table; + private final List fieldNames; + + + + /******************************************************************************* + ** Constructor. The fields are the table fields, minus any excluded, sorted by name. + *******************************************************************************/ + public JsonlRowCodec(QTableMetaData table, java.util.Collection excludedFields) + { + this.table = table; + TreeSet names = new TreeSet<>(table.getFields().keySet()); + if(excludedFields != null) + { + names.removeAll(excludedFields); + } + this.fieldNames = new ArrayList<>(names); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public List getFieldNames() + { + return (fieldNames); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public String header() + { + return (null); + } + + + + /******************************************************************************* + ** The canonical line (no trailing newline) for a record. + *******************************************************************************/ + @Override + public String encode(QRecord record) + { + StringBuilder sb = new StringBuilder("{"); + boolean first = true; + for(String name : fieldNames) + { + QFieldType type = table.getField(name).getType(); + Serializable value = record.getValue(name); + if(!first) + { + sb.append(','); + } + first = false; + appendString(sb, name); + sb.append(':'); + if(value == null) + { + sb.append("null"); + } + else + { + String text = FieldValueCodec.toCanonicalText(type, value); + if(FieldValueCodec.isBareJson(type)) + { + sb.append(text); + } + else + { + appendString(sb, text); + } + } + } + return (sb.append('}').toString()); + } + + + + /******************************************************************************* + ** Parse one line into a record. Unknown keys are kept as strings so that + ** restore can report them as unmapped. + *******************************************************************************/ + @Override + public QRecord decode(String line) throws QException + { + try + { + JsonNode node = MAPPER.readTree(line); + Map values = new LinkedHashMap<>(); + for(var it = node.fields(); it.hasNext(); ) + { + var entry = it.next(); + QFieldMetaData field = table.getFields().get(entry.getKey()); + if(field == null) + { + values.put(entry.getKey(), entry.getValue().isNull() ? null : entry.getValue().asText()); + } + else + { + values.put(entry.getKey(), FieldValueCodec.fromJson(field.getType(), entry.getValue())); + } + } + QRecord record = new QRecord(); + record.setTableName(table.getName()); + record.setValues(values); + return (record); + } + catch(Exception e) + { + throw (new QException("Could not read a backup row for table " + table.getName() + ": " + e.getMessage())); + } + } + + + + /******************************************************************************* + ** Append a JSON string with escapes. Control characters, quotes and backslashes + ** are escaped; everything else (including non-ASCII) is written as is, in UTF-8. + *******************************************************************************/ + static void appendString(StringBuilder sb, String s) + { + sb.append('"'); + for(int i = 0; i < s.length(); i++) + { + char c = s.charAt(i); + switch(c) + { + case '"' -> sb.append("\\\""); + case '\\' -> sb.append("\\\\"); + case '\n' -> sb.append("\\n"); + case '\r' -> sb.append("\\r"); + case '\t' -> sb.append("\\t"); + case '\b' -> sb.append("\\b"); + case '\f' -> sb.append("\\f"); + default -> + { + if(c < 0x20) + { + sb.append(String.format("\\u%04x", (int) c)); + } + else + { + sb.append(c); + } + } + } + } + sb.append('"'); + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/codec/RowCodec.java b/src/main/java/com/kingsrook/qbits/backups/codec/RowCodec.java new file mode 100644 index 0000000..c5880c0 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/codec/RowCodec.java @@ -0,0 +1,53 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.codec; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; + + +/******************************************************************************* + ** Encodes a record to one line of text, and decodes it back. + *******************************************************************************/ +public interface RowCodec +{ + + /*************************************************************************** + ** The field names written, in order. + ***************************************************************************/ + List getFieldNames(); + + + /*************************************************************************** + ** A header line to write at the start of each part, or null. + ***************************************************************************/ + String header(); + + + /*************************************************************************** + ** One line, without the trailing newline. + ***************************************************************************/ + String encode(QRecord record); + + + /*************************************************************************** + ** Read one line back into a record. + ***************************************************************************/ + QRecord decode(String line) throws QException; +} diff --git a/src/main/java/com/kingsrook/qbits/backups/codec/TableArtifactWriter.java b/src/main/java/com/kingsrook/qbits/backups/codec/TableArtifactWriter.java new file mode 100644 index 0000000..749f926 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/codec/TableArtifactWriter.java @@ -0,0 +1,274 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.codec; + + +import java.io.Closeable; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.zip.GZIPOutputStream; +import com.kingsrook.qbits.backups.storage.BackupStorage; +import com.kingsrook.qbits.backups.storage.WriteOptions; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** Streams the rows of one table into numbered parts in storage. Each row goes + ** through the row codec, then (per part) into a hash of the raw bytes, gzip, + ** the artifact transform (encryption) and a hash of the stored bytes, and out + ** through BackupStorage.openWrite. Memory is O(row); nothing touches a temp disk. + ** + ** Whatever the output format, a canonical JSONL hash of every row (in the order + ** written, which is primary key order) is kept: that is the rowsHash a verifier + ** recomputes (spec 8.2), independent of compression, encryption and part sizes. + *******************************************************************************/ +public class TableArtifactWriter implements Closeable +{ + private final BackupStorage storage; + private final String keyPrefix; + private final QTableMetaData table; + private final RowCodec codec; + private final JsonlRowCodec canonical; + private final boolean csv; + private final boolean gzip; + private final long targetPartBytes; + private final long maxRowsPerPart; + private final ArtifactTransform transform; + private final String runKey; + + private final MessageDigest rowsDigest = HashingOutputStream.newSha256(); + private final List parts = new ArrayList<>(); + + private int partNumber = 0; + private PartState current; + private long totalRows = 0; + private String firstKeyOfTable; + private String lastKeyOfTable; + + + + /*************************************************************************** + ** The outcome of one written part. + ***************************************************************************/ + public record PartResult(int partNumber, String key, long rows, long bytesRaw, long bytesStored, String sha256Raw, String sha256Stored, + String firstKey, String lastKey, boolean encrypted, String format, String compression) + { + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static class PartState + { + String key; + HashingOutputStream raw; + HashingOutputStream stored; + long rows; + String firstKey; + String lastKey; + } + + + + /******************************************************************************* + ** Constructor. + ** + ** @param keyPrefix the key prefix of the table directory, ending in a slash + ** @param format JSONL or CSV + ** @param compression GZIP or NONE + *******************************************************************************/ + public TableArtifactWriter(BackupStorage storage, String keyPrefix, QTableMetaData table, Collection excludedFields, String format, + String compression, long targetPartBytes, long maxRowsPerPart, ArtifactTransform transform, String runKey) + { + this.storage = storage; + this.keyPrefix = keyPrefix; + this.table = table; + this.csv = "CSV".equals(format); + this.canonical = new JsonlRowCodec(table, excludedFields); + this.codec = csv ? new CsvRowCodec(table, excludedFields) : canonical; + this.gzip = "GZIP".equals(compression); + this.targetPartBytes = targetPartBytes; + this.maxRowsPerPart = maxRowsPerPart; + this.transform = transform == null ? ArtifactTransform.IDENTITY : transform; + this.runKey = runKey; + + if(!gzip && !"NONE".equals(compression)) + { + throw (new IllegalArgumentException("Unsupported compression: " + compression)); + } + } + + + + /******************************************************************************* + ** Write one record. + *******************************************************************************/ + public void write(QRecord record) throws IOException + { + if(current == null) + { + openPart(); + } + + String pk = String.valueOf(record.getValue(table.getPrimaryKeyField())); + byte[] line = (codec.encode(record) + "\n").getBytes(StandardCharsets.UTF_8); + current.raw.write(line); + if(current.firstKey == null) + { + current.firstKey = pk; + } + current.lastKey = pk; + current.rows++; + totalRows++; + if(firstKeyOfTable == null) + { + firstKeyOfTable = pk; + } + lastKeyOfTable = pk; + + rowsDigest.update((canonical.encode(record) + "\n").getBytes(StandardCharsets.UTF_8)); + + if(current.raw.getCount() >= targetPartBytes || current.rows >= maxRowsPerPart) + { + closePart(); + } + } + + + + /******************************************************************************* + ** Finish: close the last part (writing an empty part if there were no rows at all). + *******************************************************************************/ + public List finish() throws IOException + { + if(current == null && parts.isEmpty()) + { + openPart(); + } + if(current != null) + { + closePart(); + } + return (parts); + } + + + + /******************************************************************************* + ** The canonical content hash of all rows written, as sha256:hex. Valid after finish(). + *******************************************************************************/ + public String rowsHash() + { + return ("sha256:" + HashingOutputStream.hex(rowsDigest.digest())); + } + + + + /******************************************************************************* + ** Total rows written. + *******************************************************************************/ + public long getTotalRows() + { + return (totalRows); + } + + + + /******************************************************************************* + ** First primary key written. + *******************************************************************************/ + public String getFirstKey() + { + return (firstKeyOfTable); + } + + + + /******************************************************************************* + ** Last primary key written. + *******************************************************************************/ + public String getLastKey() + { + return (lastKeyOfTable); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public void close() throws IOException + { + if(current != null) + { + // an abandoned part is never committed to storage + current.raw.close(); + current = null; + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void openPart() throws IOException + { + partNumber++; + current = new PartState(); + String extension = csv ? "csv" : "jsonl"; + current.key = keyPrefix + String.format("part-%05d.%s%s%s", partNumber, extension, gzip ? ".gz" : "", transform.keySuffix()); + + WriteOptions options = WriteOptions.create().withContentType("application/octet-stream") + .withMetadata("qqq-backup-run", runKey) + .withMetadata("qqq-backup-table", table.getName()); + OutputStream toStorage = storage.openWrite(current.key, options); + current.stored = new HashingOutputStream(toStorage); + OutputStream transformed = transform.wrap(current.stored); + OutputStream compressed = gzip ? new GZIPOutputStream(transformed) : transformed; + current.raw = new HashingOutputStream(compressed); + + String header = codec.header(); + if(header != null) + { + current.raw.write((header + "\n").getBytes(StandardCharsets.UTF_8)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void closePart() throws IOException + { + current.raw.close(); + parts.add(new PartResult(partNumber, current.key, current.rows, current.raw.getCount(), current.stored.getCount(), + current.raw.getSha256(), current.stored.getSha256(), current.firstKey, current.lastKey, transform.encrypts(), + csv ? "CSV" : "JSONL", gzip ? "GZIP" : "NONE")); + current = null; + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/files/FileCopier.java b/src/main/java/com/kingsrook/qbits/backups/files/FileCopier.java new file mode 100644 index 0000000..dd4159f --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/files/FileCopier.java @@ -0,0 +1,223 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.files; + + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Optional; +import java.util.function.Function; +import com.kingsrook.qbits.backups.codec.ArtifactTransform; +import com.kingsrook.qbits.backups.codec.HashingOutputStream; +import com.kingsrook.qbits.backups.storage.BackupStorage; +import com.kingsrook.qbits.backups.storage.StoredObjectInfo; +import com.kingsrook.qbits.backups.storage.WriteOptions; + + +/******************************************************************************* + ** Makes sure each file that a record references is in the content-addressed + ** store (spec 8.5), and says what it did. + ** + ** For each reference: head the source (missing is reported, never silently + ** skipped); find its SHA-256 (known from the reference provider, else the object + ** is read once and hashed); if the store already has that hash the file is + ** DEDUPLICATED and nothing is copied; otherwise it is copied (server-side when + ** the source and the destination are the same storage and the plan does not + ** encrypt, else streamed through the transform) and the hash observed while + ** copying must equal the hash the key was derived from. If the source changed + ** in between, the file is hashed again and copied once more. + *******************************************************************************/ +public class FileCopier +{ + public static final String COPIED = "COPIED"; + public static final String DEDUPLICATED = "DEDUPLICATED"; + public static final String MISSING = "MISSING"; + public static final String REFERENCE_ONLY = "REFERENCE_ONLY"; + + private final Function sourceResolver; + private final String destinationAlias; + private final BackupStorage destination; + private final String casPrefix; + private final ArtifactTransform transform; + private final FileStoreRegistry registry; + private final boolean referenceOnly; + private final String runKey; + + + + /*************************************************************************** + ** The catalog side of the file store: which hashes are already stored. + ***************************************************************************/ + public interface FileStoreRegistry + { + /*************************************************************************** + ** Whether the store knows this hash (and the object is expected to be there). + ***************************************************************************/ + boolean isKnown(String sha256); + + + /*************************************************************************** + ** Record a newly stored file. + ***************************************************************************/ + void register(String sha256, long sizeBytes, String contentType, String casKey, boolean encrypted); + } + + + + /*************************************************************************** + ** One line of the per-run file index. + ** + ** @param tableName the referencing table + ** @param recordKey the referencing record primary key + ** @param field the referencing field + ** @param sourceBucketAlias where the file was + ** @param sourceKey the source key + ** @param sha256 content hash (null when missing or reference-only without a known hash) + ** @param size size in bytes (-1 when unknown) + ** @param status COPIED, DEDUPLICATED, MISSING or REFERENCE_ONLY + ** @param casKey the content-addressed key in the destination, if stored + ***************************************************************************/ + public record IndexEntry(String tableName, String recordKey, String field, String sourceBucketAlias, String sourceKey, + String sha256, long size, String status, String casKey) + { + } + + + + /******************************************************************************* + ** Constructor. + ** + ** @param casPrefix prefix of the store, ending in a slash, e.g. qqq-backups/app/_files/sha256/ + *******************************************************************************/ + public FileCopier(Function sourceResolver, String destinationAlias, BackupStorage destination, String casPrefix, + ArtifactTransform transform, FileStoreRegistry registry, boolean referenceOnly, String runKey) + { + this.sourceResolver = sourceResolver; + this.destinationAlias = destinationAlias; + this.destination = destination; + this.casPrefix = casPrefix; + this.transform = transform == null ? ArtifactTransform.IDENTITY : transform; + this.registry = registry; + this.referenceOnly = referenceOnly; + this.runKey = runKey; + } + + + + /******************************************************************************* + ** The content-addressed key for a hash: prefix/ab/cd/sha256. + *******************************************************************************/ + public static String casKey(String casPrefix, String sha256) + { + return (casPrefix + sha256.substring(0, 2) + "/" + sha256.substring(2, 4) + "/" + sha256); + } + + + + /******************************************************************************* + ** Process one reference. + *******************************************************************************/ + public IndexEntry process(FileReference ref) throws IOException + { + BackupStorage source = sourceResolver.apply(ref.bucketAlias()); + Optional head = source == null ? Optional.empty() : source.head(ref.key()); + if(head.isEmpty()) + { + return (entry(ref, null, -1, MISSING, null)); + } + long size = head.get().size(); + + if(referenceOnly) + { + return (entry(ref, ref.knownSha256(), size, REFERENCE_ONLY, null)); + } + + String sha = ref.knownSha256() != null ? ref.knownSha256() : hashOf(source, ref.key()); + for(int attempt = 0; attempt < 2; attempt++) + { + String key = casKey(casPrefix, sha); + if(registry.isKnown(sha) && destination.head(key).isPresent()) + { + return (entry(ref, sha, size, DEDUPLICATED, key)); + } + + String observed = copy(source, ref, key); + if(observed == null || observed.equals(sha)) + { + registry.register(sha, size, ref.contentType(), key, transform.encrypts()); + return (entry(ref, sha, size, COPIED, key)); + } + // the source changed while it was being copied: the object we wrote is under the wrong key + destination.deletePermanent(key); + sha = observed; + } + throw (new IOException("The file " + ref.key() + " kept changing while it was being copied")); + } + + + + /******************************************************************************* + ** Copy the file to the store; returns the hash observed while streaming, or null + ** for a server-side copy (which cannot observe the bytes). + *******************************************************************************/ + private String copy(BackupStorage source, FileReference ref, String key) throws IOException + { + if(source == destination && !transform.encrypts()) + { + destination.copy(ref.key(), key); + return (null); + } + + WriteOptions options = WriteOptions.create().withContentType(ref.contentType() == null ? "application/octet-stream" : ref.contentType()) + .withMetadata("qqq-backup-run", runKey); + try(InputStream in = source.openRead(ref.key()); + HashingOutputStream plain = new HashingOutputStream(transform.wrap(destination.openWrite(key, options)))) + { + in.transferTo(plain); + plain.flush(); + plain.close(); + return (plain.getSha256()); + } + } + + + + /******************************************************************************* + ** Read an object once, to find its SHA-256. + *******************************************************************************/ + private String hashOf(BackupStorage source, String key) throws IOException + { + try(InputStream in = source.openRead(key); + HashingOutputStream sink = new HashingOutputStream(OutputStream.nullOutputStream())) + { + in.transferTo(sink); + sink.close(); + return (sink.getSha256()); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private IndexEntry entry(FileReference ref, String sha, long size, String status, String casKey) + { + return (new IndexEntry(ref.tableName(), ref.recordKey(), ref.field(), ref.bucketAlias(), ref.key(), sha, size, status, casKey)); + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/files/FileReference.java b/src/main/java/com/kingsrook/qbits/backups/files/FileReference.java new file mode 100644 index 0000000..26b76a1 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/files/FileReference.java @@ -0,0 +1,33 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.files; + + +/******************************************************************************* + ** A reference from a record field to a file in object storage. + ** + ** @param tableName the referencing table + ** @param recordKey the referencing record primary key (as text) + ** @param field the referencing field + ** @param bucketAlias the storage bucket alias the file lives in + ** @param key the object key + ** @param knownSha256 the content hash when the reference provider knows it (a content-addressed key or a stored checksum), else null + ** @param contentType the content type when known, else null + *******************************************************************************/ +public record FileReference(String tableName, String recordKey, String field, String bucketAlias, String key, String knownSha256, String contentType) +{ +} diff --git a/src/main/java/com/kingsrook/qbits/backups/files/FileReferenceProvider.java b/src/main/java/com/kingsrook/qbits/backups/files/FileReferenceProvider.java new file mode 100644 index 0000000..56198d1 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/files/FileReferenceProvider.java @@ -0,0 +1,72 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.files; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qbits.backups.scope.BackupTableMetaData; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** Finds the files a record references. Apps declare their file references once + ** (the object-storage qbit StorageReferenceProvider registrations, which an + ** adapter exposes here); the default provider reads the fileReferenceFields of + ** the table BackupTableMetaData. + *******************************************************************************/ +public interface FileReferenceProvider +{ + + /*************************************************************************** + ** The files this record references (empty when none). + ***************************************************************************/ + List referencesIn(QTableMetaData table, QRecord record); + + + + /*************************************************************************** + ** Reads BackupTableMetaData.fileReferenceFields: each named field holds a key + ** in the table fileBucketAlias bucket. + ***************************************************************************/ + class FromTableMetaData implements FileReferenceProvider + { + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public List referencesIn(QTableMetaData table, QRecord record) + { + BackupTableMetaData meta = BackupTableMetaData.of(table); + if(meta == null || meta.getFileReferenceFields().isEmpty()) + { + return (List.of()); + } + List rs = new ArrayList<>(); + for(String field : meta.getFileReferenceFields()) + { + String key = record.getValueString(field); + if(key != null && !key.isBlank()) + { + rs.add(new FileReference(table.getName(), String.valueOf(record.getValue(table.getPrimaryKeyField())), field, meta.getFileBucketAlias(), key, null, null)); + } + } + return (rs); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/manifest/BackupManifest.java b/src/main/java/com/kingsrook/qbits/backups/manifest/BackupManifest.java new file mode 100644 index 0000000..24eaa4d --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/manifest/BackupManifest.java @@ -0,0 +1,219 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.manifest; + + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.Map; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; + + +/******************************************************************************* + ** The self-describing manifest of a run (spec 8.7), written LAST: it is the + ** commit marker. A run without a manifest did not complete. It holds no + ** secrets: only key references and fingerprints, never key material. + ** + ** Unknown properties are ignored when reading, so a newer manifest is still + ** readable by an older qbit (and by the offline tools). + *******************************************************************************/ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public record BackupManifest( + int manifestVersion, + String runKey, + PlanInfo plan, + InstanceInfo instance, + String startedAt, + String finishedAt, + String requestedBy, + String securityScope, + ConsistencyInfo consistency, + EncryptionInfo encryption, + List tables, + List restoreOrder, + FilesInfo files, + List warnings, + Totals totals) +{ + public static final int CURRENT_VERSION = 1; + + private static final ObjectMapper MAPPER = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(JsonInclude.Include.NON_NULL) + public record PlanInfo(String planKey, String name, Map settings) + { + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(JsonInclude.Include.NON_NULL) + public record InstanceInfo(String name, String appVersion, String qqqVersion, List> qbits) + { + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(JsonInclude.Include.NON_NULL) + public record ConsistencyInfo(String achieved, List> perBackend) + { + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(JsonInclude.Include.NON_NULL) + public record EncryptionInfo(String mode, String scheme, String keyRef, String keyFingerprint) + { + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(JsonInclude.Include.NON_NULL) + public record ExcludedField(String name, String reason) + { + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(JsonInclude.Include.NON_NULL) + public record Part(String key, long rows, long bytesRaw, long bytesStored, String sha256Stored, String sha256Raw, String format, String compression, boolean encrypted) + { + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(JsonInclude.Include.NON_NULL) + public record TableEntry( + String name, + String role, + String backend, + String primaryKey, + String fingerprint, + Map schema, + String filter, + List excludedFields, + long rowCount, + String rowsHash, + Map keyRange, + List parts, + String consistency) + { + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(JsonInclude.Include.NON_NULL) + public record FilesInfo(String policy, List indexKeys, long objects, long bytes, long missing) + { + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(JsonInclude.Include.NON_NULL) + public record Totals(int tables, long rows, long bytesStored) + { + } + + + + /******************************************************************************* + ** The manifest as pretty JSON bytes. + *******************************************************************************/ + public byte[] toJsonBytes() + { + try + { + return (MAPPER.writeValueAsBytes(this)); + } + catch(IOException e) + { + throw (new IllegalStateException("Could not write the manifest", e)); + } + } + + + + /******************************************************************************* + ** Read a manifest. + *******************************************************************************/ + public static BackupManifest read(InputStream in) throws IOException + { + return (MAPPER.readValue(in, BackupManifest.class)); + } + + + + /******************************************************************************* + ** Read a manifest from bytes. + *******************************************************************************/ + public static BackupManifest read(byte[] bytes) throws IOException + { + return (MAPPER.readValue(bytes, BackupManifest.class)); + } + + + + /******************************************************************************* + ** The table entry with a name, or null. + *******************************************************************************/ + public TableEntry table(String name) + { + return (tables == null ? null : tables.stream().filter(t -> t.name().equals(name)).findFirst().orElse(null)); + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/manifest/TableFingerprints.java b/src/main/java/com/kingsrook/qbits/backups/manifest/TableFingerprints.java new file mode 100644 index 0000000..189caf2 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/manifest/TableFingerprints.java @@ -0,0 +1,184 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.manifest; + + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import com.kingsrook.qbits.backups.codec.HashingOutputStream; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; + + +/******************************************************************************* + ** A stable schema descriptor and SHA-256 fingerprint of a table (spec 5.5). + ** + ** This is an internal stand-in for the QQQ core TableSchemaFingerprint (core PR + ** feature/schema-fingerprint). When that ships in the next 4.1.x RC, this class + ** delegates to it and the descriptor below is kept only for reading old manifests. + ** Labels, adornments and help text are excluded, so cosmetic changes do not count. + *******************************************************************************/ +public final class TableFingerprints +{ + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private TableFingerprints() + { + } + + + + /******************************************************************************* + ** The descriptor of a table (sorted fields; plain maps and lists, so it is also + ** what goes in the manifest schema). + *******************************************************************************/ + public static Map descriptor(QTableMetaData table, String backendType) + { + Map d = new LinkedHashMap<>(); + d.put("name", table.getName()); + d.put("primaryKey", table.getPrimaryKeyField()); + d.put("backendType", backendType); + + Map fields = new TreeMap<>(); + for(QFieldMetaData field : new TreeMap<>(table.getFields()).values()) + { + Map f = new LinkedHashMap<>(); + f.put("type", field.getType().name()); + f.put("required", field.getIsRequired()); + f.put("maxLength", field.getMaxLength()); + f.put("pvs", field.getPossibleValueSourceName()); + f.put("heavy", field.getIsHeavy()); + f.put("sensitive", field.getIsSensitive()); + fields.put(field.getName(), f); + } + d.put("fields", fields); + + List uniqueKeys = new ArrayList<>(); + if(table.getUniqueKeys() != null) + { + for(UniqueKey key : table.getUniqueKeys()) + { + uniqueKeys.add(String.join("+", key.getFieldNames().stream().sorted().toList())); + } + } + uniqueKeys.sort(String::compareTo); + d.put("uniqueKeys", uniqueKeys); + + List associations = new ArrayList<>(); + if(table.getAssociations() != null) + { + for(Association association : table.getAssociations()) + { + associations.add(association.getName() + "->" + association.getAssociatedTableName()); + } + } + associations.sort(String::compareTo); + d.put("associations", associations); + return (d); + } + + + + /******************************************************************************* + ** The fingerprint of a table: sha256:hex of the canonical descriptor. + *******************************************************************************/ + public static String of(QTableMetaData table, String backendType) + { + return (fingerprintOf(descriptor(table, backendType))); + } + + + + /******************************************************************************* + ** The fingerprint of a descriptor (maps sorted by key at every level, no whitespace). + *******************************************************************************/ + public static String fingerprintOf(Map descriptor) + { + StringBuilder sb = new StringBuilder(); + canonical(descriptor, sb); + return ("sha256:" + HashingOutputStream.sha256Hex(sb.toString().getBytes(StandardCharsets.UTF_8))); + } + + + + /******************************************************************************* + ** The fingerprint of a whole instance's tables: a hash of the table fingerprints. + *******************************************************************************/ + public static String combine(List tableFingerprints) + { + return ("sha256:" + HashingOutputStream.sha256Hex(String.join("\n", tableFingerprints).getBytes(StandardCharsets.UTF_8))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + private static void canonical(Object o, StringBuilder sb) + { + if(o == null) + { + sb.append("null"); + } + else if(o instanceof Map map) + { + sb.append('{'); + boolean first = true; + for(Map.Entry e : new TreeMap<>((Map) map).entrySet()) + { + if(!first) + { + sb.append(','); + } + first = false; + sb.append('"').append(e.getKey()).append("\":"); + canonical(e.getValue(), sb); + } + sb.append('}'); + } + else if(o instanceof List list) + { + sb.append('['); + for(int i = 0; i < list.size(); i++) + { + if(i > 0) + { + sb.append(','); + } + canonical(list.get(i), sb); + } + sb.append(']'); + } + else if(o instanceof Number || o instanceof Boolean) + { + sb.append(o); + } + else + { + sb.append('"').append(String.valueOf(o).replace("\\", "\\\\").replace("\"", "\\\"")).append('"'); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/model/BackupsTables.java b/src/main/java/com/kingsrook/qbits/backups/model/BackupsTables.java new file mode 100644 index 0000000..f1d5047 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/model/BackupsTables.java @@ -0,0 +1,572 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.model; + + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.backups.BackupsQBitConfig; +import com.kingsrook.qbits.backups.permissions.BackupsPermissionChecker; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** The qbit tables (design spec section 4). All are standard visible QQQ + ** tables, prefixed backups_, in lowercase snake_case. Tables written by + ** processes are read-only in the UI. No table ever holds a key, password or + ** credential: only references such as a key name or a bucket alias. + *******************************************************************************/ +public final class BackupsTables +{ + public static final String PLAN = "backups_plan"; + public static final String PLAN_SCOPE = "backups_plan_scope"; + public static final String RUN = "backups_run"; + public static final String ARTIFACT = "backups_artifact"; + public static final String VERIFICATION = "backups_verification"; + public static final String RESTORE_RUN = "backups_restore_run"; + public static final String RESTORE_RUN_TABLE = "backups_restore_run_table"; + public static final String FILE_OBJECT = "backups_file_object"; + public static final String ID_MAP = "backups_id_map"; + + /** Tables that only processes write (read-only in the UI; no delete). */ + public static final List PROCESS_WRITTEN = List.of(RUN, ARTIFACT, VERIFICATION, RESTORE_RUN, RESTORE_RUN_TABLE, FILE_OBJECT, ID_MAP); + + /** The tables shown in the nav (sub-tables appear as relations on their parents). */ + public static final List NAV_TABLES = List.of(PLAN, RUN, RESTORE_RUN, VERIFICATION, FILE_OBJECT); + + public static final List ALL = List.of(PLAN, PLAN_SCOPE, RUN, ARTIFACT, VERIFICATION, RESTORE_RUN, RESTORE_RUN_TABLE, FILE_OBJECT, ID_MAP); + + public static final String PVS_PREFIX = "backups_"; + + private static final Map> ENUMS = new LinkedHashMap<>(); + + static + { + ENUMS.put("scope_type", List.of("INSTANCE", "APPS", "QBITS", "TABLES")); + ENUMS.put("table_role", List.of("DATA", "AUDIT", "DERIVED", "EPHEMERAL", "SECRET", "BACKUPS", "EXTERNAL")); + ENUMS.put("scope_action", List.of("INCLUDE", "EXCLUDE")); + ENUMS.put("selector_type", List.of("TABLE", "APP", "QBIT", "ROLE")); + ENUMS.put("format", List.of("JSONL", "CSV", "JSONL_AND_CSV", "PARQUET")); + ENUMS.put("compression", List.of("GZIP", "ZSTD", "NONE")); + ENUMS.put("encryption_mode", List.of("NONE", "CLIENT_AES256_GCM")); + ENUMS.put("sensitive_policy", List.of("EXCLUDE", "INCLUDE_ENCRYPTED_ONLY", "INCLUDE")); + ENUMS.put("heavy_policy", List.of("INLINE", "SEPARATE_OBJECTS", "EXCLUDE")); + ENUMS.put("file_policy", List.of("COPY", "REFERENCE_ONLY")); + ENUMS.put("file_missing_policy", List.of("WARN", "FAIL")); + ENUMS.put("consistency", List.of("SNAPSHOT_REQUIRED", "SNAPSHOT_PREFERRED", "PER_TABLE")); + ENUMS.put("consistency_achieved", List.of("INSTANCE_SNAPSHOT", "BACKEND_SNAPSHOT", "PER_TABLE", "BEST_EFFORT")); + ENUMS.put("incremental_mode", List.of("FULL", "CHANGED_SINCE_LAST")); + ENUMS.put("native_dump", List.of("NONE", "PG_DUMP_CUSTOM")); + ENUMS.put("lock_mode", List.of("NONE", "GOVERNANCE", "COMPLIANCE")); + ENUMS.put("sla_status", List.of("OK", "LATE", "FAILING", "MISSING_IN_STORAGE", "UNVERIFIED", "ABANDONED", "OWNER_INVALID", "UNKNOWN")); + ENUMS.put("run_trigger", List.of("SCHEDULED", "MANUAL", "PRE_RESTORE", "RESUME", "IMPORTED")); + ENUMS.put("run_status", List.of("QUEUED", "RUNNING", "SUCCEEDED", "SUCCEEDED_WITH_WARNINGS", "FAILED", "TIMED_OUT", "CANCELLED", "ABANDONED")); + ENUMS.put("retention_class", List.of("DAILY", "WEEKLY", "MONTHLY", "YEARLY", "NONE")); + ENUMS.put("verification_status", List.of("UNVERIFIED", "STORAGE_OK", "CONTENT_OK", "TEST_RESTORE_OK", "FAILED")); + ENUMS.put("artifact_type", List.of("TABLE_DATA", "TABLE_BLOBS", "FILE_OBJECTS_INDEX", "NATIVE_DUMP", "MANIFEST", "CHECKSUMS")); + ENUMS.put("artifact_status", List.of("WRITING", "COMPLETE", "FAILED")); + ENUMS.put("verify_level", List.of("STORAGE", "CONTENT", "TEST_RESTORE")); + ENUMS.put("verify_result", List.of("PASSED", "PASSED_WITH_WARNINGS", "FAILED", "RUNNING", "ERROR")); + ENUMS.put("restore_status", List.of("PREVIEWED", "QUEUED", "RUNNING", "SUCCEEDED", "SUCCEEDED_WITH_WARNINGS", "FAILED", "CANCELLED")); + ENUMS.put("restore_mode", List.of("INSERT_NEW", "UPSERT", "REPLACE")); + ENUMS.put("id_strategy", List.of("PRESERVE", "REMAP_NEW_IDS")); + ENUMS.put("schema_status", List.of("MATCH", "COMPATIBLE", "MISMATCH")); + } + + + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private BackupsTables() + { + } + + + + /******************************************************************************* + ** The name of the possible value source for an enum key. + *******************************************************************************/ + public static String pvs(String enumKey) + { + return (PVS_PREFIX + enumKey); + } + + + + /******************************************************************************* + ** The values of an enum, by key. + *******************************************************************************/ + public static List enumValues(String enumKey) + { + return (ENUMS.get(enumKey)); + } + + + + /******************************************************************************* + ** Permission rules shared by every qbit table, app, widget and process. + *******************************************************************************/ + public static QPermissionRules permissionRules() + { + return (new QPermissionRules() + .withLevel(PermissionLevel.HAS_ACCESS_PERMISSION) + .withPermissionBaseName(BackupsQBitConfig.PERMISSION_BASE_NAME) + .withCustomPermissionChecker(new QCodeReference(BackupsPermissionChecker.class))); + } + + + + /******************************************************************************* + ** The possible-value sources: enums and table references. + *******************************************************************************/ + public static List possibleValueSources() + { + List rs = new ArrayList<>(); + for(Map.Entry> entry : ENUMS.entrySet()) + { + List> values = new ArrayList<>(); + for(String value : entry.getValue()) + { + values.add(new QPossibleValue<>(value, StringUtils.allCapsToMixedCase(value))); + } + rs.add(new QPossibleValueSource().withName(pvs(entry.getKey())).withType(QPossibleValueSourceType.ENUM).withEnumValues(values)); + } + rs.add(QPossibleValueSource.newForTable(PLAN)); + rs.add(QPossibleValueSource.newForTable(RUN)); + rs.add(QPossibleValueSource.newForTable(RESTORE_RUN)); + return (rs); + } + + + + /******************************************************************************* + ** The joins that back the parent-to-child associations (child lists on parents). + *******************************************************************************/ + public static List joins() + { + return (List.of( + oneToMany("backupsPlanScopeJoinPlan", PLAN, PLAN_SCOPE, "backupPlanId"), + oneToMany("backupsArtifactJoinRun", RUN, ARTIFACT, "backupRunId"), + oneToMany("backupsVerificationJoinRun", RUN, VERIFICATION, "backupRunId"), + oneToMany("backupsRestoreRunJoinRun", RUN, RESTORE_RUN, "sourceBackupRunId"), + oneToMany("backupsRunJoinPlan", PLAN, RUN, "backupPlanId"), + oneToMany("backupsRestoreRunTableJoinRestoreRun", RESTORE_RUN, RESTORE_RUN_TABLE, "backupRestoreRunId"))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QJoinMetaData oneToMany(String name, String parent, String child, String childField) + { + return (new QJoinMetaData().withName(name).withType(JoinType.ONE_TO_MANY).withLeftTable(parent).withRightTable(child).withJoinOn(new JoinOn("id", childField))); + } + + + + /******************************************************************************* + ** All tables. + *******************************************************************************/ + public static List tables(BackupsQBitConfig config) + { + return (List.of(plan(config), planScope(config), run(config), artifact(config), verification(config), + restoreRun(config), restoreRunTable(config), fileObject(config), idMap(config))); + } + + + + /******************************************************************************* + ** A table with the common id/createDate/modifyDate fields. + *******************************************************************************/ + private static QTableMetaData base(BackupsQBitConfig config, String name, String label, String icon) + { + return (new QTableMetaData() + .withName(name) + .withLabel(label) + .withIcon(new QIcon(icon)) + .withBackendName(config.getBackendName()) + .withPrimaryKeyField("id") + .withPermissionRules(permissionRules()) + .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false)) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))); + } + + + + /******************************************************************************* + ** A read-only field. + *******************************************************************************/ + private static QFieldMetaData ro(String name, QFieldType type) + { + return (new QFieldMetaData(name, type).withIsEditable(false)); + } + + + + /******************************************************************************* + ** A read-only field backed by an enum possible value source. + *******************************************************************************/ + private static QFieldMetaData roEnum(String name, String enumKey) + { + return (ro(name, QFieldType.STRING).withPossibleValueSourceName(pvs(enumKey))); + } + + + + /******************************************************************************* + ** An editable enum field with a default. + *******************************************************************************/ + private static QFieldMetaData enumField(String name, String enumKey, String defaultValue) + { + QFieldMetaData field = new QFieldMetaData(name, QFieldType.STRING).withPossibleValueSourceName(pvs(enumKey)); + if(defaultValue != null) + { + field.withDefaultValue(defaultValue); + } + return (field); + } + + + + /******************************************************************************* + ** A read-only integer count. + *******************************************************************************/ + private static QFieldMetaData roInt(String name) + { + return (ro(name, QFieldType.INTEGER)); + } + + + + /******************************************************************************* + ** A read-only long (byte counts, millis). + *******************************************************************************/ + private static QFieldMetaData roLong(String name) + { + return (ro(name, QFieldType.LONG)); + } + + + + /******************************************************************************* + ** backups_plan: what, how, where, when and how long (spec 4.1). + *******************************************************************************/ + static QTableMetaData plan(BackupsQBitConfig config) + { + return (base(config, PLAN, "Backup Plan", "backup") + .withRecordLabelFormatAndFields("%s", "name") + .withUniqueKey(new UniqueKey("name")) + .withUniqueKey(new UniqueKey("planKey")) + .withAssociation(new Association().withName("scopes").withAssociatedTableName(PLAN_SCOPE).withJoinName("backupsPlanScopeJoinPlan")) + .withField(new QFieldMetaData("name", QFieldType.STRING).withIsRequired(true).withMaxLength(100)) + .withField(new QFieldMetaData("description", QFieldType.TEXT)) + .withField(new QFieldMetaData("planKey", QFieldType.STRING).withIsRequired(true).withMaxLength(60)) + .withField(new QFieldMetaData("isActive", QFieldType.BOOLEAN).withDefaultValue(true)) + .withField(enumField("scopeType", "scope_type", "INSTANCE").withIsRequired(true)) + .withField(new QFieldMetaData("includeRoles", QFieldType.STRING).withDefaultValue("DATA,AUDIT").withMaxLength(100)) + .withField(new QFieldMetaData("storageBucketAlias", QFieldType.STRING).withIsRequired(true).withMaxLength(100)) + .withField(new QFieldMetaData("pruneBucketAlias", QFieldType.STRING).withMaxLength(100)) + .withField(new QFieldMetaData("prefix", QFieldType.STRING).withMaxLength(255)) + .withField(new QFieldMetaData("critical", QFieldType.BOOLEAN).withDefaultValue(false)) + .withField(enumField("format", "format", "JSONL")) + .withField(enumField("compression", "compression", "GZIP")) + .withField(enumField("encryptionMode", "encryption_mode", "NONE")) + .withField(new QFieldMetaData("encryptionKeyRef", QFieldType.STRING).withMaxLength(100)) + .withField(enumField("sensitiveFieldPolicy", "sensitive_policy", "EXCLUDE")) + .withField(enumField("heavyFieldPolicy", "heavy_policy", "INLINE")) + .withField(enumField("fileObjectPolicy", "file_policy", "COPY")) + .withField(enumField("fileMissingPolicy", "file_missing_policy", "WARN")) + .withField(enumField("consistency", "consistency", "SNAPSHOT_PREFERRED")) + .withField(new QFieldMetaData("useReadReplica", QFieldType.BOOLEAN).withDefaultValue(false)) + .withField(enumField("incrementalMode", "incremental_mode", "FULL")) + .withField(enumField("nativeDump", "native_dump", "NONE")) + .withField(new QFieldMetaData("cronExpression", QFieldType.STRING).withMaxLength(100)) + .withField(new QFieldMetaData("cronTimeZoneId", QFieldType.STRING).withMaxLength(60)) + .withField(roInt("scheduledJobId")) + .withField(new QFieldMetaData("slaHours", QFieldType.INTEGER).withDefaultValue(26)) + .withField(new QFieldMetaData("verifySlaDays", QFieldType.INTEGER).withDefaultValue(8)) + .withField(new QFieldMetaData("verifyCronExpression", QFieldType.STRING).withMaxLength(100)) + .withField(enumField("verifyLevel", "verify_level", "CONTENT")) + .withField(new QFieldMetaData("retainDaily", QFieldType.INTEGER).withDefaultValue(7)) + .withField(new QFieldMetaData("retainWeekly", QFieldType.INTEGER).withDefaultValue(4)) + .withField(new QFieldMetaData("retainMonthly", QFieldType.INTEGER).withDefaultValue(12)) + .withField(new QFieldMetaData("retainYearly", QFieldType.INTEGER).withDefaultValue(0)) + .withField(new QFieldMetaData("retainMinimum", QFieldType.INTEGER).withDefaultValue(3)) + .withField(enumField("objectLockMode", "lock_mode", "NONE")) + .withField(new QFieldMetaData("objectLockDays", QFieldType.INTEGER)) + .withField(new QFieldMetaData("maxDurationMinutes", QFieldType.INTEGER).withDefaultValue(240)) + .withField(new QFieldMetaData("pageSize", QFieldType.INTEGER).withDefaultValue(5000)) + .withField(new QFieldMetaData("targetPartBytes", QFieldType.LONG).withDefaultValue(268435456L)) + .withField(new QFieldMetaData("ownerUserId", QFieldType.STRING).withMaxLength(100)) + .withField(ro("lastRunId", QFieldType.INTEGER).withPossibleValueSourceName(RUN)) + .withField(ro("lastSuccessRunId", QFieldType.INTEGER).withPossibleValueSourceName(RUN)) + .withField(ro("lastSuccessDate", QFieldType.DATE_TIME)) + .withField(ro("lastVerifiedRunId", QFieldType.INTEGER).withPossibleValueSourceName(RUN)) + .withField(ro("lastVerifiedDate", QFieldType.DATE_TIME)) + .withField(roEnum("slaStatus", "sla_status"))); + } + + + + /******************************************************************************* + ** backups_plan_scope: one row per inclusion or exclusion (spec 4.2). + *******************************************************************************/ + static QTableMetaData planScope(BackupsQBitConfig config) + { + return (base(config, PLAN_SCOPE, "Backup Plan Scope", "filter_list") + .withRecordLabelFormatAndFields("%s %s %s", "action", "selectorType", "selectorValue") + .withField(new QFieldMetaData("backupPlanId", QFieldType.INTEGER).withIsRequired(true).withPossibleValueSourceName(PLAN)) + .withField(enumField("action", "scope_action", "INCLUDE").withIsRequired(true)) + .withField(enumField("selectorType", "selector_type", "TABLE").withIsRequired(true)) + .withField(new QFieldMetaData("selectorValue", QFieldType.STRING).withIsRequired(true).withMaxLength(200)) + .withField(new QFieldMetaData("filterJson", QFieldType.TEXT)) + .withField(new QFieldMetaData("includeAssociatedChildren", QFieldType.BOOLEAN).withDefaultValue(true)) + .withField(new QFieldMetaData("fieldExclusions", QFieldType.STRING).withMaxLength(1000))); + } + + + + /******************************************************************************* + ** backups_run: one execution of a plan, or an ad-hoc export (spec 4.3). + *******************************************************************************/ + static QTableMetaData run(BackupsQBitConfig config) + { + return (base(config, RUN, "Backup Run", "history") + .withRecordLabelFormatAndFields("%s", "runKey") + .withUniqueKey(new UniqueKey("runKey")) + .withAssociation(new Association().withName("artifacts").withAssociatedTableName(ARTIFACT).withJoinName("backupsArtifactJoinRun")) + .withAssociation(new Association().withName("verifications").withAssociatedTableName(VERIFICATION).withJoinName("backupsVerificationJoinRun")) + .withField(ro("backupPlanId", QFieldType.INTEGER).withPossibleValueSourceName(PLAN)) + .withField(ro("runKey", QFieldType.STRING)) + .withField(roEnum("trigger", "run_trigger")) + .withField(ro("requestedByUserId", QFieldType.STRING)) + .withField(roEnum("status", "run_status")) + .withField(ro("startDate", QFieldType.DATE_TIME)) + .withField(ro("endDate", QFieldType.DATE_TIME)) + .withField(roLong("durationMs")) + .withField(ro("heartbeatDate", QFieldType.DATE_TIME)) + .withField(roInt("tablesPlanned")) + .withField(roInt("tablesDone")) + .withField(roLong("rowsTotal")) + .withField(roLong("bytesRaw")) + .withField(roLong("bytesStored")) + .withField(roInt("filesReferenced")) + .withField(roInt("filesCopied")) + .withField(roInt("filesDeduplicated")) + .withField(roInt("filesMissing")) + .withField(roLong("fileBytesCopied")) + .withField(roLong("fileBytesReferenced")) + .withField(roEnum("consistencyAchieved", "consistency_achieved")) + .withField(ro("snapshotId", QFieldType.STRING)) + .withField(ro("storageBucketAlias", QFieldType.STRING)) + .withField(ro("runPrefix", QFieldType.STRING)) + .withField(ro("manifestKey", QFieldType.STRING)) + .withField(ro("manifestSha256", QFieldType.STRING)) + .withField(ro("qqqVersion", QFieldType.STRING)) + .withField(ro("appVersion", QFieldType.STRING)) + .withField(ro("instanceName", QFieldType.STRING)) + .withField(ro("schemaFingerprint", QFieldType.STRING)) + .withField(ro("securityScope", QFieldType.STRING)) + .withField(ro("encryptionKeyRef", QFieldType.STRING)) + .withField(ro("encryptionKeyFingerprint", QFieldType.STRING)) + .withField(roEnum("retentionClass", "retention_class")) + .withField(ro("lockedUntil", QFieldType.DATE_TIME)) + .withField(ro("prunedDate", QFieldType.DATE_TIME)) + .withField(ro("prunedByUserId", QFieldType.STRING)) + .withField(roEnum("verificationStatus", "verification_status")) + .withField(roInt("warningCount")) + .withField(ro("errorMessage", QFieldType.TEXT)) + .withField(ro("errorDetail", QFieldType.TEXT))); + } + + + + /******************************************************************************* + ** backups_artifact: one stored object or part (spec 4.4). + *******************************************************************************/ + static QTableMetaData artifact(BackupsQBitConfig config) + { + return (base(config, ARTIFACT, "Backup Artifact", "inventory_2") + .withRecordLabelFormatAndFields("%s", "objectKey") + .withField(ro("backupRunId", QFieldType.INTEGER).withPossibleValueSourceName(RUN)) + .withField(roEnum("artifactType", "artifact_type")) + .withField(ro("tableName", QFieldType.STRING)) + .withField(roInt("partNumber")) + .withField(ro("objectKey", QFieldType.STRING)) + .withField(ro("format", QFieldType.STRING)) + .withField(ro("compression", QFieldType.STRING)) + .withField(ro("encrypted", QFieldType.BOOLEAN)) + .withField(roLong("rowCount")) + .withField(roLong("bytesRaw")) + .withField(roLong("bytesStored")) + .withField(ro("sha256Stored", QFieldType.STRING)) + .withField(ro("sha256Raw", QFieldType.STRING)) + .withField(ro("rowsHash", QFieldType.STRING)) + .withField(ro("firstKey", QFieldType.STRING)) + .withField(ro("lastKey", QFieldType.STRING)) + .withField(ro("tableFingerprint", QFieldType.STRING)) + .withField(roEnum("status", "artifact_status")) + .withField(ro("storageVersionId", QFieldType.STRING))); + } + + + + /******************************************************************************* + ** backups_verification: the result of a verify (spec 4.5). + *******************************************************************************/ + static QTableMetaData verification(BackupsQBitConfig config) + { + return (base(config, VERIFICATION, "Backup Verification", "verified") + .withRecordLabelFormatAndFields("%s %s", "level", "status") + .withField(ro("backupRunId", QFieldType.INTEGER).withPossibleValueSourceName(RUN)) + .withField(roEnum("level", "verify_level")) + .withField(roEnum("trigger", "run_trigger")) + .withField(ro("requestedByUserId", QFieldType.STRING)) + .withField(ro("startDate", QFieldType.DATE_TIME)) + .withField(ro("endDate", QFieldType.DATE_TIME)) + .withField(roEnum("status", "verify_result")) + .withField(roInt("tablesChecked")) + .withField(roLong("rowsChecked")) + .withField(roInt("mismatches")) + .withField(ro("scratchTarget", QFieldType.STRING)) + .withField(ro("detailJson", QFieldType.TEXT))); + } + + + + /******************************************************************************* + ** backups_restore_run: one restore (spec 4.6). + *******************************************************************************/ + static QTableMetaData restoreRun(BackupsQBitConfig config) + { + return (base(config, RESTORE_RUN, "Backup Restore", "restore") + .withRecordLabelFormatAndFields("Restore %s", "id") + .withAssociation(new Association().withName("tables").withAssociatedTableName(RESTORE_RUN_TABLE).withJoinName("backupsRestoreRunTableJoinRestoreRun")) + .withField(ro("sourceBackupRunId", QFieldType.INTEGER).withPossibleValueSourceName(RUN)) + .withField(ro("requestedByUserId", QFieldType.STRING)) + .withField(ro("startDate", QFieldType.DATE_TIME)) + .withField(ro("endDate", QFieldType.DATE_TIME)) + .withField(roEnum("status", "restore_status")) + .withField(roEnum("mode", "restore_mode")) + .withField(roEnum("idStrategy", "id_strategy")) + .withField(ro("matchOn", QFieldType.STRING)) + .withField(ro("targetInstanceName", QFieldType.STRING)) + .withField(ro("scopeSummary", QFieldType.TEXT)) + .withField(ro("preRestoreBackupRunId", QFieldType.INTEGER).withPossibleValueSourceName(RUN)) + .withField(roLong("rowsInserted")) + .withField(roLong("rowsUpdated")) + .withField(roLong("rowsUnchanged")) + .withField(roLong("rowsDeleted")) + .withField(roLong("rowsSkipped")) + .withField(roLong("rowsErrored")) + .withField(roInt("filesRestored")) + .withField(roInt("filesAlreadyPresent")) + .withField(roInt("filesConflicting")) + .withField(roInt("filesMissingFromBackup")) + .withField(roLong("fileBytesRestored")) + .withField(roInt("sequencesResynced")) + .withField(ro("automationsSuppressed", QFieldType.BOOLEAN)) + .withField(ro("esbSuppressed", QFieldType.BOOLEAN)) + .withField(ro("confirmationText", QFieldType.STRING)) + .withField(ro("errorMessage", QFieldType.TEXT))); + } + + + + /******************************************************************************* + ** backups_restore_run_table: per-table restore results (spec 4.7). + *******************************************************************************/ + static QTableMetaData restoreRunTable(BackupsQBitConfig config) + { + return (base(config, RESTORE_RUN_TABLE, "Backup Restore Table", "table_rows") + .withRecordLabelFormatAndFields("%s", "tableName") + .withField(ro("backupRestoreRunId", QFieldType.INTEGER).withPossibleValueSourceName(RESTORE_RUN)) + .withField(ro("tableName", QFieldType.STRING)) + .withField(ro("targetTableName", QFieldType.STRING)) + .withField(roLong("rowsInserted")) + .withField(roLong("rowsUpdated")) + .withField(roLong("rowsUnchanged")) + .withField(roLong("rowsDeleted")) + .withField(roLong("rowsSkipped")) + .withField(roLong("rowsErrored")) + .withField(roEnum("schemaStatus", "schema_status")) + .withField(ro("unmappedFields", QFieldType.STRING)) + .withField(ro("sequenceResynced", QFieldType.BOOLEAN)) + .withField(ro("status", QFieldType.STRING))); + } + + + + /******************************************************************************* + ** backups_file_object: the content-addressed file store (spec 4.8). + *******************************************************************************/ + static QTableMetaData fileObject(BackupsQBitConfig config) + { + return (base(config, FILE_OBJECT, "Backup File", "attach_file") + .withRecordLabelFormatAndFields("%s", "sha256") + .withUniqueKey(new UniqueKey("sha256")) + .withField(ro("sha256", QFieldType.STRING)) + .withField(roLong("sizeBytes")) + .withField(ro("contentType", QFieldType.STRING)) + .withField(ro("storageBucketAlias", QFieldType.STRING)) + .withField(ro("objectKey", QFieldType.STRING)) + .withField(ro("encrypted", QFieldType.BOOLEAN)) + .withField(ro("encryptionKeyFingerprint", QFieldType.STRING)) + .withField(ro("firstRunId", QFieldType.INTEGER).withPossibleValueSourceName(RUN)) + .withField(ro("lastReferencedRunId", QFieldType.INTEGER).withPossibleValueSourceName(RUN)) + .withField(roInt("referenceCount")) + .withField(ro("lastVerifiedDate", QFieldType.DATE_TIME)) + .withField(ro("lastVerifyStatus", QFieldType.STRING)) + .withField(ro("lockedUntil", QFieldType.DATE_TIME)) + .withField(ro("deletedDate", QFieldType.DATE_TIME))); + } + + + + /******************************************************************************* + ** backups_id_map: transient old-to-new ids for REMAP_NEW_IDS restores (spec 4.9). + *******************************************************************************/ + static QTableMetaData idMap(BackupsQBitConfig config) + { + return (base(config, ID_MAP, "Backup ID Map", "swap_horiz") + .withRecordLabelFormatAndFields("%s %s", "tableName", "oldId") + .withField(ro("restoreRunId", QFieldType.INTEGER).withPossibleValueSourceName(RESTORE_RUN)) + .withField(ro("tableName", QFieldType.STRING)) + .withField(ro("oldId", QFieldType.STRING)) + .withField(ro("newId", QFieldType.STRING))); + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/permissions/BackupsPermissionChecker.java b/src/main/java/com/kingsrook/qbits/backups/permissions/BackupsPermissionChecker.java new file mode 100644 index 0000000..2b5fa2b --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/permissions/BackupsPermissionChecker.java @@ -0,0 +1,103 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.permissions; + + +import java.util.Map; +import com.kingsrook.qbits.backups.BackupsRuntime; +import com.kingsrook.qbits.backups.model.BackupsTables; +import com.kingsrook.qqq.backend.core.actions.permissions.CustomPermissionChecker; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; + + +/******************************************************************************* + ** Permission rules for every qbit table, app, widget and process (spec 11.4): + ** view reads everything; plans are edited with backups.admin; the tables that + ** processes write are read-only in the UI; each process names its permission. + *******************************************************************************/ +public class BackupsPermissionChecker implements CustomPermissionChecker +{ + /******************************************************************************* + ** Check, throwing when denied. + *******************************************************************************/ + @Override + public void checkPermissionsThrowing(AbstractActionInput actionInput, MetaDataWithPermissionRules metaData) throws QPermissionDeniedException + { + String needed = required(actionInput, metaData); + if(needed == null) + { + throw (new QPermissionDeniedException("This Backups table is written only by the Backups qbit.")); + } + if(!has(needed)) + { + throw (new QPermissionDeniedException("This needs the " + needed + " permission.")); + } + } + + + + /******************************************************************************* + ** The permission an action needs, or null when nobody may do it. + *******************************************************************************/ + static String required(AbstractActionInput actionInput, MetaDataWithPermissionRules metaData) + { + boolean write = actionInput instanceof InsertInput || actionInput instanceof UpdateInput || actionInput instanceof DeleteInput; + if(metaData instanceof QTableMetaData table) + { + if(!write) + { + return (BackupsPermissions.VIEW); + } + return (BackupsTables.PROCESS_WRITTEN.contains(table.getName()) ? null : BackupsPermissions.ADMIN); + } + if(metaData instanceof QProcessMetaData process) + { + Map byProcess = BackupsRuntime.processPermissions(); + return (byProcess.getOrDefault(process.getName(), BackupsPermissions.ADMIN)); + } + return (BackupsPermissions.VIEW); + } + + + + /******************************************************************************* + ** Whether the current session holds a permission (or the instance is fully + ** anonymous and the config allows that, for local development). + *******************************************************************************/ + public static boolean has(String permission) + { + QInstance instance = QContext.getQInstance(); + QSession session = QContext.getQSession(); + if(session != null && session.hasPermission(permission)) + { + return (true); + } + return (BackupsRuntime.isOpenWhenAnonymous() && instance != null && instance.getAuthentication() != null + && QAuthenticationType.FULLY_ANONYMOUS.equals(instance.getAuthentication().getType())); + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/permissions/BackupsPermissions.java b/src/main/java/com/kingsrook/qbits/backups/permissions/BackupsPermissions.java new file mode 100644 index 0000000..1cc8a39 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/permissions/BackupsPermissions.java @@ -0,0 +1,42 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.permissions; + + +/******************************************************************************* + ** Permission names (design spec section 11.4). + *******************************************************************************/ +public final class BackupsPermissions +{ + /** The app, every backups table (read-only), the home. */ + public static final String VIEW = "backups.view"; + /** Run backup now, back up records, verify, cancel and resume. */ + public static final String RUN = "backups.run"; + /** Restore (insert-new and upsert), download, cancel restore. */ + public static final String RESTORE = "backups.restore"; + /** Edit plans, prune, replace restores, import catalog, test destination. */ + public static final String ADMIN = "backups.admin"; + + + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private BackupsPermissions() + { + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/processes/BackupsProcesses.java b/src/main/java/com/kingsrook/qbits/backups/processes/BackupsProcesses.java new file mode 100644 index 0000000..7827ab0 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/processes/BackupsProcesses.java @@ -0,0 +1,263 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.processes; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.backups.BackupsRuntime; +import com.kingsrook.qbits.backups.model.BackupsTables; +import com.kingsrook.qbits.backups.permissions.BackupsPermissions; +import com.kingsrook.qbits.backups.run.BackupEngine; +import com.kingsrook.qbits.backups.run.BackupRunFactory; +import com.kingsrook.qbits.backups.run.BackupRunRequest; +import com.kingsrook.qbits.backups.run.BackupRunResult; +import com.kingsrook.qbits.backups.scope.ScopeResolver; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; + + +/******************************************************************************* + ** The qbit processes (spec 9). Each is a standard streamed ETL with the standard + ** Review and Result summary lines; none is in the nav; labels are plain. They + ** appear as record actions on the table that has the records they act on. + *******************************************************************************/ +public final class BackupsProcesses +{ + public static final String RUN_PLAN = "backupRunPlan"; + + + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private BackupsProcesses() + { + } + + + + /******************************************************************************* + ** All processes. + *******************************************************************************/ + public static List processes() + { + List processes = new ArrayList<>(); + processes.add(permission(etl(RUN_PLAN, "Run backup now", BackupsTables.PLAN, RunPlanTransform.class, RunPlanLoad.class, + "This is a preview of what the backup will include. Nothing is written until you continue."), BackupsPermissions.RUN)); + return (processes); + } + + + + /******************************************************************************* + ** Record the permission a process needs. + *******************************************************************************/ + public static QProcessMetaData permission(QProcessMetaData process, String permission) + { + BackupsRuntime.processPermissions().put(process.getName(), permission); + return (process); + } + + + + /******************************************************************************* + ** A standard streamed ETL process with full validation (Review is the dry run). + *******************************************************************************/ + public static QProcessMetaData etl(String name, String label, String table, Class transform, + Class load, String preview) + { + Map values = new HashMap<>(); + values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, table); + values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table); + values.put(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION, true); + values.put(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true); + values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, preview); + QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(ExtractViaQueryStep.class, transform, load, values); + process.setName(name); + process.setLabel(label); + process.setTableName(table); + process.setPermissionRules(BackupsTables.permissionRules()); + return (process); + } + + + + /******************************************************************************* + ** A summary line with future (Review) and past (Result) wording. + *******************************************************************************/ + public static ProcessSummaryLine line(Status status, String singularFuture, String pluralFuture, String singularPast, String pluralPast) + { + ProcessSummaryLine line = new ProcessSummaryLine(status, 0, null); + line.setSingularFutureMessage(singularFuture); + line.setPluralFutureMessage(pluralFuture); + line.setSingularPastMessage(singularPast); + line.setPluralPastMessage(pluralPast); + return (line); + } + + + + /******************************************************************************* + ** Run backup now, review: what each selected plan will back up. + *******************************************************************************/ + public static class RunPlanTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface + { + private final ProcessSummaryLine plans = line(Status.OK, "plan will run", "plans will run", "plan was run", "plans were run"); + private final ProcessSummaryLine tables = line(Status.INFO, "table will be backed up", "tables will be backed up", "table was backed up", "tables were backed up"); + private final ProcessSummaryLine excluded = line(Status.INFO, "table is left out by its role or scope", "tables are left out by their role or scope", + "table was left out by its role or scope", "tables were left out by their role or scope"); + private final ProcessSummaryLine inactive = line(Status.WARNING, "plan is not active (it can still be run by hand)", "plans are not active (they can still be run by hand)", + "plan was not active", "plans were not active"); + private final ProcessSummaryLine invalid = line(Status.ERROR, "plan cannot run", "plans cannot run", "plan could not run", "plans could not run"); + private final List samples = new ArrayList<>(); + + + + /******************************************************************************* + ** Resolve each plan scope without writing anything. + *******************************************************************************/ + @Override + public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + for(QRecord plan : input.getRecords()) + { + try + { + String alias = plan.getValueString("storageBucketAlias"); + if(alias == null) + { + throw (new QException("no storage bucket alias")); + } + BackupsRuntime.storageFor(alias); + ScopeResolver.ResolvedScope scope = BackupRunFactory.resolveScope(plan); + plans.incrementCountAndAddPrimaryKey(plan.getValue("id")); + tables.incrementCount(scope.tables().size()); + excluded.incrementCount(scope.excluded().size()); + if(!Boolean.TRUE.equals(plan.getValueBoolean("isActive"))) + { + inactive.incrementCountAndAddPrimaryKey(plan.getValue("id")); + } + for(ScopeResolver.ResolvedTable table : scope.tables().stream().limit(5).toList()) + { + samples.add(table.name()); + } + output.getRecords().add(plan); + } + catch(Exception e) + { + invalid.incrementCountAndAddPrimaryKey(plan.getValue("id")); + invalid.setMessageSuffix(": " + e.getMessage()); + } + } + } + + + + /******************************************************************************* + ** Review lines. + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen) + { + ArrayList lines = new ArrayList<>(); + plans.addSelfToListIfAnyCount(lines); + tables.addSelfToListIfAnyCount(lines); + excluded.addSelfToListIfAnyCount(lines); + inactive.addSelfToListIfAnyCount(lines); + invalid.addSelfToListIfAnyCount(lines); + return (lines); + } + } + + + + /******************************************************************************* + ** Run backup now, execute: run the engine for each plan. + *******************************************************************************/ + public static class RunPlanLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface + { + private final ProcessSummaryLine succeeded = line(Status.OK, "backup will succeed", "backups succeeded", "backup succeeded", "backups succeeded"); + private final ProcessSummaryLine warnings = line(Status.WARNING, "backup finished with warnings", "backups finished with warnings", + "backup finished with warnings", "backups finished with warnings"); + private final ProcessSummaryLine failed = line(Status.ERROR, "backup failed", "backups failed", "backup failed", "backups failed"); + private final ProcessSummaryLine rows = line(Status.INFO, "row was written", "rows were written", "row was written", "rows were written"); + private final ProcessSummaryLine files = line(Status.INFO, "referenced file was copied or already stored", "referenced files were copied or already stored", + "referenced file was copied or already stored", "referenced files were copied or already stored"); + private final List results = new ArrayList<>(); + + + + /******************************************************************************* + ** Run the plans. + *******************************************************************************/ + @Override + public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + for(QRecord plan : input.getRecords()) + { + BackupRunRequest request = BackupRunFactory.forPlan(plan, "MANUAL"); + BackupRunResult result = new BackupEngine().execute(request); + results.add(result); + switch(result.getStatus()) + { + case "SUCCEEDED" -> succeeded.incrementCountAndAddPrimaryKey(result.getRunId()); + case "SUCCEEDED_WITH_WARNINGS" -> warnings.incrementCountAndAddPrimaryKey(result.getRunId()); + default -> + { + failed.incrementCountAndAddPrimaryKey(result.getRunId()); + failed.setMessageSuffix(result.getErrorMessage() == null ? null : ": " + result.getErrorMessage()); + } + } + rows.incrementCount((int) Math.min(Integer.MAX_VALUE, result.getRows())); + files.incrementCount(result.getFilesCopied() + result.getFilesDeduplicated()); + output.getRecords().add(plan); + } + } + + + + /******************************************************************************* + ** Result lines, with the run in the primary keys so it links to the run. + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen) + { + ArrayList lines = new ArrayList<>(); + succeeded.addSelfToListIfAnyCount(lines); + warnings.addSelfToListIfAnyCount(lines); + failed.addSelfToListIfAnyCount(lines); + rows.addSelfToListIfAnyCount(lines); + files.addSelfToListIfAnyCount(lines); + return (lines); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/run/BackupCatalog.java b/src/main/java/com/kingsrook/qbits/backups/run/BackupCatalog.java new file mode 100644 index 0000000..feb4348 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/run/BackupCatalog.java @@ -0,0 +1,254 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.run; + + +import java.io.Serializable; +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import com.kingsrook.qbits.backups.files.FileCopier; +import com.kingsrook.qbits.backups.model.BackupsTables; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; + + +/******************************************************************************* + ** Reads and writes the qbit own tables (the catalog) through the standard QQQ + ** actions. Every table is a normal, visible QQQ table; this class only wraps + ** the calls the processes make, under a permission bypass (the callers have + ** already checked backups.* permissions). + *******************************************************************************/ +public class BackupCatalog +{ + + /******************************************************************************* + ** Insert one row, returning its id. + *******************************************************************************/ + public Integer insert(String table, Map values) throws QException + { + QRecord record = new QRecord(); + record.setValues(new LinkedHashMap<>(values)); + InsertInput input = new InsertInput(table); + input.setRecords(List.of(record)); + input.withPermissionBypass("backups catalog"); + List rs = new InsertAction().execute(input).getRecords(); + if(rs.get(0).getErrors() != null && !rs.get(0).getErrors().isEmpty()) + { + throw (new QException("Could not write to " + table + ": " + rs.get(0).getErrors().get(0).getMessage())); + } + return (rs.get(0).getValueInteger("id")); + } + + + + /******************************************************************************* + ** Update some fields of one row. + *******************************************************************************/ + public void update(String table, Integer id, Map values) throws QException + { + Map all = new LinkedHashMap<>(values); + all.put("id", id); + QRecord record = new QRecord(); + record.setValues(all); + UpdateInput input = new UpdateInput(table); + input.setRecords(List.of(record)); + input.withPermissionBypass("backups catalog"); + List rs = new UpdateAction().execute(input).getRecords(); + if(rs.get(0).getErrors() != null && !rs.get(0).getErrors().isEmpty()) + { + throw (new QException("Could not update " + table + " " + id + ": " + rs.get(0).getErrors().get(0).getMessage())); + } + } + + + + /******************************************************************************* + ** One row by id. + *******************************************************************************/ + public Optional get(String table, Integer id) throws QException + { + return (query(table, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, id))).stream().findFirst()); + } + + + + /******************************************************************************* + ** Query a table (catalog tables are small; no paging). + *******************************************************************************/ + public List query(String table, QQueryFilter filter) throws QException + { + QueryInput input = new QueryInput(table); + input.setFilter(filter); + input.withPermissionBypass("backups catalog"); + return (new QueryAction().execute(input).getRecords()); + } + + + + /******************************************************************************* + ** Query by one equals criterion, newest first by id. + *******************************************************************************/ + public List where(String table, String field, Serializable value) throws QException + { + return (query(table, new QQueryFilter(new QFilterCriteria(field, QCriteriaOperator.EQUALS, value)).withOrderBy(new QFilterOrderBy("id", false)))); + } + + + + /******************************************************************************* + ** Insert one artifact row. + *******************************************************************************/ + public Integer insertArtifact(Integer runId, String type, String tableName, Integer partNumber, String objectKey, String format, String compression, + boolean encrypted, long rows, long bytesRaw, long bytesStored, String sha256Stored, String sha256Raw, String rowsHash, + String firstKey, String lastKey, String tableFingerprint) throws QException + { + Map v = new LinkedHashMap<>(); + v.put("backupRunId", runId); + v.put("artifactType", type); + v.put("tableName", tableName); + v.put("partNumber", partNumber); + v.put("objectKey", objectKey); + v.put("format", format); + v.put("compression", compression); + v.put("encrypted", encrypted); + v.put("rowCount", rows); + v.put("bytesRaw", bytesRaw); + v.put("bytesStored", bytesStored); + v.put("sha256Stored", sha256Stored); + v.put("sha256Raw", sha256Raw); + v.put("rowsHash", rowsHash); + v.put("firstKey", firstKey); + v.put("lastKey", lastKey); + v.put("tableFingerprint", tableFingerprint); + v.put("status", "COMPLETE"); + return (insert(BackupsTables.ARTIFACT, v)); + } + + + + /******************************************************************************* + ** The file store registry over backups_file_object: known hashes are those with + ** a live (not deleted) row. + *******************************************************************************/ + public FileCopier.FileStoreRegistry fileStoreRegistry(String bucketAlias, Integer runId, String keyFingerprint) + { + return (new FileCopier.FileStoreRegistry() + { + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public boolean isKnown(String sha256) + { + try + { + return (existing(sha256).isPresent()); + } + catch(QException e) + { + throw (new IllegalStateException("Could not look up a stored file: " + e.getMessage(), e)); + } + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public void register(String sha256, long sizeBytes, String contentType, String casKey, boolean encrypted) + { + try + { + Optional existing = existing(sha256); + if(existing.isPresent()) + { + return; + } + List deleted = where(BackupsTables.FILE_OBJECT, "sha256", sha256); + Map v = new LinkedHashMap<>(); + v.put("sha256", sha256); + v.put("sizeBytes", sizeBytes); + v.put("contentType", contentType); + v.put("storageBucketAlias", bucketAlias); + v.put("objectKey", casKey); + v.put("encrypted", encrypted); + v.put("encryptionKeyFingerprint", keyFingerprint); + v.put("firstRunId", runId); + v.put("lastReferencedRunId", runId); + v.put("referenceCount", 0); + v.put("deletedDate", null); + if(deleted.isEmpty()) + { + insert(BackupsTables.FILE_OBJECT, v); + } + else + { + update(BackupsTables.FILE_OBJECT, deleted.get(0).getValueInteger("id"), v); + } + } + catch(QException e) + { + throw (new IllegalStateException("Could not record a stored file: " + e.getMessage(), e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private Optional existing(String sha256) throws QException + { + return (where(BackupsTables.FILE_OBJECT, "sha256", sha256).stream().filter(r -> r.getValue("deletedDate") == null).findFirst()); + } + }); + } + + + + /******************************************************************************* + ** The runs that are RUNNING for a plan, with a heartbeat newer than the cutoff. + *******************************************************************************/ + public List liveRunsForPlan(Integer planId, Instant heartbeatCutoff) throws QException + { + List live = new ArrayList<>(); + for(QRecord run : where(BackupsTables.RUN, "backupPlanId", planId)) + { + Instant heartbeat = run.getValueInstant("heartbeatDate"); + if("RUNNING".equals(run.getValueString("status")) && heartbeat != null && heartbeat.isAfter(heartbeatCutoff)) + { + live.add(run); + } + } + return (live); + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/run/BackupEngine.java b/src/main/java/com/kingsrook/qbits/backups/run/BackupEngine.java new file mode 100644 index 0000000..1e7449c --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/run/BackupEngine.java @@ -0,0 +1,768 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.run; + + +import java.io.IOException; +import java.io.OutputStream; +import java.io.Serializable; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.function.Function; +import java.util.zip.GZIPOutputStream; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.kingsrook.qbits.backups.BackupsRuntime; +import com.kingsrook.qbits.backups.alerts.BackupAlertSink; +import com.kingsrook.qbits.backups.codec.HashingOutputStream; +import com.kingsrook.qbits.backups.codec.TableArtifactWriter; +import com.kingsrook.qbits.backups.files.FileCopier; +import com.kingsrook.qbits.backups.files.FileReference; +import com.kingsrook.qbits.backups.files.FileReferenceProvider; +import com.kingsrook.qbits.backups.manifest.BackupManifest; +import com.kingsrook.qbits.backups.manifest.TableFingerprints; +import com.kingsrook.qbits.backups.model.BackupsTables; +import com.kingsrook.qbits.backups.scope.ScopeResolver; +import com.kingsrook.qbits.backups.storage.BackupStorage; +import com.kingsrook.qbits.backups.storage.WriteOptions; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Executes one backup run (spec 8 and 9.2). + ** + ** The order is the contract: the run row is created RUNNING; every table is + ** streamed in primary key order into numbered parts (with their SHA-256 and the + ** rowsHash); the files the rows reference are copied into the deduplicated + ** content-addressed store; checksums.sha256 is written; and the manifest is + ** written LAST, then checked with a head. A run with no manifest did not + ** complete. The run row is updated every page (heartbeat), and a failure marks + ** the run FAILED with a scrubbed message (never row values). + *******************************************************************************/ +public class BackupEngine +{ + private static final QLogger LOG = QLogger.getLogger(BackupEngine.class); + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private static final DateTimeFormatter RUN_KEY_FORMAT = DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'").withZone(ZoneOffset.UTC); + private static final DateTimeFormatter DATE_PATH = DateTimeFormatter.ofPattern("yyyy/MM/dd").withZone(ZoneOffset.UTC); + + private static final long MAX_ROWS_PER_PART = 1_000_000L; + + private final BackupCatalog catalog = new BackupCatalog(); + + + + /******************************************************************************* + ** Run a backup. Never throws for a backup failure: the result carries the + ** status and message (and the run row is FAILED). + *******************************************************************************/ + public BackupRunResult execute(BackupRunRequest request) + { + BackupRunResult result = new BackupRunResult(); + Instant started = request.getClock().instant(); + String runKey = RUN_KEY_FORMAT.format(started) + "-" + UUID.randomUUID().toString().substring(0, 4); + String planPrefix = request.getSettings().string("prefix", "qqq-backups"); + String instancePrefix = trimSlashes(planPrefix) + "/" + request.getInstanceName() + "/"; + String runPrefix = instancePrefix + request.getSettings().planKey() + "/" + DATE_PATH.format(started) + "/" + runKey + "/"; + result.setRunKey(runKey); + result.setRunPrefix(runPrefix); + + try + { + Map runValues = new LinkedHashMap<>(); + runValues.put("backupPlanId", request.getSettings().getPlanId()); + runValues.put("runKey", runKey); + runValues.put("trigger", request.getTrigger()); + runValues.put("requestedByUserId", request.getRequestedBy()); + runValues.put("status", "RUNNING"); + runValues.put("startDate", started); + runValues.put("heartbeatDate", started); + runValues.put("tablesPlanned", request.getScope().tables().size()); + runValues.put("storageBucketAlias", request.getBucketAlias()); + runValues.put("runPrefix", runPrefix); + runValues.put("instanceName", request.getInstanceName()); + runValues.put("appVersion", request.getAppVersion()); + runValues.put("securityScope", request.getSecurityScope()); + runValues.put("encryptionKeyRef", request.getEncryptionKeyRef()); + runValues.put("encryptionKeyFingerprint", request.getEncryptionKeyFingerprint()); + runValues.put("verificationStatus", "UNVERIFIED"); + result.setRunId(catalog.insert(BackupsTables.RUN, runValues)); + } + catch(Exception e) + { + result.setStatus("FAILED"); + result.setErrorMessage("Could not create the run record: " + e.getMessage()); + return (result); + } + + try + { + doRun(request, result, runPrefix, instancePrefix, started); + } + catch(CancelledException e) + { + finish(request, result, "CANCELLED", "Cancelled", started); + } + catch(TimedOutException e) + { + finish(request, result, "TIMED_OUT", e.getMessage(), started); + } + catch(Exception e) + { + LOG.warn("Backup run failed", e, logPair("runKey", runKey)); + finish(request, result, "FAILED", scrub(e), started); + } + return (result); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void doRun(BackupRunRequest request, BackupRunResult result, String runPrefix, String instancePrefix, Instant started) throws Exception + { + QInstance qInstance = QContext.getQInstance(); + BackupStorage storage = request.getStorage(); + var settings = request.getSettings(); + String format = settings.format(); + String compression = settings.compression(); + boolean copyFiles = "COPY".equals(settings.fileObjectPolicy()); + long maxMillis = settings.integer("maxDurationMinutes", 240) * 60_000L; + + if("ZSTD".equals(compression) || "PARQUET".equals(format)) + { + throw (new IllegalStateException("The " + ("ZSTD".equals(compression) ? "ZSTD compression" : "PARQUET format") + " is not available yet; use GZIP and JSONL or CSV.")); + } + String sensitivePolicy = settings.string("sensitiveFieldPolicy", "EXCLUDE"); + if("INCLUDE_ENCRYPTED_ONLY".equals(sensitivePolicy) && !request.getTransform().encrypts()) + { + throw (new IllegalStateException("The sensitive field policy INCLUDE_ENCRYPTED_ONLY requires client-side encryption.")); + } + String consistency = consistencyFor(settings.string("consistency", "SNAPSHOT_PREFERRED"), result); + + Function sourceResolver = request.getSourceResolver() != null ? request.getSourceResolver() : alias -> storage; + FileCopier copier = new FileCopier(sourceResolver, request.getBucketAlias(), storage, instancePrefix + "_files/sha256/", request.getTransform(), + catalog.fileStoreRegistry(request.getBucketAlias(), result.getRunId(), request.getEncryptionKeyFingerprint()), !copyFiles, result.getRunKey()); + + List manifestTables = new ArrayList<>(); + List checksums = new ArrayList<>(); + List fingerprints = new ArrayList<>(); + FileIndexWriter indexWriter = null; + long rows = 0; + long bytesStored = 0; + long bytesRaw = 0; + int done = 0; + int referenced = 0; + int copied = 0; + int deduplicated = 0; + int missing = 0; + long fileBytes = 0; + long fileBytesRef = 0; + + try + { + for(ScopeResolver.ResolvedTable resolved : request.getScope().tables()) + { + QTableMetaData table = qInstance.getTable(resolved.name()); + String backendType = qInstance.getBackend(table.getBackendName()) == null ? null : qInstance.getBackend(table.getBackendName()).getBackendType(); + + List excluded = excludedFields(table, resolved, sensitivePolicy, settings.string("heavyFieldPolicy", "INLINE")); + List excludedNames = excluded.stream().map(BackupManifest.ExcludedField::name).toList(); + + List writers = new ArrayList<>(); + String tablePrefix = runPrefix + "tables/" + table.getName() + "/"; + for(String f : "JSONL_AND_CSV".equals(format) ? List.of("JSONL", "CSV") : List.of("CSV".equals(format) ? "CSV" : "JSONL")) + { + writers.add(new TableArtifactWriter(storage, tablePrefix, table, excludedNames, f, compression, settings.targetPartBytes(), MAX_ROWS_PER_PART, + request.getTransform(), result.getRunKey())); + } + + List references = new ArrayList<>(); + long[] counter = { 0 }; + Instant deadline = started.plusMillis(maxMillis); + Throwable[] writeError = { null }; + + request.getReader().read(table, resolved.filterJson(), null, settings.pageSize(), null, record -> + { + try + { + for(TableArtifactWriter writer : writers) + { + writer.write(record); + } + } + catch(IOException e) + { + throw (new IllegalStateException("Could not write a row of " + table.getName() + ": " + e.getMessage(), e)); + } + if(copyFiles || "REFERENCE_ONLY".equals(settings.fileObjectPolicy())) + { + for(FileReferenceProvider provider : request.getFileProviders()) + { + references.addAll(provider.referencesIn(table, record)); + } + } + counter[0]++; + if(counter[0] % settings.pageSize() == 0) + { + heartbeat(request, result, deadline, table.getName(), counter[0]); + } + }); + + String rowsHash = null; + List parts = new ArrayList<>(); + String fingerprint = TableFingerprints.of(table, backendType); + fingerprints.add(fingerprint); + for(TableArtifactWriter writer : writers) + { + List results = writer.finish(); + rowsHash = writer.rowsHash(); + for(TableArtifactWriter.PartResult part : results) + { + String relative = part.key().substring(runPrefix.length()); + parts.add(new BackupManifest.Part(relative, part.rows(), part.bytesRaw(), part.bytesStored(), part.sha256Stored(), part.sha256Raw(), part.format(), part.compression(), part.encrypted())); + checksums.add(part.sha256Stored() + " " + relative); + bytesStored += part.bytesStored(); + bytesRaw += part.bytesRaw(); + catalog.insertArtifact(result.getRunId(), "TABLE_DATA", table.getName(), part.partNumber(), relative, part.format(), part.compression(), + part.encrypted(), part.rows(), part.bytesRaw(), part.bytesStored(), part.sha256Stored(), part.sha256Raw(), + part.partNumber() == results.size() ? rowsHash : null, part.firstKey(), part.lastKey(), fingerprint); + } + } + rows += counter[0]; + + for(FileReference ref : references) + { + if(indexWriter == null) + { + indexWriter = new FileIndexWriter(storage, runPrefix + "files/index-00001.jsonl.gz", result.getRunKey()); + } + FileCopier.IndexEntry entry = copier.process(ref); + indexWriter.write(entry); + referenced++; + switch(entry.status()) + { + case FileCopier.COPIED -> + { + copied++; + fileBytes += entry.size(); + } + case FileCopier.DEDUPLICATED -> deduplicated++; + case FileCopier.MISSING -> missing++; + default -> fileBytesRef += Math.max(0, entry.size()); + } + if(request.getCancelRequested().getAsBoolean()) + { + throw (new CancelledException()); + } + } + + Map keyRange = new LinkedHashMap<>(); + keyRange.put("min", writers.get(0).getFirstKey()); + keyRange.put("max", writers.get(0).getLastKey()); + manifestTables.add(new BackupManifest.TableEntry(table.getName(), resolved.role().name(), table.getBackendName(), table.getPrimaryKeyField(), + fingerprint, TableFingerprints.descriptor(table, backendType), resolved.filterJson(), excluded, counter[0], rowsHash, keyRange, parts, consistency)); + done++; + catalogProgress(result, done, rows, bytesRaw, bytesStored); + if(request.getCancelRequested().getAsBoolean()) + { + throw (new CancelledException()); + } + } + } + finally + { + if(indexWriter != null) + { + indexWriter.close(); + } + } + + List indexKeys = new ArrayList<>(); + if(indexWriter != null) + { + indexKeys.add("files/index-00001.jsonl.gz"); + checksums.add(indexWriter.sha256Stored() + " files/index-00001.jsonl.gz"); + catalog.insertArtifact(result.getRunId(), "FILE_OBJECTS_INDEX", null, 1, "files/index-00001.jsonl.gz", "JSONL", "GZIP", request.getTransform().encrypts(), + indexWriter.rows(), indexWriter.bytesRaw(), indexWriter.bytesStored(), indexWriter.sha256Stored(), indexWriter.sha256Raw(), null, null, null, null); + bytesStored += indexWriter.bytesStored(); + } + + if(missing > 0) + { + if("FAIL".equals(settings.string("fileMissingPolicy", "WARN"))) + { + throw (new IllegalStateException(missing + " referenced files are missing at the source")); + } + result.getWarnings().add(missing + " referenced files were missing at the source"); + } + + ///////////////////////////////////////////////////////// + // checksums, then the manifest LAST (the commit mark) // + ///////////////////////////////////////////////////////// + byte[] checksumBytes = (String.join("\n", checksums) + "\n").getBytes(StandardCharsets.UTF_8); + putBytes(storage, runPrefix + "checksums.sha256", checksumBytes, result.getRunKey()); + catalog.insertArtifact(result.getRunId(), "CHECKSUMS", null, 1, "checksums.sha256", "TEXT", "NONE", false, checksums.size(), checksumBytes.length, + checksumBytes.length, HashingOutputStream.sha256Hex(checksumBytes), HashingOutputStream.sha256Hex(checksumBytes), null, null, null, null); + + Instant finished = request.getClock().instant(); + List order = request.getScope().names(); + BackupManifest manifest = new BackupManifest(BackupManifest.CURRENT_VERSION, result.getRunKey(), + new BackupManifest.PlanInfo(settings.planKey(), request.getPlanName(), settings.manifestSettings()), + new BackupManifest.InstanceInfo(request.getInstanceName(), request.getAppVersion(), qqqVersion(), List.of()), + started.toString(), finished.toString(), request.getRequestedBy(), request.getSecurityScope(), + new BackupManifest.ConsistencyInfo(result.getConsistencyAchieved(), List.of()), + request.getTransform().encrypts() + ? new BackupManifest.EncryptionInfo("CLIENT_AES256_GCM", "tink-streaming-aead-AES256_GCM_HKDF_1MB", request.getEncryptionKeyRef(), request.getEncryptionKeyFingerprint()) + : new BackupManifest.EncryptionInfo("NONE", null, null, null), + manifestTables, order, + new BackupManifest.FilesInfo(settings.fileObjectPolicy(), indexKeys, copied + deduplicated, fileBytes + fileBytesRef, missing), + result.getWarnings(), + new BackupManifest.Totals(manifestTables.size(), rows, bytesStored)); + byte[] manifestBytes = manifest.toJsonBytes(); + String manifestKey = runPrefix + "manifest.json"; + putBytes(storage, manifestKey, manifestBytes, result.getRunKey()); + if(storage.head(manifestKey).isEmpty()) + { + throw (new IllegalStateException("The manifest was written but is not visible in the bucket")); + } + catalog.insertArtifact(result.getRunId(), "MANIFEST", null, 1, "manifest.json", "JSON", "NONE", false, manifestTables.size(), manifestBytes.length, + manifestBytes.length, HashingOutputStream.sha256Hex(manifestBytes), HashingOutputStream.sha256Hex(manifestBytes), null, null, null, null); + + result.setManifestKey(manifestKey); + result.setManifestSha256(HashingOutputStream.sha256Hex(manifestBytes)); + result.setTables(manifestTables.size()); + result.setRows(rows); + result.setBytesRaw(bytesRaw); + result.setBytesStored(bytesStored); + result.setFilesReferenced(referenced); + result.setFilesCopied(copied); + result.setFilesDeduplicated(deduplicated); + result.setFilesMissing(missing); + result.setFileBytesCopied(fileBytes); + + Map extra = new LinkedHashMap<>(); + extra.put("manifestKey", manifestKey); + extra.put("manifestSha256", result.getManifestSha256()); + extra.put("schemaFingerprint", TableFingerprints.combine(fingerprints)); + extra.put("qqqVersion", qqqVersion()); + extra.put("consistencyAchieved", result.getConsistencyAchieved()); + extra.put("filesReferenced", referenced); + extra.put("filesCopied", copied); + extra.put("filesDeduplicated", deduplicated); + extra.put("filesMissing", missing); + extra.put("fileBytesCopied", fileBytes); + extra.put("fileBytesReferenced", fileBytesRef); + extra.put("tablesDone", manifestTables.size()); + extra.put("rowsTotal", rows); + extra.put("bytesRaw", bytesRaw); + extra.put("bytesStored", bytesStored); + extra.put("warningCount", result.getWarnings().size()); + finishWith(request, result, result.getWarnings().isEmpty() ? "SUCCEEDED" : "SUCCEEDED_WITH_WARNINGS", null, started, extra); + } + + + + /******************************************************************************* + ** Work out the consistency achieved. Until the core consistent-read hook ships + ** (QQQ core PR feature/consistent-read-transactions, next 4.1.x RC), no backend + ** reports a snapshot, so: SNAPSHOT_REQUIRED fails before writing anything, + ** SNAPSHOT_PREFERRED records BEST_EFFORT with a warning, PER_TABLE records PER_TABLE. + *******************************************************************************/ + private String consistencyFor(String requested, BackupRunResult result) + { + switch(requested) + { + case "SNAPSHOT_REQUIRED": + throw (new IllegalStateException("The plan requires a consistent snapshot, which this version of QQQ cannot open yet. Use SNAPSHOT_PREFERRED or PER_TABLE.")); + case "PER_TABLE": + result.setConsistencyAchieved("PER_TABLE"); + return ("PER_TABLE"); + default: + result.setConsistencyAchieved("BEST_EFFORT"); + result.getWarnings().add("No consistent snapshot was available: every table was read on its own (BEST_EFFORT)"); + return ("BEST_EFFORT"); + } + } + + + + /******************************************************************************* + ** The fields left out of a table, by policy. The primary key is never left out. + *******************************************************************************/ + private List excludedFields(QTableMetaData table, ScopeResolver.ResolvedTable resolved, String sensitivePolicy, String heavyPolicy) + { + Map out = new LinkedHashMap<>(); + for(QFieldMetaData field : table.getFields().values()) + { + if(field.getName().equals(table.getPrimaryKeyField())) + { + continue; + } + if("EXCLUDE".equals(sensitivePolicy) && (field.getIsSensitive() || field.getType() == QFieldType.PASSWORD)) + { + out.put(field.getName(), "SENSITIVE_POLICY"); + } + else if("EXCLUDE".equals(heavyPolicy) && field.getIsHeavy()) + { + out.put(field.getName(), "HEAVY_POLICY"); + } + } + for(String name : resolved.excludedFields()) + { + if(table.getFields().containsKey(name) && !name.equals(table.getPrimaryKeyField())) + { + out.put(name, "SCOPE"); + } + } + return (out.entrySet().stream().map(e -> new BackupManifest.ExcludedField(e.getKey(), e.getValue())).toList()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void heartbeat(BackupRunRequest request, BackupRunResult result, Instant deadline, String tableName, long rowsSoFar) + { + Instant now = request.getClock().instant(); + if(request.getCancelRequested().getAsBoolean()) + { + throw (new CancelledException()); + } + if(now.isAfter(deadline)) + { + throw (new TimedOutException("The run exceeded its maximum duration")); + } + try + { + catalog.update(BackupsTables.RUN, result.getRunId(), Map.of("heartbeatDate", now)); + } + catch(Exception e) + { + LOG.warn("Could not record a heartbeat", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void catalogProgress(BackupRunResult result, int done, long rows, long bytesRaw, long bytesStored) + { + try + { + Map v = new LinkedHashMap<>(); + v.put("tablesDone", done); + v.put("rowsTotal", rows); + v.put("bytesRaw", bytesRaw); + v.put("bytesStored", bytesStored); + catalog.update(BackupsTables.RUN, result.getRunId(), v); + } + catch(Exception e) + { + LOG.warn("Could not record progress", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void finish(BackupRunRequest request, BackupRunResult result, String status, String message, Instant started) + { + try + { + finishWith(request, result, status, message, started, new LinkedHashMap<>()); + } + catch(Exception e) + { + LOG.warn("Could not finish the run record", e); + result.setStatus(status); + result.setErrorMessage(message); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void finishWith(BackupRunRequest request, BackupRunResult result, String status, String message, Instant started, Map extra) throws Exception + { + Instant end = request.getClock().instant(); + Map v = new LinkedHashMap<>(extra); + v.put("status", status); + v.put("endDate", end); + v.put("durationMs", end.toEpochMilli() - started.toEpochMilli()); + v.put("heartbeatDate", end); + v.put("errorMessage", message); + v.put("warningCount", result.getWarnings().size()); + catalog.update(BackupsTables.RUN, result.getRunId(), v); + result.setStatus(status); + result.setErrorMessage(message); + signal(request, result, status, message); + } + + + + /******************************************************************************* + ** Update the plan denormalized fields, check in the heartbeat, and raise or resolve alerts. + *******************************************************************************/ + private void signal(BackupRunRequest request, BackupRunResult result, String status, String message) throws Exception + { + Integer planId = request.getSettings().getPlanId(); + if(planId == null) + { + return; + } + boolean ok = "SUCCEEDED".equals(status) || "SUCCEEDED_WITH_WARNINGS".equals(status); + Map plan = new LinkedHashMap<>(); + plan.put("lastRunId", result.getRunId()); + if(ok) + { + plan.put("lastSuccessRunId", result.getRunId()); + plan.put("lastSuccessDate", request.getClock().instant()); + plan.put("slaStatus", "OK"); + } + catalog.update(BackupsTables.PLAN, planId, plan); + + BackupAlertSink sink = BackupsRuntime.getAlertSink(); + String planKey = request.getSettings().planKey(); + if(ok) + { + sink.resolve("backups.run.failed", "backups.run.failed:" + planKey); + sink.heartbeat("backups.plan:" + planKey, request.getSettings().integer("slaHours", 26) * 3600L, 0L); + if(result.getFilesMissing() > 0) + { + sink.raise("backups.files.missing", "backups.files.missing:" + planKey, BackupAlertSink.Severity.WARNING, "Referenced files are missing", + Map.of("plan", planKey, "missing", String.valueOf(result.getFilesMissing()), "run", result.getRunKey())); + } + else + { + sink.resolve("backups.files.missing", "backups.files.missing:" + planKey); + } + } + else if(!"CANCELLED".equals(status)) + { + sink.raise("backups.run.failed", "backups.run.failed:" + planKey, BackupAlertSink.Severity.WARNING, "A backup run failed", + Map.of("plan", planKey, "status", status, "run", result.getRunKey())); + } + } + + + + /******************************************************************************* + ** A message safe to store: the exception type and message, never row values (the + ** engine never puts values in messages), truncated. + *******************************************************************************/ + private static String scrub(Exception e) + { + String message = e.getClass().getSimpleName() + ": " + e.getMessage(); + return (message.length() > 2000 ? message.substring(0, 2000) : message); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void putBytes(BackupStorage storage, String key, byte[] bytes, String runKey) throws IOException + { + try(OutputStream out = storage.openWrite(key, WriteOptions.create().withMetadata("qqq-backup-run", runKey))) + { + out.write(bytes); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static String trimSlashes(String s) + { + String r = s; + while(r.startsWith("/")) + { + r = r.substring(1); + } + while(r.endsWith("/")) + { + r = r.substring(0, r.length() - 1); + } + return (r); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + static String qqqVersion() + { + String v = QInstance.class.getPackage().getImplementationVersion(); + return (v == null ? "unknown" : v); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + static class CancelledException extends RuntimeException + { + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + static class TimedOutException extends RuntimeException + { + /******************************************************************************* + ** Constructor. + *******************************************************************************/ + TimedOutException(String message) + { + super(message); + } + } + + + + /*************************************************************************** + ** The per-run file index, written as it goes: one JSON line per reference. + ***************************************************************************/ + static class FileIndexWriter + { + private final HashingOutputStream raw; + private final HashingOutputStream stored; + private long rows = 0; + + + + /******************************************************************************* + ** Constructor. + *******************************************************************************/ + FileIndexWriter(BackupStorage storage, String key, String runKey) throws IOException + { + stored = new HashingOutputStream(storage.openWrite(key, WriteOptions.create().withMetadata("qqq-backup-run", runKey))); + raw = new HashingOutputStream(new GZIPOutputStream(stored)); + } + + + + /******************************************************************************* + ** Write one entry. + *******************************************************************************/ + void write(FileCopier.IndexEntry entry) throws IOException + { + Map line = new LinkedHashMap<>(); + line.put("table", entry.tableName()); + line.put("recordKey", entry.recordKey()); + line.put("field", entry.field()); + line.put("sourceBucket", entry.sourceBucketAlias()); + line.put("sourceKey", entry.sourceKey()); + line.put("sha256", entry.sha256()); + line.put("size", entry.size()); + line.put("status", entry.status()); + line.put("casKey", entry.casKey()); + raw.write((MAPPER.writeValueAsString(line) + "\n").getBytes(StandardCharsets.UTF_8)); + rows++; + } + + + + /******************************************************************************* + ** Close the stream. + *******************************************************************************/ + void close() throws IOException + { + raw.close(); + } + + + + /******************************************************************************* + ** Rows written. + *******************************************************************************/ + long rows() + { + return (rows); + } + + + + /******************************************************************************* + ** Raw byte count. + *******************************************************************************/ + long bytesRaw() + { + return (raw.getCount()); + } + + + + /******************************************************************************* + ** Stored byte count. + *******************************************************************************/ + long bytesStored() + { + return (stored.getCount()); + } + + + + /******************************************************************************* + ** Raw hash. + *******************************************************************************/ + String sha256Raw() + { + return (raw.getSha256()); + } + + + + /******************************************************************************* + ** Stored hash. + *******************************************************************************/ + String sha256Stored() + { + return (stored.getSha256()); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/run/BackupRunFactory.java b/src/main/java/com/kingsrook/qbits/backups/run/BackupRunFactory.java new file mode 100644 index 0000000..8fa6abd --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/run/BackupRunFactory.java @@ -0,0 +1,98 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.run; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qbits.backups.BackupsQBitConfig; +import com.kingsrook.qbits.backups.BackupsRuntime; +import com.kingsrook.qbits.backups.model.BackupsTables; +import com.kingsrook.qbits.backups.scope.PlanSettings; +import com.kingsrook.qbits.backups.scope.ScopeResolver; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.session.QSession; + + +/******************************************************************************* + ** Builds the BackupRunRequest for a plan record: loads the scope rows, resolves + ** the tables, and picks the storage by the plan bucket alias. + *******************************************************************************/ +public final class BackupRunFactory +{ + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private BackupRunFactory() + { + } + + + + /******************************************************************************* + ** The scope rules of a plan. + *******************************************************************************/ + public static List scopeRules(Integer planId) throws QException + { + List rules = new ArrayList<>(); + for(QRecord row : new BackupCatalog().where(BackupsTables.PLAN_SCOPE, "backupPlanId", planId)) + { + rules.add(new ScopeResolver.ScopeRule("INCLUDE".equals(row.getValueString("action")), row.getValueString("selectorType"), row.getValueString("selectorValue"), + row.getValueString("filterJson"), !Boolean.FALSE.equals(row.getValueBoolean("includeAssociatedChildren")), row.getValueString("fieldExclusions"))); + } + return (rules); + } + + + + /******************************************************************************* + ** Resolve the scope of a plan record. + *******************************************************************************/ + public static ScopeResolver.ResolvedScope resolveScope(QRecord plan) throws QException + { + return (new ScopeResolver(QContext.getQInstance()).resolve(PlanSettings.of(plan), scopeRules(plan.getValueInteger("id")))); + } + + + + /******************************************************************************* + ** The request to run a plan. + *******************************************************************************/ + public static BackupRunRequest forPlan(QRecord plan, String trigger) throws QException + { + BackupsQBitConfig config = BackupsRuntime.getConfig(); + PlanSettings settings = PlanSettings.of(plan); + QSession session = QContext.getQSession(); + String user = session == null || session.getUser() == null ? "system:scheduler" : session.getUser().getIdReference(); + + return (new BackupRunRequest() + .withSettings(settings) + .withScope(resolveScope(plan)) + .withTrigger(trigger) + .withRequestedBy("SCHEDULED".equals(trigger) ? "system:scheduler" : user) + .withSecurityScope("SCHEDULED".equals(trigger) ? "FULL" : "FULL") + .withBucketAlias(settings.string("storageBucketAlias", null)) + .withStorage(BackupsRuntime.storageFor(settings.string("storageBucketAlias", null))) + .withSourceResolver(BackupsRuntime::storageFor) + .withInstanceName(config.getInstanceName() != null ? config.getInstanceName() : "qqq") + .withPlanName(plan.getValueString("name")) + .withClock(BackupsRuntime.getClock())); + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/run/BackupRunRequest.java b/src/main/java/com/kingsrook/qbits/backups/run/BackupRunRequest.java new file mode 100644 index 0000000..1f9e6b7 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/run/BackupRunRequest.java @@ -0,0 +1,413 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.run; + + +import java.time.Clock; +import java.util.ArrayList; +import java.util.List; +import java.util.function.BooleanSupplier; +import com.kingsrook.qbits.backups.codec.ArtifactTransform; +import com.kingsrook.qbits.backups.files.FileReferenceProvider; +import com.kingsrook.qbits.backups.scope.PlanSettings; +import com.kingsrook.qbits.backups.scope.ScopeResolver; +import com.kingsrook.qbits.backups.storage.BackupStorage; + + +/******************************************************************************* + ** Everything a backup run needs, resolved by the caller (the process step, the + ** scheduler or a test). + *******************************************************************************/ +public class BackupRunRequest +{ + private PlanSettings settings; + private ScopeResolver.ResolvedScope scope; + private String trigger = "MANUAL"; + private String requestedBy = "system"; + private String securityScope = "FULL"; + private BackupStorage storage; + private String bucketAlias; + private String instanceName = "qqq"; + private String appVersion; + private String planName; + private ArtifactTransform transform = ArtifactTransform.IDENTITY; + private String encryptionKeyRef; + private String encryptionKeyFingerprint; + private List fileProviders = new ArrayList<>(List.of(new FileReferenceProvider.FromTableMetaData())); + private TableReader reader = new TableReader.Keyset(); + private Clock clock = Clock.systemUTC(); + private BooleanSupplier cancelRequested = () -> false; + private java.util.function.Function sourceResolver; + + + + /******************************************************************************* + ** Getter for settings + *******************************************************************************/ + public PlanSettings getSettings() + { + return (settings); + } + + + + /******************************************************************************* + ** Fluent setter for settings + *******************************************************************************/ + public BackupRunRequest withSettings(PlanSettings settings) + { + this.settings = settings; + return (this); + } + + + + /******************************************************************************* + ** Getter for scope + *******************************************************************************/ + public ScopeResolver.ResolvedScope getScope() + { + return (scope); + } + + + + /******************************************************************************* + ** Fluent setter for scope + *******************************************************************************/ + public BackupRunRequest withScope(ScopeResolver.ResolvedScope scope) + { + this.scope = scope; + return (this); + } + + + + /******************************************************************************* + ** Getter for trigger + *******************************************************************************/ + public String getTrigger() + { + return (trigger); + } + + + + /******************************************************************************* + ** Fluent setter for trigger + *******************************************************************************/ + public BackupRunRequest withTrigger(String trigger) + { + this.trigger = trigger; + return (this); + } + + + + /******************************************************************************* + ** Getter for requestedBy + *******************************************************************************/ + public String getRequestedBy() + { + return (requestedBy); + } + + + + /******************************************************************************* + ** Fluent setter for requestedBy + *******************************************************************************/ + public BackupRunRequest withRequestedBy(String requestedBy) + { + this.requestedBy = requestedBy; + return (this); + } + + + + /******************************************************************************* + ** Getter for securityScope (FULL, or USER when the run saw only what a user may see) + *******************************************************************************/ + public String getSecurityScope() + { + return (securityScope); + } + + + + /******************************************************************************* + ** Fluent setter for securityScope + *******************************************************************************/ + public BackupRunRequest withSecurityScope(String securityScope) + { + this.securityScope = securityScope; + return (this); + } + + + + /******************************************************************************* + ** Getter for storage + *******************************************************************************/ + public BackupStorage getStorage() + { + return (storage); + } + + + + /******************************************************************************* + ** Fluent setter for storage + *******************************************************************************/ + public BackupRunRequest withStorage(BackupStorage storage) + { + this.storage = storage; + return (this); + } + + + + /******************************************************************************* + ** Getter for bucketAlias + *******************************************************************************/ + public String getBucketAlias() + { + return (bucketAlias); + } + + + + /******************************************************************************* + ** Fluent setter for bucketAlias + *******************************************************************************/ + public BackupRunRequest withBucketAlias(String bucketAlias) + { + this.bucketAlias = bucketAlias; + return (this); + } + + + + /******************************************************************************* + ** Getter for instanceName + *******************************************************************************/ + public String getInstanceName() + { + return (instanceName); + } + + + + /******************************************************************************* + ** Fluent setter for instanceName + *******************************************************************************/ + public BackupRunRequest withInstanceName(String instanceName) + { + this.instanceName = instanceName; + return (this); + } + + + + /******************************************************************************* + ** Getter for appVersion + *******************************************************************************/ + public String getAppVersion() + { + return (appVersion); + } + + + + /******************************************************************************* + ** Fluent setter for appVersion + *******************************************************************************/ + public BackupRunRequest withAppVersion(String appVersion) + { + this.appVersion = appVersion; + return (this); + } + + + + /******************************************************************************* + ** Getter for planName + *******************************************************************************/ + public String getPlanName() + { + return (planName); + } + + + + /******************************************************************************* + ** Fluent setter for planName + *******************************************************************************/ + public BackupRunRequest withPlanName(String planName) + { + this.planName = planName; + return (this); + } + + + + /******************************************************************************* + ** Getter for transform + *******************************************************************************/ + public ArtifactTransform getTransform() + { + return (transform); + } + + + + /******************************************************************************* + ** Set the artifact transform (encryption) with the key reference and fingerprint to record. + *******************************************************************************/ + public BackupRunRequest withTransform(ArtifactTransform transform, String keyRef, String keyFingerprint) + { + this.transform = transform; + this.encryptionKeyRef = keyRef; + this.encryptionKeyFingerprint = keyFingerprint; + return (this); + } + + + + /******************************************************************************* + ** Getter for encryptionKeyRef + *******************************************************************************/ + public String getEncryptionKeyRef() + { + return (encryptionKeyRef); + } + + + + /******************************************************************************* + ** Getter for encryptionKeyFingerprint + *******************************************************************************/ + public String getEncryptionKeyFingerprint() + { + return (encryptionKeyFingerprint); + } + + + + /******************************************************************************* + ** Getter for fileProviders + *******************************************************************************/ + public List getFileProviders() + { + return (fileProviders); + } + + + + /******************************************************************************* + ** Add a file reference provider. + *******************************************************************************/ + public BackupRunRequest withFileProvider(FileReferenceProvider provider) + { + this.fileProviders.add(provider); + return (this); + } + + + + /******************************************************************************* + ** Getter for reader + *******************************************************************************/ + public TableReader getReader() + { + return (reader); + } + + + + /******************************************************************************* + ** Fluent setter for reader + *******************************************************************************/ + public BackupRunRequest withReader(TableReader reader) + { + this.reader = reader; + return (this); + } + + + + /******************************************************************************* + ** Getter for clock + *******************************************************************************/ + public Clock getClock() + { + return (clock); + } + + + + /******************************************************************************* + ** Fluent setter for clock + *******************************************************************************/ + public BackupRunRequest withClock(Clock clock) + { + this.clock = clock; + return (this); + } + + + + /******************************************************************************* + ** Getter for cancelRequested + *******************************************************************************/ + public BooleanSupplier getCancelRequested() + { + return (cancelRequested); + } + + + + /******************************************************************************* + ** Fluent setter for cancelRequested + *******************************************************************************/ + public BackupRunRequest withCancelRequested(BooleanSupplier cancelRequested) + { + this.cancelRequested = cancelRequested; + return (this); + } + + + + /******************************************************************************* + ** Getter for sourceResolver (bucket alias of the referencing files to storage); defaults to the plan storage + *******************************************************************************/ + public java.util.function.Function getSourceResolver() + { + return (sourceResolver); + } + + + + /******************************************************************************* + ** Fluent setter for sourceResolver + *******************************************************************************/ + public BackupRunRequest withSourceResolver(java.util.function.Function sourceResolver) + { + this.sourceResolver = sourceResolver; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/run/BackupRunResult.java b/src/main/java/com/kingsrook/qbits/backups/run/BackupRunResult.java new file mode 100644 index 0000000..20c6c46 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/run/BackupRunResult.java @@ -0,0 +1,407 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.run; + + +import java.util.ArrayList; +import java.util.List; + + +/******************************************************************************* + ** What a backup run did. + *******************************************************************************/ +public class BackupRunResult +{ + private Integer runId; + private String runKey; + private String runPrefix; + private String status; + private String manifestKey; + private String manifestSha256; + private String consistencyAchieved; + private int tables; + private long rows; + private long bytesStored; + private long bytesRaw; + private int filesReferenced; + private int filesCopied; + private int filesDeduplicated; + private int filesMissing; + private long fileBytesCopied; + private String errorMessage; + private List warnings = new ArrayList<>(); + + + + /******************************************************************************* + ** Whether the run produced a usable backup (has a manifest). + *******************************************************************************/ + public boolean isUsable() + { + return (manifestKey != null && ("SUCCEEDED".equals(status) || "SUCCEEDED_WITH_WARNINGS".equals(status))); + } + + + + /******************************************************************************* + ** Getter for runId + *******************************************************************************/ + public Integer getRunId() + { + return (runId); + } + + + + /******************************************************************************* + ** Setter for runId + *******************************************************************************/ + public void setRunId(Integer runId) + { + this.runId = runId; + } + + + + /******************************************************************************* + ** Getter for runKey + *******************************************************************************/ + public String getRunKey() + { + return (runKey); + } + + + + /******************************************************************************* + ** Setter for runKey + *******************************************************************************/ + public void setRunKey(String runKey) + { + this.runKey = runKey; + } + + + + /******************************************************************************* + ** Getter for runPrefix + *******************************************************************************/ + public String getRunPrefix() + { + return (runPrefix); + } + + + + /******************************************************************************* + ** Setter for runPrefix + *******************************************************************************/ + public void setRunPrefix(String runPrefix) + { + this.runPrefix = runPrefix; + } + + + + /******************************************************************************* + ** Getter for status + *******************************************************************************/ + public String getStatus() + { + return (status); + } + + + + /******************************************************************************* + ** Setter for status + *******************************************************************************/ + public void setStatus(String status) + { + this.status = status; + } + + + + /******************************************************************************* + ** Getter for manifestKey + *******************************************************************************/ + public String getManifestKey() + { + return (manifestKey); + } + + + + /******************************************************************************* + ** Setter for manifestKey + *******************************************************************************/ + public void setManifestKey(String manifestKey) + { + this.manifestKey = manifestKey; + } + + + + /******************************************************************************* + ** Getter for manifestSha256 + *******************************************************************************/ + public String getManifestSha256() + { + return (manifestSha256); + } + + + + /******************************************************************************* + ** Setter for manifestSha256 + *******************************************************************************/ + public void setManifestSha256(String manifestSha256) + { + this.manifestSha256 = manifestSha256; + } + + + + /******************************************************************************* + ** Getter for consistencyAchieved + *******************************************************************************/ + public String getConsistencyAchieved() + { + return (consistencyAchieved); + } + + + + /******************************************************************************* + ** Setter for consistencyAchieved + *******************************************************************************/ + public void setConsistencyAchieved(String consistencyAchieved) + { + this.consistencyAchieved = consistencyAchieved; + } + + + + /******************************************************************************* + ** Getter for tables + *******************************************************************************/ + public int getTables() + { + return (tables); + } + + + + /******************************************************************************* + ** Setter for tables + *******************************************************************************/ + public void setTables(int tables) + { + this.tables = tables; + } + + + + /******************************************************************************* + ** Getter for rows + *******************************************************************************/ + public long getRows() + { + return (rows); + } + + + + /******************************************************************************* + ** Setter for rows + *******************************************************************************/ + public void setRows(long rows) + { + this.rows = rows; + } + + + + /******************************************************************************* + ** Getter for bytesStored + *******************************************************************************/ + public long getBytesStored() + { + return (bytesStored); + } + + + + /******************************************************************************* + ** Setter for bytesStored + *******************************************************************************/ + public void setBytesStored(long bytesStored) + { + this.bytesStored = bytesStored; + } + + + + /******************************************************************************* + ** Getter for bytesRaw + *******************************************************************************/ + public long getBytesRaw() + { + return (bytesRaw); + } + + + + /******************************************************************************* + ** Setter for bytesRaw + *******************************************************************************/ + public void setBytesRaw(long bytesRaw) + { + this.bytesRaw = bytesRaw; + } + + + + /******************************************************************************* + ** Getter for filesReferenced + *******************************************************************************/ + public int getFilesReferenced() + { + return (filesReferenced); + } + + + + /******************************************************************************* + ** Setter for filesReferenced + *******************************************************************************/ + public void setFilesReferenced(int filesReferenced) + { + this.filesReferenced = filesReferenced; + } + + + + /******************************************************************************* + ** Getter for filesCopied + *******************************************************************************/ + public int getFilesCopied() + { + return (filesCopied); + } + + + + /******************************************************************************* + ** Setter for filesCopied + *******************************************************************************/ + public void setFilesCopied(int filesCopied) + { + this.filesCopied = filesCopied; + } + + + + /******************************************************************************* + ** Getter for filesDeduplicated + *******************************************************************************/ + public int getFilesDeduplicated() + { + return (filesDeduplicated); + } + + + + /******************************************************************************* + ** Setter for filesDeduplicated + *******************************************************************************/ + public void setFilesDeduplicated(int filesDeduplicated) + { + this.filesDeduplicated = filesDeduplicated; + } + + + + /******************************************************************************* + ** Getter for filesMissing + *******************************************************************************/ + public int getFilesMissing() + { + return (filesMissing); + } + + + + /******************************************************************************* + ** Setter for filesMissing + *******************************************************************************/ + public void setFilesMissing(int filesMissing) + { + this.filesMissing = filesMissing; + } + + + + /******************************************************************************* + ** Getter for fileBytesCopied + *******************************************************************************/ + public long getFileBytesCopied() + { + return (fileBytesCopied); + } + + + + /******************************************************************************* + ** Setter for fileBytesCopied + *******************************************************************************/ + public void setFileBytesCopied(long fileBytesCopied) + { + this.fileBytesCopied = fileBytesCopied; + } + + + + /******************************************************************************* + ** Getter for errorMessage + *******************************************************************************/ + public String getErrorMessage() + { + return (errorMessage); + } + + + + /******************************************************************************* + ** Setter for errorMessage + *******************************************************************************/ + public void setErrorMessage(String errorMessage) + { + this.errorMessage = errorMessage; + } + + + + /******************************************************************************* + ** Getter for warnings + *******************************************************************************/ + public List getWarnings() + { + return (warnings); + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/run/TableReader.java b/src/main/java/com/kingsrook/qbits/backups/run/TableReader.java new file mode 100644 index 0000000..3dc445d --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/run/TableReader.java @@ -0,0 +1,128 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.run; + + +import java.io.Serializable; +import java.util.List; +import java.util.function.Consumer; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; + + +/******************************************************************************* + ** Reads the rows of one table in primary key order, in keyset pages (WHERE pk > + ** lastKey ORDER BY pk LIMIT n), so the cost stays flat with depth and a read can + ** resume from a key. A read raw: hidden fields, passwords and heavy fields are + ** returned as stored, and possible values are not translated. + ** + ** This is an internal stand-in for the QQQ core KeysetPager and QueryInput + ** withRawRead (core PR feature/keyset-pager-raw-read); the run engine only + ** depends on this interface, so it is swapped when that ships in the next RC. + *******************************************************************************/ +public interface TableReader +{ + + /*************************************************************************** + ** Read every row of the table matching the optional filter, after startAfterKey + ** (exclusive, null for the beginning), calling the sink for each in key order. + ***************************************************************************/ + void read(QTableMetaData table, String filterJson, Serializable startAfterKey, int pageSize, QBackendTransaction transaction, Consumer sink) throws QException; + + + /*************************************************************************** + ** The default implementation over QueryAction. + ***************************************************************************/ + class Keyset implements TableReader + { + /******************************************************************************* + ** + *******************************************************************************/ + private static QQueryFilter parseFilter(String filterJson) throws QException + { + try + { + return (filterJson == null || filterJson.isBlank() ? null : JsonUtils.toObject(filterJson, QQueryFilter.class)); + } + catch(java.io.IOException e) + { + throw (new QException("Could not read a scope filter: " + e.getMessage())); + } + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public void read(QTableMetaData table, String filterJson, Serializable startAfterKey, int pageSize, QBackendTransaction transaction, Consumer sink) throws QException + { + String pk = table.getPrimaryKeyField(); + Serializable lastKey = startAfterKey; + QQueryFilter user = parseFilter(filterJson); + + while(true) + { + QQueryFilter filter = new QQueryFilter(); + if(lastKey != null) + { + filter.addCriteria(new QFilterCriteria(pk, QCriteriaOperator.GREATER_THAN, List.of(lastKey))); + } + if(user != null) + { + filter.addSubFilter(user); + } + filter.withOrderBy(new QFilterOrderBy(pk, true)); + filter.setLimit(pageSize); + + QueryInput queryInput = new QueryInput(table.getName()); + queryInput.setFilter(filter); + queryInput.setShouldOmitHiddenFields(false); + queryInput.setShouldMaskPasswords(false); + queryInput.setShouldFetchHeavyFields(true); + queryInput.setShouldTranslatePossibleValues(false); + queryInput.setShouldGenerateDisplayValues(false); + if(transaction != null) + { + queryInput.setTransaction(transaction); + } + queryInput.withPermissionBypass("backups run"); + + List page = new QueryAction().execute(queryInput).getRecords(); + for(QRecord record : page) + { + sink.accept(record); + lastKey = record.getValue(pk); + } + if(page.size() < pageSize) + { + return; + } + } + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/scope/BackupTableMetaData.java b/src/main/java/com/kingsrook/qbits/backups/scope/BackupTableMetaData.java new file mode 100644 index 0000000..60270dc --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/scope/BackupTableMetaData.java @@ -0,0 +1,147 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.scope; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QSupplementalTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** Supplemental table meta-data: the backup role of a table, and the fields + ** that reference files in object storage (when the app does not register them + ** through the object-storage reference providers). + *******************************************************************************/ +public class BackupTableMetaData extends QSupplementalTableMetaData +{ + public static final String TYPE = "backups"; + + private TableRole role; + private List fileReferenceFields = new ArrayList<>(); + private String fileBucketAlias; + + + + /******************************************************************************* + ** The BackupTableMetaData on a table, or null. + *******************************************************************************/ + public static BackupTableMetaData of(QTableMetaData table) + { + return (table == null ? null : QSupplementalTableMetaData.of(table, TYPE)); + } + + + + /******************************************************************************* + ** The BackupTableMetaData on a table, adding an empty one if needed. + *******************************************************************************/ + public static BackupTableMetaData ofOrWithNew(QTableMetaData table) + { + return (QSupplementalTableMetaData.ofOrWithNew(table, TYPE, BackupTableMetaData::new)); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public String getType() + { + return (TYPE); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + protected QSupplementalTableMetaData finishClone(QSupplementalTableMetaData abstractClone) + { + BackupTableMetaData clone = (BackupTableMetaData) abstractClone; + clone.role = this.role; + clone.fileReferenceFields = new ArrayList<>(this.fileReferenceFields); + clone.fileBucketAlias = this.fileBucketAlias; + return (clone); + } + + + + /******************************************************************************* + ** Getter for role + *******************************************************************************/ + public TableRole getRole() + { + return (role); + } + + + + /******************************************************************************* + ** Fluent setter for role + *******************************************************************************/ + public BackupTableMetaData withRole(TableRole role) + { + this.role = role; + return (this); + } + + + + /******************************************************************************* + ** Getter for fileReferenceFields (fields holding a storage key in the source bucket) + *******************************************************************************/ + public List getFileReferenceFields() + { + return (fileReferenceFields); + } + + + + /******************************************************************************* + ** Fluent setter for fileReferenceFields + *******************************************************************************/ + public BackupTableMetaData withFileReferenceFields(List fileReferenceFields) + { + this.fileReferenceFields = fileReferenceFields; + return (this); + } + + + + /******************************************************************************* + ** Getter for fileBucketAlias (the bucket the fileReferenceFields keys live in) + *******************************************************************************/ + public String getFileBucketAlias() + { + return (fileBucketAlias); + } + + + + /******************************************************************************* + ** Fluent setter for fileBucketAlias + *******************************************************************************/ + public BackupTableMetaData withFileBucketAlias(String fileBucketAlias) + { + this.fileBucketAlias = fileBucketAlias; + return (this); + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/scope/PlanSettings.java b/src/main/java/com/kingsrook/qbits/backups/scope/PlanSettings.java new file mode 100644 index 0000000..180f24d --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/scope/PlanSettings.java @@ -0,0 +1,229 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.scope; + + +import java.io.Serializable; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** The settings of a backup plan (a backups_plan record), with the defaults of + ** spec 4.1 applied for anything not set. Ad-hoc exports (no plan row) build one + ** with the static factory and the with() method. + *******************************************************************************/ +public class PlanSettings +{ + private final Map values = new LinkedHashMap<>(); + + private Integer planId; + + + + /******************************************************************************* + ** From a plan record. + *******************************************************************************/ + public static PlanSettings of(QRecord plan) + { + PlanSettings settings = new PlanSettings(); + if(plan != null) + { + settings.planId = plan.getValueInteger("id"); + settings.values.putAll(plan.getValues()); + } + return (settings); + } + + + + /******************************************************************************* + ** An ad-hoc (no plan row) settings object with all defaults. + *******************************************************************************/ + public static PlanSettings adHoc() + { + return (new PlanSettings()); + } + + + + /******************************************************************************* + ** Set one value (fluent). + *******************************************************************************/ + public PlanSettings with(String name, Serializable value) + { + values.put(name, value); + return (this); + } + + + + /******************************************************************************* + ** The plan id, or null for an ad-hoc run. + *******************************************************************************/ + public Integer getPlanId() + { + return (planId); + } + + + + /******************************************************************************* + ** A string setting. + *******************************************************************************/ + public String string(String name, String defaultValue) + { + String value = ValueUtils.getValueAsString(values.get(name)); + return (value == null || value.isEmpty() ? defaultValue : value); + } + + + + /******************************************************************************* + ** An integer setting. + *******************************************************************************/ + public Integer integer(String name, Integer defaultValue) + { + Integer value = ValueUtils.getValueAsInteger(values.get(name)); + return (value == null ? defaultValue : value); + } + + + + /******************************************************************************* + ** A long setting. + *******************************************************************************/ + public Long longValue(String name, Long defaultValue) + { + Long value = ValueUtils.getValueAsLong(values.get(name)); + return (value == null ? defaultValue : value); + } + + + + /******************************************************************************* + ** A boolean setting. + *******************************************************************************/ + public boolean bool(String name, boolean defaultValue) + { + Boolean value = ValueUtils.getValueAsBoolean(values.get(name)); + return (value == null ? defaultValue : value); + } + + + + /******************************************************************************* + ** The roles a plan includes by default (comma separated). + *******************************************************************************/ + public List includeRoles() + { + String csv = string("includeRoles", "DATA,AUDIT"); + return (java.util.Arrays.stream(csv.split(",")).map(String::trim).filter(s -> !s.isEmpty()).map(TableRole::valueOf).toList()); + } + + + + /******************************************************************************* + ** The plan key (slug used in object keys). + *******************************************************************************/ + public String planKey() + { + return (string("planKey", "adhoc")); + } + + + + /******************************************************************************* + ** The scope type. + *******************************************************************************/ + public String scopeType() + { + return (string("scopeType", "INSTANCE")); + } + + + + /******************************************************************************* + ** The format. + *******************************************************************************/ + public String format() + { + return (string("format", "JSONL")); + } + + + + /******************************************************************************* + ** The compression. + *******************************************************************************/ + public String compression() + { + return (string("compression", "GZIP")); + } + + + + /******************************************************************************* + ** The file object policy: COPY (default) or REFERENCE_ONLY. + *******************************************************************************/ + public String fileObjectPolicy() + { + return (string("fileObjectPolicy", "COPY")); + } + + + + /******************************************************************************* + ** The page size. + *******************************************************************************/ + public int pageSize() + { + return (integer("pageSize", 5000)); + } + + + + /******************************************************************************* + ** Target part size in bytes. + *******************************************************************************/ + public long targetPartBytes() + { + return (longValue("targetPartBytes", 268435456L)); + } + + + + /******************************************************************************* + ** The settings as a map (for the manifest; contains no secrets by design). + *******************************************************************************/ + public Map manifestSettings() + { + Map map = new LinkedHashMap<>(); + for(String key : List.of("scopeType", "includeRoles", "format", "compression", "encryptionMode", "sensitiveFieldPolicy", "heavyFieldPolicy", + "fileObjectPolicy", "fileMissingPolicy", "consistency", "incrementalMode", "pageSize", "targetPartBytes")) + { + if(values.get(key) != null) + { + map.put(key, values.get(key)); + } + } + return (map); + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/scope/ScopeResolver.java b/src/main/java/com/kingsrook/qbits/backups/scope/ScopeResolver.java new file mode 100644 index 0000000..3ecfef9 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/scope/ScopeResolver.java @@ -0,0 +1,448 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.scope; + + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** Turns a plan (scope type, included roles, scope rows) into the ordered list of + ** tables a run will back up (spec 4.2). + ** + ** Resolution: start from the scope type and roles, apply the INCLUDE rows, then + ** the EXCLUDE rows, add associated children, and drop SECRET and BACKUPS tables. + ** Tables are ordered parents first, using associations, joins and table-backed + ** possible values. Every table that was left out is reported with its reason. + *******************************************************************************/ +public class ScopeResolver +{ + private final QInstance qInstance; + + + + /******************************************************************************* + ** Constructor. + *******************************************************************************/ + public ScopeResolver(QInstance qInstance) + { + this.qInstance = qInstance; + } + + + + /*************************************************************************** + ** One scope row. + ** + ** @param include true for INCLUDE, false for EXCLUDE + ** @param selectorType TABLE, APP, QBIT or ROLE + ** @param selectorValue the table, app, qbit or role + ** @param filterJson optional QQueryFilter json (TABLE only) + ** @param includeAssociatedChildren also include tables reachable through associations + ** @param fieldExclusions comma separated field names to leave out + ***************************************************************************/ + public record ScopeRule(boolean include, String selectorType, String selectorValue, String filterJson, boolean includeAssociatedChildren, String fieldExclusions) + { + } + + + + /*************************************************************************** + ** One table that will be backed up. + ***************************************************************************/ + public record ResolvedTable(String name, TableRole role, String filterJson, List excludedFields) + { + } + + + + /*************************************************************************** + ** A table that was left out, and why. + ***************************************************************************/ + public record ExcludedTable(String name, String reason) + { + } + + + + /*************************************************************************** + ** The outcome of resolving a scope. + ** + ** @param tables the tables, parents first + ** @param excluded tables left out, by reason + ** @param cyclic tables that could not be ordered (reference cycle), in name order at the end + ***************************************************************************/ + public record ResolvedScope(List tables, List excluded, List cyclic) + { + /******************************************************************************* + ** The table names, in restore order. + *******************************************************************************/ + public List names() + { + return (tables.stream().map(ResolvedTable::name).toList()); + } + } + + + + /******************************************************************************* + ** Resolve a scope. + *******************************************************************************/ + public ResolvedScope resolve(PlanSettings settings, List rules) + { + List roles = settings.includeRoles(); + + Set selected = new LinkedHashSet<>(); + Set explicit = new LinkedHashSet<>(); + Map filters = new LinkedHashMap<>(); + Map fieldsOut = new LinkedHashMap<>(); + Set expandChildrenFor = new LinkedHashSet<>(); + + if("INSTANCE".equals(settings.scopeType())) + { + for(QTableMetaData table : sortedTables()) + { + if(roles.contains(TableRoles.roleOf(qInstance, table))) + { + selected.add(table.getName()); + } + } + } + + for(ScopeRule rule : rules) + { + if(!rule.include()) + { + continue; + } + for(String name : tablesFor(rule, roles)) + { + selected.add(name); + if("TABLE".equals(rule.selectorType())) + { + explicit.add(name); + if(rule.filterJson() != null && !rule.filterJson().isBlank()) + { + filters.put(name, rule.filterJson()); + } + if(rule.fieldExclusions() != null && !rule.fieldExclusions().isBlank()) + { + fieldsOut.put(name, rule.fieldExclusions()); + } + } + if(rule.includeAssociatedChildren()) + { + expandChildrenFor.add(name); + } + } + } + + for(ScopeRule rule : rules) + { + if(rule.include()) + { + continue; + } + for(String name : tablesFor(rule, List.of(TableRole.values()))) + { + selected.remove(name); + explicit.remove(name); + } + } + + for(String name : new ArrayList<>(expandChildrenFor)) + { + addAssociatedChildren(name, selected); + } + + List excluded = new ArrayList<>(); + List kept = new ArrayList<>(); + for(String name : selected) + { + QTableMetaData table = qInstance.getTable(name); + if(table == null) + { + excluded.add(new ExcludedTable(name, "NOT_FOUND")); + continue; + } + TableRole role = TableRoles.roleOf(qInstance, table); + if(role == TableRole.SECRET) + { + excluded.add(new ExcludedTable(name, "SECRET")); + } + else if(role == TableRole.BACKUPS) + { + excluded.add(new ExcludedTable(name, "BACKUPS")); + } + else if((role == TableRole.EPHEMERAL || role == TableRole.DERIVED || role == TableRole.EXTERNAL) && !explicit.contains(name) && !roles.contains(role)) + { + excluded.add(new ExcludedTable(name, role.name())); + } + else + { + kept.add(name); + } + } + + List cyclic = new ArrayList<>(); + List ordered = order(kept, cyclic); + + List tables = new ArrayList<>(); + for(String name : ordered) + { + List out = new ArrayList<>(); + if(fieldsOut.get(name) != null) + { + for(String field : fieldsOut.get(name).split(",")) + { + if(!field.isBlank()) + { + out.add(field.trim()); + } + } + } + tables.add(new ResolvedTable(name, TableRoles.roleOf(qInstance, qInstance.getTable(name)), filters.get(name), out)); + } + return (new ResolvedScope(tables, excluded, cyclic)); + } + + + + /******************************************************************************* + ** The tables a rule selects. + *******************************************************************************/ + private Collection tablesFor(ScopeRule rule, List rolesForExpansion) + { + Set names = new LinkedHashSet<>(); + switch(rule.selectorType()) + { + case "TABLE" -> names.add(rule.selectorValue()); + case "ROLE" -> + { + TableRole role = TableRole.valueOf(rule.selectorValue()); + for(QTableMetaData table : sortedTables()) + { + if(TableRoles.roleOf(qInstance, table) == role) + { + names.add(table.getName()); + } + } + } + case "QBIT" -> + { + for(QTableMetaData table : sortedTables()) + { + if(rule.selectorValue().equals(table.getSourceQBitName()) && rolesForExpansion.contains(TableRoles.roleOf(qInstance, table))) + { + names.add(table.getName()); + } + } + } + case "APP" -> + { + QAppMetaData app = qInstance.getApp(rule.selectorValue()); + if(app != null) + { + Set inApp = new LinkedHashSet<>(); + collectAppTables(app, inApp, new LinkedHashSet<>()); + for(String name : inApp) + { + QTableMetaData table = qInstance.getTable(name); + if(table != null && rolesForExpansion.contains(TableRoles.roleOf(qInstance, table))) + { + names.add(name); + } + } + } + } + default -> + { + } + } + return (names); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void collectAppTables(QAppMetaData app, Set tables, Set seenApps) + { + if(!seenApps.add(app.getName())) + { + return; + } + for(QAppChildMetaData child : app.getChildren() == null ? List.of() : app.getChildren()) + { + if(child instanceof QAppMetaData childApp) + { + collectAppTables(childApp, tables, seenApps); + } + else if(child instanceof QTableMetaData table) + { + tables.add(table.getName()); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void addAssociatedChildren(String name, Set selected) + { + QTableMetaData table = qInstance.getTable(name); + if(table == null || table.getAssociations() == null) + { + return; + } + for(Association association : table.getAssociations()) + { + if(selected.add(association.getAssociatedTableName())) + { + addAssociatedChildren(association.getAssociatedTableName(), selected); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private List sortedTables() + { + return (new ArrayList<>(new TreeMap<>(qInstance.getTables()).values())); + } + + + + /******************************************************************************* + ** The tables that must be restored before the given table (its parents). + *******************************************************************************/ + Set parentsOf(String name, Set universe) + { + Set parents = new LinkedHashSet<>(); + QTableMetaData table = qInstance.getTable(name); + + for(QTableMetaData other : qInstance.getTables().values()) + { + if(other.getAssociations() != null) + { + for(Association association : other.getAssociations()) + { + if(name.equals(association.getAssociatedTableName())) + { + parents.add(other.getName()); + } + } + } + } + + for(QJoinMetaData join : qInstance.getJoins().values()) + { + if(join.getType() == JoinType.MANY_TO_ONE && name.equals(join.getLeftTable())) + { + parents.add(join.getRightTable()); + } + else if(join.getType() == JoinType.ONE_TO_MANY && name.equals(join.getRightTable())) + { + parents.add(join.getLeftTable()); + } + } + + for(QFieldMetaData field : table.getFields().values()) + { + if(field.getPossibleValueSourceName() != null) + { + QPossibleValueSource pvs = qInstance.getPossibleValueSource(field.getPossibleValueSourceName()); + if(pvs != null && pvs.getType() == QPossibleValueSourceType.TABLE && pvs.getTableName() != null) + { + parents.add(pvs.getTableName()); + } + } + } + + parents.remove(name); + parents.retainAll(universe); + return (parents); + } + + + + /******************************************************************************* + ** Topological order, parents first, stable by name. Tables in a reference cycle + ** are appended in name order and reported through the cyclic list. + *******************************************************************************/ + List order(List names, List cyclic) + { + Set universe = new LinkedHashSet<>(names); + Map> remaining = new TreeMap<>(); + for(String name : names) + { + remaining.put(name, parentsOf(name, universe)); + } + + List ordered = new ArrayList<>(); + while(!remaining.isEmpty()) + { + List ready = new ArrayList<>(); + for(Map.Entry> entry : remaining.entrySet()) + { + if(entry.getValue().isEmpty()) + { + ready.add(entry.getKey()); + } + } + if(ready.isEmpty()) + { + cyclic.addAll(remaining.keySet()); + ordered.addAll(remaining.keySet()); + break; + } + for(String name : ready) + { + ordered.add(name); + remaining.remove(name); + } + for(Set parents : remaining.values()) + { + parents.removeAll(ready); + } + } + return (ordered); + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/scope/TableRole.java b/src/main/java/com/kingsrook/qbits/backups/scope/TableRole.java new file mode 100644 index 0000000..7002043 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/scope/TableRole.java @@ -0,0 +1,40 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.scope; + + +/******************************************************************************* + ** What a table is for, as far as backups are concerned (spec 3.2). Plans select + ** tables by role. + *******************************************************************************/ +public enum TableRole +{ + /** Business data (included by default). */ + DATA, + /** Audit tables (included by default, configurable). */ + AUDIT, + /** Rebuildable from other data: search indexes, rollups, caches. Excluded by default. */ + DERIVED, + /** Sessions, idempotency keys, scheduler internals. Excluded by default. */ + EPHEMERAL, + /** Tokens, salts, credentials. NEVER backed up, even when selected explicitly. */ + SECRET, + /** The tables of this qbit: the catalog is rebuilt from manifests. */ + BACKUPS, + /** Tables backed by another system (API). Excluded by default; selectable. */ + EXTERNAL +} diff --git a/src/main/java/com/kingsrook/qbits/backups/scope/TableRoles.java b/src/main/java/com/kingsrook/qbits/backups/scope/TableRoles.java new file mode 100644 index 0000000..602ab46 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/scope/TableRoles.java @@ -0,0 +1,130 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.scope; + + +import java.util.Set; +import com.kingsrook.qbits.backups.BackupsQBitConfig; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** Works out the backup role of a table: the declared role when there is one, + ** else inferred by the rules of spec 3.2. + *******************************************************************************/ +public final class TableRoles +{ + private static final Set AUDIT_TABLES = Set.of("audit", "auditTable", "auditUser", "auditDetail"); + + private static final String[] SECRET_NAME_HINTS = { "refreshToken", "RefreshToken", "oauthToken", "OAuthToken", "apiKey" }; + + private static final String[] EPHEMERAL_NAME_PREFIXES = { "qrtz_" }; + private static final String[] EPHEMERAL_NAME_PARTS = { "Session", "Idempotency", "RedirectState", "AsyncJob" }; + + + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private TableRoles() + { + } + + + + /******************************************************************************* + ** The role of a table. + *******************************************************************************/ + public static TableRole roleOf(QInstance qInstance, QTableMetaData table) + { + BackupTableMetaData declared = BackupTableMetaData.of(table); + if(declared != null && declared.getRole() != null) + { + return (declared.getRole()); + } + + String name = table.getName(); + if(name.startsWith(BackupsQBitConfig.TABLE_NAME_PREFIX)) + { + return (TableRole.BACKUPS); + } + for(String hint : SECRET_NAME_HINTS) + { + if(name.contains(hint)) + { + return (TableRole.SECRET); + } + } + if(isHiddenWithSecretFields(table)) + { + return (TableRole.SECRET); + } + if(AUDIT_TABLES.contains(name)) + { + return (TableRole.AUDIT); + } + if(table.getCacheOf() != null) + { + return (TableRole.DERIVED); + } + for(String prefix : EPHEMERAL_NAME_PREFIXES) + { + if(name.startsWith(prefix)) + { + return (TableRole.EPHEMERAL); + } + } + for(String part : EPHEMERAL_NAME_PARTS) + { + if(name.contains(part) || name.toLowerCase().contains(part.toLowerCase())) + { + return (TableRole.EPHEMERAL); + } + } + if(qInstance != null && table.getBackendName() != null && qInstance.getBackend(table.getBackendName()) != null + && "api".equalsIgnoreCase(qInstance.getBackend(table.getBackendName()).getBackendType())) + { + return (TableRole.EXTERNAL); + } + return (TableRole.DATA); + } + + + + /******************************************************************************* + ** A table that is hidden and has a password or sensitive field (other than the + ** keys and dates) is treated as a table of secrets. + *******************************************************************************/ + static boolean isHiddenWithSecretFields(QTableMetaData table) + { + if(!table.getIsHidden()) + { + return (false); + } + for(QFieldMetaData field : table.getFields().values()) + { + if(field.getType() == QFieldType.PASSWORD || field.getIsSensitive()) + { + return (true); + } + } + return (false); + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/storage/BackupStorage.java b/src/main/java/com/kingsrook/qbits/backups/storage/BackupStorage.java new file mode 100644 index 0000000..9b5ed18 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/storage/BackupStorage.java @@ -0,0 +1,107 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.storage; + + +import java.io.InputStream; +import java.io.OutputStream; +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.Set; + + +/******************************************************************************* + ** The narrow storage API that the Backups qbit needs. It mirrors the + ** qbit-object-storage bucket API (put, get, head, list, copy, delete, presign), + ** so the production implementation is a thin adapter over a StorageBucket, and + ** tests use InMemoryBackupStorage. Keys are relative to the bucket. + ** + ** All methods throw BackupStorageException (a RuntimeException) on failure. + *******************************************************************************/ +public interface BackupStorage +{ + + /*************************************************************************** + ** Open a stream to write an object of unknown length. The object becomes + ** visible when the stream is closed successfully. + ***************************************************************************/ + OutputStream openWrite(String key, WriteOptions options); + + + /*************************************************************************** + ** Open a stream over an object. + ***************************************************************************/ + InputStream openRead(String key); + + + /*************************************************************************** + ** Existence and size. + ***************************************************************************/ + Optional head(String key); + + + /*************************************************************************** + ** All objects with the given key prefix (recursive, in key order). + ***************************************************************************/ + List list(String prefix); + + + /*************************************************************************** + ** Server-side copy within the same bucket (or connection). + ***************************************************************************/ + void copy(String sourceKey, String destinationKey); + + + /*************************************************************************** + ** Permanently delete an object, every version. Object-locked objects are + ** not deleted (DeleteResult.LOCKED); a missing object is NOT_FOUND. + ***************************************************************************/ + DeleteResult deletePermanent(String key); + + + /*************************************************************************** + ** A time-limited download link. + ***************************************************************************/ + String presignGet(String key, Duration ttl, String downloadName); + + + /*************************************************************************** + ** What the bucket can do (object lock, server-side copy, ...). + ***************************************************************************/ + Set capabilities(); + + + + /*************************************************************************** + ** Result of a permanent delete. + ***************************************************************************/ + enum DeleteResult + { + DELETED, NOT_FOUND, LOCKED + } + + + + /*************************************************************************** + ** Capabilities, as probed by the storage qbit. + ***************************************************************************/ + enum StorageCapability + { + OBJECT_LOCK, VERSIONING, SERVER_SIDE_COPY, MULTIPART + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/storage/BackupStorageException.java b/src/main/java/com/kingsrook/qbits/backups/storage/BackupStorageException.java new file mode 100644 index 0000000..acf9ffa --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/storage/BackupStorageException.java @@ -0,0 +1,69 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.storage; + + +/******************************************************************************* + ** A storage failure, with a reason that mirrors the StorageException reasons + ** of the object-storage qbit. + *******************************************************************************/ +public class BackupStorageException extends RuntimeException +{ + private final Reason reason; + + + + /******************************************************************************* + ** Constructor. + *******************************************************************************/ + public BackupStorageException(Reason reason, String message) + { + super(message); + this.reason = reason; + } + + + + /******************************************************************************* + ** Constructor with a cause. + *******************************************************************************/ + public BackupStorageException(Reason reason, String message, Throwable cause) + { + super(message, cause); + this.reason = reason; + } + + + + /******************************************************************************* + ** Getter for reason + *******************************************************************************/ + public Reason getReason() + { + return (reason); + } + + + + /*************************************************************************** + ** Failure reasons. + ***************************************************************************/ + public enum Reason + { + NOT_FOUND, ACCESS_DENIED, UNSUPPORTED, PRECONDITION_FAILED, THROTTLED, PROVIDER_ERROR + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/storage/InMemoryBackupStorage.java b/src/main/java/com/kingsrook/qbits/backups/storage/InMemoryBackupStorage.java new file mode 100644 index 0000000..e2fda97 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/storage/InMemoryBackupStorage.java @@ -0,0 +1,300 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.storage; + + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeMap; +import java.util.function.Predicate; + + +/******************************************************************************* + ** A BackupStorage held in memory. It is the test implementation and a + ** stand-in for local development. It supports object lock, and has hooks for + ** failure injection (a predicate over keys that makes a write fail). + *******************************************************************************/ +public class InMemoryBackupStorage implements BackupStorage +{ + private final TreeMap objects = new TreeMap<>(); + + private Clock clock = Clock.systemUTC(); + private Predicate failWritesFor; + private Set capabilities = EnumSet.of(StorageCapability.OBJECT_LOCK, StorageCapability.SERVER_SIDE_COPY, StorageCapability.MULTIPART); + + private record Entry(byte[] bytes, Map metadata, Instant lastModified, Instant lockedUntil) + { + } + + + + /******************************************************************************* + ** Use a fixed clock (for object lock and last modified in tests). + *******************************************************************************/ + public InMemoryBackupStorage withClock(Clock clock) + { + this.clock = clock; + return (this); + } + + + + /******************************************************************************* + ** Make writes of matching keys fail with a provider error (failure injection). + *******************************************************************************/ + public InMemoryBackupStorage withFailWritesFor(Predicate failWritesFor) + { + this.failWritesFor = failWritesFor; + return (this); + } + + + + /******************************************************************************* + ** Set the capabilities this storage reports. + *******************************************************************************/ + public InMemoryBackupStorage withCapabilities(Set capabilities) + { + this.capabilities = capabilities; + return (this); + } + + + + /******************************************************************************* + ** Test helper: remove everything. + *******************************************************************************/ + public synchronized void clear() + { + objects.clear(); + failWritesFor = null; + } + + + + /******************************************************************************* + ** Test helper: put bytes directly. + *******************************************************************************/ + public synchronized void putBytes(String key, byte[] bytes) + { + objects.put(key, new Entry(bytes, new LinkedHashMap<>(), clock.instant(), null)); + } + + + + /******************************************************************************* + ** Test helper: read bytes directly. + *******************************************************************************/ + public synchronized byte[] getBytes(String key) + { + Entry entry = objects.get(key); + return (entry == null ? null : entry.bytes()); + } + + + + /******************************************************************************* + ** Test helper: remove an object ignoring any lock (simulating an out-of-band delete). + *******************************************************************************/ + public synchronized void removeIgnoringLock(String key) + { + objects.remove(key); + } + + + + /******************************************************************************* + ** Test helper: replace an object bytes (simulating corruption). + *******************************************************************************/ + public synchronized void corrupt(String key, byte[] bytes) + { + Entry old = objects.get(key); + objects.put(key, new Entry(bytes, old == null ? new LinkedHashMap<>() : old.metadata(), clock.instant(), old == null ? null : old.lockedUntil())); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public OutputStream openWrite(String key, WriteOptions options) + { + if(failWritesFor != null && failWritesFor.test(key)) + { + throw (new BackupStorageException(BackupStorageException.Reason.PROVIDER_ERROR, "injected write failure for " + key)); + } + if(options.getObjectLockMode() != null && !capabilities.contains(StorageCapability.OBJECT_LOCK)) + { + throw (new BackupStorageException(BackupStorageException.Reason.UNSUPPORTED, "object lock is not supported")); + } + + return (new ByteArrayOutputStream() + { + private boolean closed = false; + + + + /******************************************************************************* + ** commit on close + *******************************************************************************/ + @Override + public void close() throws IOException + { + super.close(); + if(!closed) + { + closed = true; + synchronized(InMemoryBackupStorage.this) + { + objects.put(key, new Entry(toByteArray(), new LinkedHashMap<>(options.getMetadata()), clock.instant(), options.getRetainUntil())); + } + } + } + }); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public synchronized InputStream openRead(String key) + { + Entry entry = objects.get(key); + if(entry == null) + { + throw (new BackupStorageException(BackupStorageException.Reason.NOT_FOUND, "no such object: " + key)); + } + return (new ByteArrayInputStream(entry.bytes())); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public synchronized Optional head(String key) + { + Entry entry = objects.get(key); + return (entry == null ? Optional.empty() : Optional.of(info(key, entry))); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public synchronized List list(String prefix) + { + List rs = new ArrayList<>(); + for(Map.Entry e : objects.tailMap(prefix == null ? "" : prefix).entrySet()) + { + if(prefix != null && !e.getKey().startsWith(prefix)) + { + break; + } + rs.add(info(e.getKey(), e.getValue())); + } + return (rs); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public synchronized void copy(String sourceKey, String destinationKey) + { + Entry entry = objects.get(sourceKey); + if(entry == null) + { + throw (new BackupStorageException(BackupStorageException.Reason.NOT_FOUND, "no such object: " + sourceKey)); + } + objects.put(destinationKey, new Entry(entry.bytes(), entry.metadata(), clock.instant(), null)); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public synchronized DeleteResult deletePermanent(String key) + { + Entry entry = objects.get(key); + if(entry == null) + { + return (DeleteResult.NOT_FOUND); + } + if(entry.lockedUntil() != null && entry.lockedUntil().isAfter(clock.instant())) + { + return (DeleteResult.LOCKED); + } + objects.remove(key); + return (DeleteResult.DELETED); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public String presignGet(String key, Duration ttl, String downloadName) + { + return ("memory://" + key + "?ttl=" + ttl.toSeconds()); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public Set capabilities() + { + return (capabilities); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private StoredObjectInfo info(String key, Entry entry) + { + return (new StoredObjectInfo(key, entry.bytes().length, entry.lastModified(), entry.metadata(), entry.lockedUntil())); + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/storage/StoredObjectInfo.java b/src/main/java/com/kingsrook/qbits/backups/storage/StoredObjectInfo.java new file mode 100644 index 0000000..7d332d7 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/storage/StoredObjectInfo.java @@ -0,0 +1,35 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.storage; + + +import java.time.Instant; +import java.util.Map; + + +/******************************************************************************* + ** What the bucket says about one object. + ** + ** @param key key relative to the bucket + ** @param size size in bytes + ** @param lastModified last modified instant (may be null) + ** @param metadata user metadata + ** @param lockedUntil object-lock retain-until, or null + *******************************************************************************/ +public record StoredObjectInfo(String key, long size, Instant lastModified, Map metadata, Instant lockedUntil) +{ +} diff --git a/src/main/java/com/kingsrook/qbits/backups/storage/WriteOptions.java b/src/main/java/com/kingsrook/qbits/backups/storage/WriteOptions.java new file mode 100644 index 0000000..7e322f4 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/storage/WriteOptions.java @@ -0,0 +1,118 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.storage; + + +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.Map; + + +/******************************************************************************* + ** Options for writing one object. + *******************************************************************************/ +public class WriteOptions +{ + private String contentType = "application/octet-stream"; + private Map metadata = new LinkedHashMap<>(); + private String objectLockMode; + private Instant retainUntil; + + + + /******************************************************************************* + ** Start with defaults. + *******************************************************************************/ + public static WriteOptions create() + { + return (new WriteOptions()); + } + + + + /******************************************************************************* + ** Getter for contentType + *******************************************************************************/ + public String getContentType() + { + return (contentType); + } + + + + /******************************************************************************* + ** Fluent setter for contentType + *******************************************************************************/ + public WriteOptions withContentType(String contentType) + { + this.contentType = contentType; + return (this); + } + + + + /******************************************************************************* + ** Getter for metadata (user metadata, which describes the object even without the catalog) + *******************************************************************************/ + public Map getMetadata() + { + return (metadata); + } + + + + /******************************************************************************* + ** Add one metadata entry. + *******************************************************************************/ + public WriteOptions withMetadata(String key, String value) + { + this.metadata.put(key, value); + return (this); + } + + + + /******************************************************************************* + ** Getter for objectLockMode (GOVERNANCE or COMPLIANCE), null for none + *******************************************************************************/ + public String getObjectLockMode() + { + return (objectLockMode); + } + + + + /******************************************************************************* + ** Getter for retainUntil + *******************************************************************************/ + public Instant getRetainUntil() + { + return (retainUntil); + } + + + + /******************************************************************************* + ** Request an object lock on the written object. + *******************************************************************************/ + public WriteOptions withObjectLock(String mode, Instant retainUntil) + { + this.objectLockMode = mode; + this.retainUntil = retainUntil; + return (this); + } +} diff --git a/src/test/java/com/kingsrook/qbits/backups/BackupsTestUtils.java b/src/test/java/com/kingsrook/qbits/backups/BackupsTestUtils.java new file mode 100644 index 0000000..88ed3ff --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/backups/BackupsTestUtils.java @@ -0,0 +1,168 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups; + + +import java.io.Serializable; +import java.math.BigDecimal; +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.backups.scope.BackupTableMetaData; +import com.kingsrook.qbits.backups.storage.BackupStorage; +import com.kingsrook.qbits.backups.storage.InMemoryBackupStorage; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; + + +/******************************************************************************* + ** Test fixtures: an instance on the memory backend with the Backups qbit and a + ** small blog app (authors, posts with a media file key, an ephemeral table). + *******************************************************************************/ +public final class BackupsTestUtils +{ + public static final String BACKEND = "memory"; + + public static final InMemoryBackupStorage BACKUPS = new InMemoryBackupStorage(); + public static final InMemoryBackupStorage MEDIA = new InMemoryBackupStorage(); + + + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private BackupsTestUtils() + { + } + + + + /******************************************************************************* + ** Build the instance, install the qbit, and initialize the context. + *******************************************************************************/ + public static QInstance init() throws Exception + { + BackupsRuntime.reset(); + MemoryRecordStore.getInstance().reset(); + BACKUPS.clear(); + MEDIA.clear(); + + QInstance qInstance = new QInstance(); + qInstance.withInstanceDefaultAuthentication(new com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData().withType(com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType.FULLY_ANONYMOUS).withName("anon")); + qInstance.addBackend(new QBackendMetaData().withName(BACKEND).withBackendType(MemoryBackendModule.class)); + + qInstance.addTable(new QTableMetaData().withName("author").withBackendName(BACKEND).withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("name", QFieldType.STRING)) + .withField(new QFieldMetaData("bio", QFieldType.TEXT))); + + qInstance.addTable(new QTableMetaData().withName("blogPost").withBackendName(BACKEND).withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("authorId", QFieldType.INTEGER).withPossibleValueSourceName("author")) + .withField(new QFieldMetaData("title", QFieldType.STRING)) + .withField(new QFieldMetaData("body", QFieldType.TEXT)) + .withField(new QFieldMetaData("price", QFieldType.DECIMAL)) + .withField(new QFieldMetaData("published", QFieldType.BOOLEAN)) + .withField(new QFieldMetaData("publishedOn", QFieldType.DATE)) + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME)) + .withField(new QFieldMetaData("mediaKey", QFieldType.STRING)) + .withField(new QFieldMetaData("secretNote", QFieldType.STRING).withIsSensitive(true)) + .withSupplementalMetaData(new BackupTableMetaData().withFileReferenceFields(List.of("mediaKey")).withFileBucketAlias("media"))); + qInstance.addPossibleValueSource(com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource.newForTable("author")); + + qInstance.addTable(new QTableMetaData().withName("userSession").withBackendName(BACKEND).withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("token", QFieldType.STRING))); + + Map buckets = Map.of("backups", BACKUPS, "media", MEDIA); + BackupsQBitConfig config = new BackupsQBitConfig().withBackendName(BACKEND).withInstanceName("testapp").withStorageResolver(buckets::get).withOpenWhenAnonymous(true); + new BackupsQBitProducer().withQBitConfig(config).withNamespace("backups").produce(qInstance); + + new QInstanceValidator().validate(qInstance); + QContext.init(qInstance, new QSession()); + return (qInstance); + } + + + + /******************************************************************************* + ** Insert rows into a table. + *******************************************************************************/ + public static void insert(String table, List> rows) throws QException + { + InsertInput input = new InsertInput(table); + input.setRecords(rows.stream().map(r -> + { + QRecord record = new QRecord(); + record.setValues(new LinkedHashMap<>(r)); + return (record); + }).toList()); + new InsertAction().execute(input); + } + + + + /******************************************************************************* + ** The standard blog data: 3 authors, and the posts, some with media. + *******************************************************************************/ + public static void loadBlog(int posts) throws Exception + { + insert("author", List.of( + Map.of("id", 1, "name", "Ada", "bio", "Line one\nLine two, with \"quotes\" and unicode é中"), + Map.of("id", 2, "name", "Grace", "bio", ""), + Map.of("id", 3, "name", "Linus"))); + List> rows = new java.util.ArrayList<>(); + for(int i = 1; i <= posts; i++) + { + Map row = new LinkedHashMap<>(); + row.put("id", i); + row.put("authorId", 1 + (i % 3)); + row.put("title", "Post " + i); + row.put("body", "Body of " + i + ", with a comma"); + row.put("price", new BigDecimal("12.3400").add(BigDecimal.valueOf(i))); + row.put("published", i % 2 == 0); + row.put("publishedOn", java.time.LocalDate.of(2026, 1, 1).plusDays(i)); + row.put("createDate", Instant.parse("2026-10-12T02:00:00.123456Z").plusSeconds(i)); + row.put("secretNote", "do-not-leak-" + i); + if(i % 2 == 1) + { + String key = "media/photo-" + (i % 3) + ".jpg"; + row.put("mediaKey", key); + if(MEDIA.getBytes(key) == null) + { + MEDIA.putBytes(key, ("photo bytes " + (i % 3)).getBytes()); + } + } + rows.add(row); + } + insert("blogPost", rows); + insert("userSession", List.of(Map.of("id", 1, "token", "tok"))); + } +} diff --git a/src/test/java/com/kingsrook/qbits/backups/run/BackupEngineTest.java b/src/test/java/com/kingsrook/qbits/backups/run/BackupEngineTest.java new file mode 100644 index 0000000..8b3ca0b --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/backups/run/BackupEngineTest.java @@ -0,0 +1,282 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.run; + + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.zip.GZIPInputStream; +import com.kingsrook.qbits.backups.BackupsTestUtils; +import com.kingsrook.qbits.backups.manifest.BackupManifest; +import com.kingsrook.qbits.backups.model.BackupsTables; +import com.kingsrook.qbits.backups.scope.PlanSettings; +import com.kingsrook.qbits.backups.scope.ScopeResolver; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** End to end tests of a backup run on the memory backend: layout, manifest last, + ** canonical rows, policies, file copy and deduplication, failure handling. + *******************************************************************************/ +class BackupEngineTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void setUp() throws Exception + { + BackupsTestUtils.init(); + BackupsTestUtils.loadBlog(7); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QRecord plan(Map overrides) throws Exception + { + Map values = new java.util.LinkedHashMap<>(Map.of("name", "Nightly", "planKey", "nightly", "isActive", true, + "scopeType", "INSTANCE", "storageBucketAlias", "backups", "slaHours", 26)); + values.putAll(overrides); + Integer id = new BackupCatalog().insert(BackupsTables.PLAN, values); + return (new BackupCatalog().get(BackupsTables.PLAN, id).orElseThrow()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private BackupRunResult run(QRecord plan) throws Exception + { + return (new BackupEngine().execute(BackupRunFactory.forPlan(plan, "MANUAL"))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + static String gunzip(byte[] bytes) throws IOException + { + return (new String(new GZIPInputStream(new ByteArrayInputStream(bytes)).readAllBytes(), StandardCharsets.UTF_8)); + } + + + + /******************************************************************************* + ** A nightly run writes parts, checksums and the manifest LAST, and the manifest + ** matches the rows and the stored bytes. + *******************************************************************************/ + @Test + void testRunWritesManifestAndParts() throws Exception + { + BackupRunResult result = run(plan(Map.of())); + assertThat(result.getStatus()).isEqualTo("SUCCEEDED_WITH_WARNINGS"); + assertThat(result.getWarnings()).anyMatch(w -> w.contains("BEST_EFFORT")); + assertThat(result.isUsable()).isTrue(); + + BackupManifest manifest = BackupManifest.read(BackupsTestUtils.BACKUPS.getBytes(result.getManifestKey())); + assertThat(manifest.runKey()).isEqualTo(result.getRunKey()); + assertThat(manifest.restoreOrder()).containsSubsequence("author", "blogPost"); + assertThat(manifest.table("userSession")).as("ephemeral tables are not backed up").isNull(); + assertThat(manifest.table("backups_plan")).as("the qbit own tables are not backed up").isNull(); + + BackupManifest.TableEntry posts = manifest.table("blogPost"); + assertThat(posts.rowCount()).isEqualTo(7); + assertThat(posts.excludedFields()).extracting(BackupManifest.ExcludedField::name).containsExactly("secretNote"); + assertThat(posts.fingerprint()).startsWith("sha256:"); + + BackupManifest.Part part = posts.parts().get(0); + byte[] stored = BackupsTestUtils.BACKUPS.getBytes(result.getRunPrefix() + part.key()); + assertThat(com.kingsrook.qbits.backups.codec.HashingOutputStream.sha256Hex(stored)).isEqualTo(part.sha256Stored()); + String text = gunzip(stored); + assertThat(text.split("\n")).hasSize(7); + assertThat(text).doesNotContain("do-not-leak").doesNotContain("secretNote"); + assertThat(text.split("\n")[0]).startsWith("{\"authorId\":2,\"body\":\"Body of 1, with a comma\",\"createDate\":\"20").contains(",\"id\":1,\"mediaKey\":\"media/photo-1.jpg\",\"price\":\"13.3400\""); + assertThat(text).contains("\"price\":\"13.3400\"").contains("\"publishedOn\":\"2026-01-02\""); + assertThat(com.kingsrook.qbits.backups.codec.HashingOutputStream.sha256Hex(text.getBytes(StandardCharsets.UTF_8))).isEqualTo(part.sha256Raw()); + + String checksums = new String(BackupsTestUtils.BACKUPS.getBytes(result.getRunPrefix() + "checksums.sha256"), StandardCharsets.UTF_8); + assertThat(checksums).contains(part.sha256Stored() + " " + part.key()); + + QRecord run = new BackupCatalog().get(BackupsTables.RUN, result.getRunId()).orElseThrow(); + assertThat(run.getValueString("status")).isEqualTo("SUCCEEDED_WITH_WARNINGS"); + assertThat(run.getValueString("manifestKey")).isEqualTo(result.getManifestKey()); + assertThat(run.getValueLong("rowsTotal")).isEqualTo(10L); + assertThat(new BackupCatalog().where(BackupsTables.ARTIFACT, "backupRunId", result.getRunId())).hasSizeGreaterThanOrEqualTo(4); + + QRecord plan = new BackupCatalog().where(BackupsTables.PLAN, "planKey", "nightly").get(0); + assertThat(plan.getValueInteger("lastSuccessRunId")).isEqualTo(result.getRunId()); + assertThat(plan.getValueString("slaStatus")).isEqualTo("OK"); + } + + + + /******************************************************************************* + ** Referenced files are copied into the content-addressed store; the second run + ** copies nothing new (deduplicated), and rows referencing the same bytes share one object. + *******************************************************************************/ + @Test + void testFilesAreCopiedThenDeduplicated() throws Exception + { + QRecord plan = plan(Map.of()); + BackupRunResult first = run(plan); + assertThat(first.getFilesReferenced()).isEqualTo(4); + assertThat(first.getFilesCopied()).isEqualTo(3); + assertThat(first.getFilesDeduplicated()).isEqualTo(1); + assertThat(first.getFilesMissing()).isZero(); + + List fileObjects = new BackupCatalog().query(BackupsTables.FILE_OBJECT, new com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter()); + assertThat(fileObjects).hasSize(3); + for(QRecord fileObject : fileObjects) + { + byte[] stored = BackupsTestUtils.BACKUPS.getBytes(fileObject.getValueString("objectKey")); + assertThat(com.kingsrook.qbits.backups.codec.HashingOutputStream.sha256Hex(stored)).isEqualTo(fileObject.getValueString("sha256")); + assertThat(fileObject.getValueString("objectKey")).contains("/_files/sha256/" + fileObject.getValueString("sha256").substring(0, 2) + "/"); + } + + BackupRunResult second = run(plan); + assertThat(second.getFilesCopied()).isZero(); + assertThat(second.getFilesDeduplicated()).isEqualTo(4); + assertThat(new BackupCatalog().query(BackupsTables.FILE_OBJECT, new com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter())).hasSize(3); + + String index = gunzip(BackupsTestUtils.BACKUPS.getBytes(first.getRunPrefix() + "files/index-00001.jsonl.gz")); + assertThat(index.split("\n")).hasSize(4); + assertThat(index).contains("\"status\":\"COPIED\"").contains("\"status\":\"DEDUPLICATED\""); + } + + + + /******************************************************************************* + ** REFERENCE_ONLY records the key and size without copying; a missing source is + ** listed and (under WARN) the run ends with warnings. + *******************************************************************************/ + @Test + void testReferenceOnlyAndMissingFiles() throws Exception + { + BackupRunResult refOnly = run(plan(Map.of("fileObjectPolicy", "REFERENCE_ONLY"))); + assertThat(refOnly.getFilesCopied()).isZero(); + assertThat(new BackupCatalog().query(BackupsTables.FILE_OBJECT, new com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter())).isEmpty(); + assertThat(gunzip(BackupsTestUtils.BACKUPS.getBytes(refOnly.getRunPrefix() + "files/index-00001.jsonl.gz"))).contains("REFERENCE_ONLY"); + + BackupsTestUtils.MEDIA.removeIgnoringLock("media/photo-1.jpg"); + BackupRunResult missing = run(new BackupCatalog().where(BackupsTables.PLAN, "planKey", "nightly").get(0)); + assertThat(missing.getFilesMissing()).isGreaterThan(0); + assertThat(missing.getWarnings()).anyMatch(w -> w.contains("missing")); + assertThat(missing.getStatus()).isEqualTo("SUCCEEDED_WITH_WARNINGS"); + } + + + + /******************************************************************************* + ** A storage failure mid-run leaves the run FAILED with no manifest (so it is + ** not usable and the checker treats the prefix as debris). + *******************************************************************************/ + @Test + void testStorageFailureLeavesNoManifest() throws Exception + { + BackupsTestUtils.BACKUPS.withFailWritesFor(key -> key.contains("/tables/blogPost/")); + try + { + BackupRunResult result = run(plan(Map.of())); + assertThat(result.getStatus()).isEqualTo("FAILED"); + assertThat(result.isUsable()).isFalse(); + assertThat(result.getErrorMessage()).contains("injected write failure"); + assertThat(BackupsTestUtils.BACKUPS.list(result.getRunPrefix() + "manifest.json")).isEmpty(); + assertThat(new BackupCatalog().get(BackupsTables.RUN, result.getRunId()).orElseThrow().getValueString("status")).isEqualTo("FAILED"); + } + finally + { + BackupsTestUtils.BACKUPS.withFailWritesFor(null); + } + } + + + + /******************************************************************************* + ** Required snapshots fail before anything is written (no backend can open one yet). + *******************************************************************************/ + @Test + void testSnapshotRequiredFailsBeforeWriting() throws Exception + { + BackupRunResult result = run(plan(Map.of("consistency", "SNAPSHOT_REQUIRED"))); + assertThat(result.getStatus()).isEqualTo("FAILED"); + assertThat(BackupsTestUtils.BACKUPS.list("")).isEmpty(); + } + + + + /******************************************************************************* + ** CSV (and gzip) output, a page size smaller than the table (keyset paging), part + ** rotation by rows, and the rowsHash staying the same whatever the format. + *******************************************************************************/ + @Test + void testCsvAndPaging() throws Exception + { + BackupRunResult jsonl = run(plan(Map.of("pageSize", 2))); + BackupRunResult csv = run(plan(Map.of("name", "Csv", "planKey", "csv", "format", "CSV", "pageSize", 3))); + + BackupManifest jm = BackupManifest.read(BackupsTestUtils.BACKUPS.getBytes(jsonl.getManifestKey())); + BackupManifest cm = BackupManifest.read(BackupsTestUtils.BACKUPS.getBytes(csv.getManifestKey())); + assertThat(cm.table("blogPost").rowsHash()).isEqualTo(jm.table("blogPost").rowsHash()); + assertThat(cm.table("blogPost").parts().get(0).key()).endsWith(".csv.gz"); + + String text = gunzip(BackupsTestUtils.BACKUPS.getBytes(csv.getRunPrefix() + cm.table("blogPost").parts().get(0).key())); + List records = com.kingsrook.qbits.backups.codec.CsvRowCodec.splitRecords(text); + assertThat(records).hasSize(8); + assertThat(records.get(0)).startsWith("authorId,body,createDate"); + String authors = gunzip(BackupsTestUtils.BACKUPS.getBytes(csv.getRunPrefix() + cm.table("author").parts().get(0).key())); + assertThat(com.kingsrook.qbits.backups.codec.CsvRowCodec.splitRecords(authors)).hasSize(4); + assertThat(authors).contains("\\N").contains("\"Line one\nLine two, with \"\"quotes\"\""); + } + + + + /******************************************************************************* + ** Scope: explicit tables with a filter and field exclusions, and SECRET never included. + *******************************************************************************/ + @Test + void testScopeRulesFilterAndExclusions() throws Exception + { + QRecord plan = plan(Map.of("scopeType", "TABLES")); + new BackupCatalog().insert(BackupsTables.PLAN_SCOPE, Map.of("backupPlanId", plan.getValueInteger("id"), "action", "INCLUDE", "selectorType", "TABLE", + "selectorValue", "blogPost", "filterJson", "{\"criteria\":[{\"fieldName\":\"published\",\"operator\":\"EQUALS\",\"values\":[true]}]}", + "includeAssociatedChildren", false, "fieldExclusions", "body")); + BackupRunResult result = run(plan); + BackupManifest m = BackupManifest.read(BackupsTestUtils.BACKUPS.getBytes(result.getManifestKey())); + assertThat(m.tables()).extracting(BackupManifest.TableEntry::name).containsExactly("blogPost"); + assertThat(m.table("blogPost").rowCount()).isEqualTo(3); + assertThat(m.table("blogPost").excludedFields()).extracting(BackupManifest.ExcludedField::name).contains("body", "secretNote"); + assertThat(gunzip(BackupsTestUtils.BACKUPS.getBytes(result.getRunPrefix() + m.table("blogPost").parts().get(0).key()))).doesNotContain("\"body\""); + + ScopeResolver.ResolvedScope scope = new ScopeResolver(QContext.getQInstance()).resolve(PlanSettings.adHoc().with("scopeType", "TABLES"), + List.of(new ScopeResolver.ScopeRule(true, "TABLE", "backups_plan", null, false, null))); + assertThat(scope.tables()).isEmpty(); + assertThat(scope.excluded()).extracting(ScopeResolver.ExcludedTable::reason).containsExactly("BACKUPS"); + } +} -- 2.54.0