feat: backups foundation - tables, scope, codecs, manifest, backup engine, file copy, home

Tables (spec 4), permissions, scope resolver and table roles, canonical JSONL
and CSV codecs, per-part SHA-256 and rowsHash, manifest written last, the backup
engine with referenced-file copy into the deduplicated store, a narrow storage
interface with an in-memory implementation, Run backup now (streamed ETL), and the
Backups app home. Stand-ins for unreleased core hooks are marked in the code.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

src/main/java/com/kingsrook/qbits/backups/BackupsQBitConfig.java

@@ -18,16 +18,20 @@ package com.kingsrook.qbits.backups;
 
 
 import java.util.List;
+import java.util.function.Function;
+import com.kingsrook.qbits.backups.storage.BackupStorage;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
 /*******************************************************************************
- ** Host-facing configuration for the Backups QBit.
+ ** Configuration of the Backups qbit.
  **
- ** Tables are named backups_<name> (lowercase snake_case, backend name equal
- ** to the table name); permissions are backups.<action>.
+ ** Storage is resolved through a function from a bucket alias (the value of
+ ** backups_plan.storageBucketAlias) to a BackupStorage.  The production resolver
+ ** is backed by qbit-object-storage; tests and local development use the
+ ** in-memory implementation.  No credential is ever held here or in a table.
  *******************************************************************************/
 public class BackupsQBitConfig implements QBitConfig
 {
@@ -35,6 +39,18 @@ public class BackupsQBitConfig implements QBitConfig
    public static final String PERMISSION_BASE_NAME = "backups";
 
    private String backendName;
+   private String appName  = "backups";
+   private String appLabel = "Backups";
+   private String parentAppName;
+   private String schedulerName;
+   private String instanceName;
+
+   private Integer maxConcurrentRuns       = 1;
+   private Integer slaCheckIntervalSeconds = 300;
+   private Boolean openWhenAnonymous       = false;
+   private Boolean prometheusEndpoint      = false;
+
+   private Function<String, BackupStorage> storageResolver;
 
 
 
@@ -89,4 +105,214 @@ public class BackupsQBitConfig implements QBitConfig
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for appName
+    *******************************************************************************/
+   public String getAppName()
+   {
+      return (appName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for appName
+    *******************************************************************************/
+   public BackupsQBitConfig withAppName(String appName)
+   {
+      this.appName = appName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for appLabel
+    *******************************************************************************/
+   public String getAppLabel()
+   {
+      return (appLabel);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for appLabel
+    *******************************************************************************/
+   public BackupsQBitConfig withAppLabel(String appLabel)
+   {
+      this.appLabel = appLabel;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for parentAppName (for example "admin", so this is Admin, Backups).
+    *******************************************************************************/
+   public String getParentAppName()
+   {
+      return (parentAppName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for parentAppName
+    *******************************************************************************/
+   public BackupsQBitConfig withParentAppName(String parentAppName)
+   {
+      this.parentAppName = parentAppName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for schedulerName (the QQQ scheduler used for plan jobs and the SLA checker)
+    *******************************************************************************/
+   public String getSchedulerName()
+   {
+      return (schedulerName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for schedulerName
+    *******************************************************************************/
+   public BackupsQBitConfig withSchedulerName(String schedulerName)
+   {
+      this.schedulerName = schedulerName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for instanceName (used in object keys); defaults to the QInstance name
+    *******************************************************************************/
+   public String getInstanceName()
+   {
+      return (instanceName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for instanceName
+    *******************************************************************************/
+   public BackupsQBitConfig withInstanceName(String instanceName)
+   {
+      this.instanceName = instanceName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for maxConcurrentRuns
+    *******************************************************************************/
+   public Integer getMaxConcurrentRuns()
+   {
+      return (maxConcurrentRuns);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for maxConcurrentRuns
+    *******************************************************************************/
+   public BackupsQBitConfig withMaxConcurrentRuns(Integer maxConcurrentRuns)
+   {
+      this.maxConcurrentRuns = maxConcurrentRuns;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for slaCheckIntervalSeconds
+    *******************************************************************************/
+   public Integer getSlaCheckIntervalSeconds()
+   {
+      return (slaCheckIntervalSeconds);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for slaCheckIntervalSeconds
+    *******************************************************************************/
+   public BackupsQBitConfig withSlaCheckIntervalSeconds(Integer slaCheckIntervalSeconds)
+   {
+      this.slaCheckIntervalSeconds = slaCheckIntervalSeconds;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for openWhenAnonymous (local development only)
+    *******************************************************************************/
+   public Boolean getOpenWhenAnonymous()
+   {
+      return (openWhenAnonymous);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for openWhenAnonymous
+    *******************************************************************************/
+   public BackupsQBitConfig withOpenWhenAnonymous(Boolean openWhenAnonymous)
+   {
+      this.openWhenAnonymous = openWhenAnonymous;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for prometheusEndpoint
+    *******************************************************************************/
+   public Boolean getPrometheusEndpoint()
+   {
+      return (prometheusEndpoint);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for prometheusEndpoint
+    *******************************************************************************/
+   public BackupsQBitConfig withPrometheusEndpoint(Boolean prometheusEndpoint)
+   {
+      this.prometheusEndpoint = prometheusEndpoint;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for storageResolver
+    *******************************************************************************/
+   public Function<String, BackupStorage> getStorageResolver()
+   {
+      return (storageResolver);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for storageResolver: bucket alias to storage
+    *******************************************************************************/
+   public BackupsQBitConfig withStorageResolver(Function<String, BackupStorage> storageResolver)
+   {
+      this.storageResolver = storageResolver;
+      return (this);
+   }
+
 }

src/main/java/com/kingsrook/qbits/backups/BackupsQBitProducer.java

@@ -100,4 +100,54 @@ public class BackupsQBitProducer implements QBitMetaDataProducer<BackupsQBitConf
       return (qBitConfig != null);
    }
 
+
+
+   /*******************************************************************************
+    ** Build everything and add it to the instance: possible-value sources, joins,
+    ** tables, processes, widgets and the app.
+    *******************************************************************************/
+   @Override
+   public com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput produce(com.kingsrook.qqq.backend.core.model.metadata.QInstance qInstance) throws com.kingsrook.qqq.backend.core.exceptions.QException
+   {
+      java.util.List<String> errors = new java.util.ArrayList<>();
+      qBitConfig.validate(qInstance, errors);
+      if(!errors.isEmpty())
+      {
+         throw (new com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfigValidationException(qBitConfig, errors));
+      }
+      QBitMetaData qBit = getQBitMetaData();
+      qInstance.addQBit(qBit);
+      BackupsRuntime.install(qBitConfig);
+
+      com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput output = new com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput();
+      for(var possibleValueSource : com.kingsrook.qbits.backups.model.BackupsTables.possibleValueSources())
+      {
+         output.add(possibleValueSource);
+      }
+      for(var join : com.kingsrook.qbits.backups.model.BackupsTables.joins())
+      {
+         output.add(join);
+      }
+      java.util.List<com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData> tables = new java.util.ArrayList<>();
+      for(var table : com.kingsrook.qbits.backups.model.BackupsTables.tables(qBitConfig))
+      {
+         var customized = qBitConfig.getTableMetaDataCustomizer() == null ? table : qBitConfig.getTableMetaDataCustomizer().customizeMetaData(qInstance, table);
+         customized.setSourceQBitName(qBit.getName());
+         tables.add(customized);
+         output.add(customized);
+      }
+      for(var process : com.kingsrook.qbits.backups.processes.BackupsProcesses.processes())
+      {
+         process.setSourceQBitName(qBit.getName());
+         output.add(process);
+      }
+      for(var widget : com.kingsrook.qbits.backups.app.BackupsApp.widgets())
+      {
+         output.add(widget);
+      }
+      output.add(com.kingsrook.qbits.backups.app.BackupsApp.app(qBitConfig, tables));
+      output.addSelfToInstance(qInstance);
+      return (new com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput());
+   }
+
 }

src/main/java/com/kingsrook/qbits/backups/BackupsRuntime.java

@@ -0,0 +1,195 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups;
+
+
+import java.time.Clock;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Function;
+import com.kingsrook.qbits.backups.alerts.BackupAlertSink;
+import com.kingsrook.qbits.backups.alerts.NoOpBackupAlertSink;
+import com.kingsrook.qbits.backups.storage.BackupStorage;
+import com.kingsrook.qbits.backups.storage.BackupStorageException;
+
+
+/*******************************************************************************
+ ** Process-wide state of the installed qbit: the config, the clock, the alert
+ ** sink and the per-process permission map.  Static because QQQ instantiates
+ ** steps and permission checkers by class reference.
+ *******************************************************************************/
+public final class BackupsRuntime
+{
+   private static final Map<String, String> PROCESS_PERMISSIONS = new ConcurrentHashMap<>();
+
+   private static volatile BackupsQBitConfig config;
+   private static volatile Clock             clock           = Clock.systemUTC();
+   private static volatile BackupAlertSink   alertSink       = new NoOpBackupAlertSink();
+   private static volatile Function<String, String> environment = System::getenv;
+
+
+
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private BackupsRuntime()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Install the config (called by the producer).
+    *******************************************************************************/
+   public static void install(BackupsQBitConfig newConfig)
+   {
+      config = newConfig;
+   }
+
+
+
+   /*******************************************************************************
+    ** Forget everything (tests).
+    *******************************************************************************/
+   public static void reset()
+   {
+      config = null;
+      clock = Clock.systemUTC();
+      alertSink = new NoOpBackupAlertSink();
+      environment = System::getenv;
+      PROCESS_PERMISSIONS.clear();
+   }
+
+
+
+   /*******************************************************************************
+    ** The installed config, or null.
+    *******************************************************************************/
+   public static BackupsQBitConfig getConfig()
+   {
+      return (config);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the qbit is installed.
+    *******************************************************************************/
+   public static boolean isInstalled()
+   {
+      return (config != null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether permissions are open on fully anonymous instances (local development).
+    *******************************************************************************/
+   public static boolean isOpenWhenAnonymous()
+   {
+      return (config != null && Boolean.TRUE.equals(config.getOpenWhenAnonymous()));
+   }
+
+
+
+   /*******************************************************************************
+    ** The clock (replaceable in tests).
+    *******************************************************************************/
+   public static Clock getClock()
+   {
+      return (clock);
+   }
+
+
+
+   /*******************************************************************************
+    ** Replace the clock.
+    *******************************************************************************/
+   public static void setClock(Clock newClock)
+   {
+      clock = newClock;
+   }
+
+
+
+   /*******************************************************************************
+    ** The alert sink.
+    *******************************************************************************/
+   public static BackupAlertSink getAlertSink()
+   {
+      return (alertSink);
+   }
+
+
+
+   /*******************************************************************************
+    ** Bind an alert sink (the Alerts qbit adapter, or a test double).
+    *******************************************************************************/
+   public static void setAlertSink(BackupAlertSink newSink)
+   {
+      alertSink = newSink == null ? new NoOpBackupAlertSink() : newSink;
+   }
+
+
+
+   /*******************************************************************************
+    ** Environment lookup (replaceable in tests).
+    *******************************************************************************/
+   public static Function<String, String> getEnvironment()
+   {
+      return (environment);
+   }
+
+
+
+   /*******************************************************************************
+    ** Replace the environment lookup.
+    *******************************************************************************/
+   public static void setEnvironment(Function<String, String> newEnvironment)
+   {
+      environment = newEnvironment;
+   }
+
+
+
+   /*******************************************************************************
+    ** The permission each qbit process needs, by process name.
+    *******************************************************************************/
+   public static Map<String, String> processPermissions()
+   {
+      return (PROCESS_PERMISSIONS);
+   }
+
+
+
+   /*******************************************************************************
+    ** Resolve the storage for a bucket alias.
+    *******************************************************************************/
+   public static BackupStorage storageFor(String bucketAlias)
+   {
+      if(config == null || config.getStorageResolver() == null)
+      {
+         throw (new BackupStorageException(BackupStorageException.Reason.UNSUPPORTED, "No storage resolver is configured for the Backups qbit"));
+      }
+      BackupStorage storage = config.getStorageResolver().apply(bucketAlias);
+      if(storage == null)
+      {
+         throw (new BackupStorageException(BackupStorageException.Reason.NOT_FOUND, "No storage bucket is configured with the alias: " + bucketAlias));
+      }
+      return (storage);
+   }
+}

src/main/java/com/kingsrook/qbits/backups/alerts/BackupAlertSink.java

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.alerts;
+
+
+import java.util.Map;
+
+
+/*******************************************************************************
+ ** Where backups raises and resolves alerts.  The adapter for the Alerts qbit
+ ** producer SPI binds here when that qbit is installed; otherwise the no-op
+ ** sink is used and every condition is still visible on the Backups home and in
+ ** the tables.  An alert has a stable dedupe key; counts and links only, never
+ ** values, keys or presigned URLs.
+ *******************************************************************************/
+public interface BackupAlertSink
+{
+
+   /***************************************************************************
+    ** Raise (or refresh) an alert.
+    **
+    ** @param type alert type, e.g. backups.plan.missed
+    ** @param dedupeKey stable key, type plus plan or run
+    ** @param severity severity
+    ** @param title short title
+    ** @param detail counts and context only
+    ***************************************************************************/
+   void raise(String type, String dedupeKey, Severity severity, String title, Map<String, String> detail);
+
+
+   /***************************************************************************
+    ** Resolve an alert that was raised under the dedupe key.
+    ***************************************************************************/
+   void resolve(String type, String dedupeKey);
+
+
+   /***************************************************************************
+    ** Check in a heartbeat monitor: the plan ran successfully.
+    ***************************************************************************/
+   void heartbeat(String monitorKey, long expectedEverySeconds, long graceSeconds);
+
+
+
+   /***************************************************************************
+    ** Severities.
+    ***************************************************************************/
+   enum Severity
+   {
+      INFO, WARNING, ERROR, CRITICAL
+   }
+}

src/main/java/com/kingsrook/qbits/backups/alerts/NoOpBackupAlertSink.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.alerts;
+
+
+import java.util.Map;
+
+
+/*******************************************************************************
+ ** The sink used when the Alerts qbit is not installed.
+ *******************************************************************************/
+public class NoOpBackupAlertSink implements BackupAlertSink
+{
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   public void raise(String type, String dedupeKey, Severity severity, String title, Map<String, String> detail)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   public void resolve(String type, String dedupeKey)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   public void heartbeat(String monitorKey, long expectedEverySeconds, long graceSeconds)
+   {
+   }
+}

src/main/java/com/kingsrook/qbits/backups/app/BackupsApp.java

@@ -0,0 +1,152 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.app;
+
+
+import java.util.List;
+import com.kingsrook.qbits.backups.BackupsQBitConfig;
+import com.kingsrook.qbits.backups.model.BackupsTables;
+import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
+/*******************************************************************************
+ ** The Backups app (a sub-app under Admin when configured) and its widgets.  The
+ ** home is the single owner of backup data: every card is built from the qbit
+ ** tables, and each has an honest empty state.  The nav lists the main tables;
+ ** sub-tables appear as child lists on their parents.
+ *******************************************************************************/
+public final class BackupsApp
+{
+   public static final String HOME_WIDGET        = "backupsHome";
+   public static final String HEALTH_CARD_WIDGET = "backupsHealthCard";
+
+
+
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private BackupsApp()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** The widgets.
+    *******************************************************************************/
+   public static List<QWidgetMetaData> widgets()
+   {
+      return (List.of(
+         new QWidgetMetaData().withName(HOME_WIDGET).withLabel("Backups").withType(WidgetType.HTML.getType())
+            .withCodeReference(new QCodeReference(HomeRenderer.class)).withGridColumns(12).withIsCard(false).withPermissionRules(BackupsTables.permissionRules()),
+         new QWidgetMetaData().withName(HEALTH_CARD_WIDGET).withLabel("Backups").withType(WidgetType.HTML.getType())
+            .withCodeReference(new QCodeReference(HealthCardRenderer.class)).withGridColumns(12).withIsCard(true).withPermissionRules(BackupsTables.permissionRules())));
+   }
+
+
+
+   /*******************************************************************************
+    ** The app: the main tables only; sub-tables are relations on their parents.
+    *******************************************************************************/
+   public static QAppMetaData app(BackupsQBitConfig config, List<QTableMetaData> tables)
+   {
+      QAppMetaData app = new QAppMetaData()
+         .withName(config.getAppName())
+         .withLabel(config.getAppLabel())
+         .withIcon(new QIcon("backup"))
+         .withPermissionRules(BackupsTables.permissionRules())
+         .withWidgets(List.of(HOME_WIDGET))
+         .withSection(new QAppSection().withName("backups").withLabel("Backups").withIcon(new QIcon("backup"))
+            .withTables(List.of(BackupsTables.PLAN, BackupsTables.RUN)))
+         .withSection(new QAppSection().withName("recovery").withLabel("Restore and verify").withIcon(new QIcon("restore"))
+            .withTables(List.of(BackupsTables.RESTORE_RUN, BackupsTables.VERIFICATION)))
+         .withSection(new QAppSection().withName("files").withLabel("Files").withIcon(new QIcon("attach_file"))
+            .withTables(List.of(BackupsTables.FILE_OBJECT)));
+      if(config.getParentAppName() != null)
+      {
+         app.setParentAppName(config.getParentAppName());
+      }
+      for(QTableMetaData table : tables)
+      {
+         if(BackupsTables.NAV_TABLES.contains(table.getName()))
+         {
+            app.withChild(table);
+         }
+      }
+      return (app);
+   }
+
+
+
+   /*******************************************************************************
+    ** The home.
+    *******************************************************************************/
+   public static class HomeRenderer extends AbstractWidgetRenderer
+   {
+      /*******************************************************************************
+       ** Render.
+       *******************************************************************************/
+      @Override
+      public RenderWidgetOutput render(RenderWidgetInput input) throws QException
+      {
+         return (new RenderWidgetOutput(new RawHTML(null, BackupsHomeView.render(BackupsHomeData.load(), pathPrefix()))));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The status card for Admin, Health.
+    *******************************************************************************/
+   public static class HealthCardRenderer extends AbstractWidgetRenderer
+   {
+      /*******************************************************************************
+       ** Render.
+       *******************************************************************************/
+      @Override
+      public RenderWidgetOutput render(RenderWidgetInput input) throws QException
+      {
+         return (new RenderWidgetOutput(new RawHTML("Backups", BackupsHomeView.renderHealthCard(BackupsHomeData.load(), pathPrefix()))));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The path prefix of table and record links.
+    *******************************************************************************/
+   static String pathPrefix()
+   {
+      var config = com.kingsrook.qbits.backups.BackupsRuntime.getConfig();
+      if(config == null)
+      {
+         return ("/backups");
+      }
+      return (config.getParentAppName() != null ? "/" + config.getParentAppName() + "/" + config.getAppName() : "/" + config.getAppName());
+   }
+}

src/main/java/com/kingsrook/qbits/backups/app/BackupsHomeData.java

@@ -0,0 +1,255 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.app;
+
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qbits.backups.BackupsRuntime;
+import com.kingsrook.qbits.backups.model.BackupsTables;
+import com.kingsrook.qbits.backups.run.BackupCatalog;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** The data behind the Backups home: plan status, what needs attention, recent
+ ** activity and the size trend.  Everything is read from the qbit tables; nothing
+ ** is invented.  Home cards read the denormalized plan fields (slaStatus, last
+ ** success) that the processes and the SLA checker maintain.
+ *******************************************************************************/
+public class BackupsHomeData
+{
+   /** Plan statuses that need attention (the plan or its backups have a problem). */
+   public static final List<String> ATTENTION = List.of("LATE", "FAILING", "MISSING_IN_STORAGE", "UNVERIFIED", "ABANDONED", "OWNER_INVALID");
+
+   private final List<QRecord> plans;
+   private final List<QRecord> recentRuns;
+   private final List<QRecord> failedRuns;
+   private final List<QRecord> verifications;
+   private final List<QRecord> restores;
+   private final Instant       now;
+
+
+
+   /*******************************************************************************
+    ** Constructor.
+    *******************************************************************************/
+   BackupsHomeData(List<QRecord> plans, List<QRecord> recentRuns, List<QRecord> failedRuns, List<QRecord> verifications, List<QRecord> restores, Instant now)
+   {
+      this.plans = plans;
+      this.recentRuns = recentRuns;
+      this.failedRuns = failedRuns;
+      this.verifications = verifications;
+      this.restores = restores;
+      this.now = now;
+   }
+
+
+
+   /*******************************************************************************
+    ** Load from the tables.
+    *******************************************************************************/
+   public static BackupsHomeData load() throws QException
+   {
+      BackupCatalog catalog = new BackupCatalog();
+      Instant       now     = BackupsRuntime.getClock().instant();
+      List<QRecord> plans   = catalog.query(BackupsTables.PLAN, new QQueryFilter().withOrderBy(new QFilterOrderBy("name", true)));
+
+      QQueryFilter recent = new QQueryFilter().withOrderBy(new QFilterOrderBy("id", false));
+      recent.setLimit(200);
+      List<QRecord> runs = catalog.query(BackupsTables.RUN, recent);
+      List<QRecord> failed = new ArrayList<>();
+      Instant weekAgo = now.minus(Duration.ofDays(7));
+      for(QRecord run : runs)
+      {
+         Instant start = run.getValueInstant("startDate");
+         if(List.of("FAILED", "TIMED_OUT", "ABANDONED").contains(run.getValueString("status")) && (start == null || start.isAfter(weekAgo)))
+         {
+            failed.add(run);
+         }
+      }
+      QQueryFilter verificationFilter = new QQueryFilter().withOrderBy(new QFilterOrderBy("id", false));
+      verificationFilter.setLimit(100);
+      QQueryFilter restoreFilter = new QQueryFilter().withOrderBy(new QFilterOrderBy("id", false));
+      restoreFilter.setLimit(20);
+      return (new BackupsHomeData(plans, runs, failed, catalog.query(BackupsTables.VERIFICATION, verificationFilter),
+         catalog.query(BackupsTables.RESTORE_RUN, restoreFilter), now));
+   }
+
+
+
+   /*******************************************************************************
+    ** Plans.
+    *******************************************************************************/
+   public List<QRecord> plans()
+   {
+      return (plans);
+   }
+
+
+
+   /*******************************************************************************
+    ** Recent runs (newest first).
+    *******************************************************************************/
+   public List<QRecord> recentRuns()
+   {
+      return (recentRuns);
+   }
+
+
+
+   /*******************************************************************************
+    ** Runs that failed in the last 7 days.
+    *******************************************************************************/
+   public List<QRecord> failedRuns()
+   {
+      return (failedRuns);
+   }
+
+
+
+   /*******************************************************************************
+    ** Verifications (newest first).
+    *******************************************************************************/
+   public List<QRecord> verifications()
+   {
+      return (verifications);
+   }
+
+
+
+   /*******************************************************************************
+    ** Restores (newest first).
+    *******************************************************************************/
+   public List<QRecord> restores()
+   {
+      return (restores);
+   }
+
+
+
+   /*******************************************************************************
+    ** The time the data was loaded.
+    *******************************************************************************/
+   public Instant now()
+   {
+      return (now);
+   }
+
+
+
+   /*******************************************************************************
+    ** Plans that need attention.
+    *******************************************************************************/
+   public List<QRecord> plansNeedingAttention()
+   {
+      return (plans.stream().filter(p -> Boolean.TRUE.equals(p.getValueBoolean("isActive")) && ATTENTION.contains(p.getValueString("slaStatus"))).toList());
+   }
+
+
+
+   /*******************************************************************************
+    ** The overall state: NO_PLANS, ATTENTION or OK.
+    *******************************************************************************/
+   public String overallState()
+   {
+      if(plans.isEmpty())
+      {
+         return ("NO_PLANS");
+      }
+      return (plansNeedingAttention().isEmpty() && failedRuns.isEmpty() ? "OK" : "ATTENTION");
+   }
+
+
+
+   /*******************************************************************************
+    ** Hours since the last successful backup of a plan, or null when it has none.
+    *******************************************************************************/
+   public Long ageHours(QRecord plan)
+   {
+      Instant last = plan.getValueInstant("lastSuccessDate");
+      return (last == null ? null : Duration.between(last, now).toHours());
+   }
+
+
+
+   /*******************************************************************************
+    ** The latest test restore of a plan, or null.
+    *******************************************************************************/
+   public QRecord latestTestRestore(Integer planId)
+   {
+      for(QRecord verification : verifications)
+      {
+         if("TEST_RESTORE".equals(verification.getValueString("level")))
+         {
+            Integer runId = verification.getValueInteger("backupRunId");
+            for(QRecord run : recentRuns)
+            {
+               if(runId != null && runId.equals(run.getValueInteger("id")) && planId.equals(run.getValueInteger("backupPlanId")))
+               {
+                  return (verification);
+               }
+            }
+         }
+      }
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Successful runs of a plan, oldest first, for the trend (bytes stored, rows, duration).
+    *******************************************************************************/
+   public List<QRecord> successfulRuns(Integer planId)
+   {
+      List<QRecord> rs = new ArrayList<>();
+      for(QRecord run : recentRuns)
+      {
+         if(planId.equals(run.getValueInteger("backupPlanId")) && run.getValueString("status") != null && run.getValueString("status").startsWith("SUCCEEDED"))
+         {
+            rs.add(0, run);
+         }
+      }
+      return (rs);
+   }
+
+
+
+   /*******************************************************************************
+    ** Runs kept per retention class for a plan (as classified by the last prune).
+    *******************************************************************************/
+   public Map<String, Integer> retentionCounts(Integer planId)
+   {
+      Map<String, Integer> counts = new LinkedHashMap<>();
+      for(QRecord run : recentRuns)
+      {
+         if(planId.equals(run.getValueInteger("backupPlanId")) && run.getValueString("status") != null && run.getValueString("status").startsWith("SUCCEEDED")
+            && run.getValue("prunedDate") == null)
+         {
+            counts.merge(run.getValueString("retentionClass") == null ? "UNCLASSIFIED" : run.getValueString("retentionClass"), 1, Integer::sum);
+         }
+      }
+      return (counts);
+   }
+}

src/main/java/com/kingsrook/qbits/backups/app/BackupsHomeView.java

@@ -0,0 +1,396 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.app;
+
+
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qbits.backups.model.BackupsTables;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** Renders the Backups home: one card per concern, each with an honest empty
+ ** state.  Cards stack to one column on narrow screens, long names wrap, and
+ ** there are no tables and no inner horizontal scroll (definition lists and
+ ** lists only), so it reads at every width in light and dark mode (colors are
+ ** inherited, not set).
+ *******************************************************************************/
+public final class BackupsHomeView
+{
+   private static final DateTimeFormatter WHEN = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm 'UTC'").withZone(ZoneOffset.UTC);
+
+   private static final String GRID  = "display:grid;grid-template-columns:repeat(auto-fit,minmax(min(100%,22rem),1fr));gap:1rem;";
+   private static final String CARD  = "border:1px solid rgba(127,127,127,.3);border-radius:8px;padding:1rem;min-width:0;overflow-wrap:anywhere;";
+   private static final String LIST  = "display:grid;grid-template-columns:minmax(7rem,auto) 1fr;gap:.25rem .75rem;margin:0 0 .75rem;font-size:.9rem;";
+   private static final String ITEMS = "margin:0 0 .75rem;padding-left:1.25rem;font-size:.9rem;";
+   private static final String MUTED = "opacity:.75;";
+
+
+
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private BackupsHomeView()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** The whole home; pathPrefix is the prefix of table and record links.
+    *******************************************************************************/
+   public static String render(BackupsHomeData data, String pathPrefix)
+   {
+      StringBuilder html = new StringBuilder("<div class=\"backups-home\" style=\"" + GRID + "\">");
+      planStatus(html, data, pathPrefix);
+      needsAttention(html, data, pathPrefix);
+      latestTestRestore(html, data, pathPrefix);
+      trend(html, data);
+      recentActivity(html, data, pathPrefix);
+      retention(html, data);
+      return (html.append("</div>").toString());
+   }
+
+
+
+   /*******************************************************************************
+    ** The one status card for Admin, Health.
+    *******************************************************************************/
+   public static String renderHealthCard(BackupsHomeData data, String pathPrefix)
+   {
+      String text = switch(data.overallState())
+      {
+         case "NO_PLANS" -> "No backup plans yet.";
+         case "ATTENTION" -> "Needs attention: see the Backups home.";
+         default -> "All backups are within their SLA.";
+      };
+      return ("<div style=\"" + CARD + "\"><strong>Backups</strong><p>" + esc(text) + "</p><a href=\"" + esc(pathPrefix) + "\">Open the Backups home</a></div>");
+   }
+
+
+
+   /*******************************************************************************
+    ** Plan status: last successful backup, age against the SLA, last run, verification.
+    *******************************************************************************/
+   private static void planStatus(StringBuilder html, BackupsHomeData data, String prefix)
+   {
+      open(html, "Plan status");
+      if(data.plans().isEmpty())
+      {
+         empty(html, "No backup plans yet. Create one from Backup plans, New.");
+      }
+      for(QRecord plan : data.plans())
+      {
+         Long   age = data.ageHours(plan);
+         Integer sla = plan.getValueInteger("slaHours");
+         html.append("<dl style=\"").append(LIST).append("\">");
+         html.append("<dt><strong>Plan</strong></dt><dd><a href=\"").append(esc(prefix + "/" + BackupsTables.PLAN + "/" + plan.getValue("id"))).append("\">")
+            .append(esc(plan.getValueString("name"))).append("</a>").append(Boolean.TRUE.equals(plan.getValueBoolean("isActive")) ? "" : " (not active)").append("</dd>");
+         row(html, "Status", label(plan.getValueString("slaStatus")));
+         row(html, "Last backup", plan.getValueInstant("lastSuccessDate") == null ? "No successful backup yet" : WHEN.format(plan.getValueInstant("lastSuccessDate")));
+         if(age != null && sla != null)
+         {
+            row(html, "Age against SLA", age + " h of " + sla + " h" + (age > sla ? " - late" : ""));
+         }
+         row(html, "Last verified", plan.getValueInstant("lastVerifiedDate") == null ? "Not verified yet" : WHEN.format(plan.getValueInstant("lastVerifiedDate")));
+         row(html, "Schedule", plan.getValueString("cronExpression") == null ? "Manual only" : plan.getValueString("cronExpression"));
+         if(plan.getValueString("objectLockMode") != null && !"NONE".equals(plan.getValueString("objectLockMode")))
+         {
+            row(html, "Object lock", plan.getValueString("objectLockMode") + ", " + plan.getValueInteger("objectLockDays") + " days");
+         }
+         html.append("</dl>");
+      }
+      link(html, prefix, BackupsTables.PLAN, "Backup plans");
+      close(html);
+   }
+
+
+
+   /*******************************************************************************
+    ** Needs attention.
+    *******************************************************************************/
+   private static void needsAttention(StringBuilder html, BackupsHomeData data, String prefix)
+   {
+      open(html, "Needs attention");
+      List<QRecord> plans = data.plansNeedingAttention();
+      if(data.plans().isEmpty())
+      {
+         empty(html, "Nothing to check yet: no plans.");
+      }
+      else if(plans.isEmpty() && data.failedRuns().isEmpty())
+      {
+         empty(html, "Nothing needs attention.");
+      }
+      else
+      {
+         html.append("<ul style=\"").append(ITEMS).append("\">");
+         for(QRecord plan : plans)
+         {
+            html.append("<li><a href=\"").append(esc(prefix + "/" + BackupsTables.PLAN + "/" + plan.getValue("id"))).append("\">").append(esc(plan.getValueString("name")))
+               .append("</a>: ").append(esc(label(plan.getValueString("slaStatus")))).append("</li>");
+         }
+         for(QRecord run : data.failedRuns())
+         {
+            html.append("<li><a href=\"").append(esc(prefix + "/" + BackupsTables.RUN + "/" + run.getValue("id"))).append("\">Run ").append(esc(run.getValueString("runKey")))
+               .append("</a> ").append(esc(label(run.getValueString("status")))).append("</li>");
+         }
+         html.append("</ul>");
+      }
+      link(html, prefix, BackupsTables.RUN, "All runs");
+      close(html);
+   }
+
+
+
+   /*******************************************************************************
+    ** Latest test restore per plan.
+    *******************************************************************************/
+   private static void latestTestRestore(StringBuilder html, BackupsHomeData data, String prefix)
+   {
+      open(html, "Latest test restore");
+      boolean any = false;
+      for(QRecord plan : data.plans())
+      {
+         QRecord v = data.latestTestRestore(plan.getValueInteger("id"));
+         if(v == null)
+         {
+            continue;
+         }
+         any = true;
+         html.append("<dl style=\"").append(LIST).append("\">");
+         row(html, "Plan", plan.getValueString("name"));
+         row(html, "Result", label(v.getValueString("status")));
+         row(html, "Compared", v.getValueInteger("tablesChecked") + " tables, " + v.getValueLong("rowsChecked") + " rows, " + v.getValueInteger("mismatches") + " mismatches");
+         row(html, "Scratch target", v.getValueString("scratchTarget") == null ? "Not recorded" : v.getValueString("scratchTarget"));
+         html.append("</dl>");
+      }
+      if(!any)
+      {
+         empty(html, "No test restore yet. Run Verify backup, Test restore on a run, or set a verify schedule on the plan.");
+      }
+      link(html, prefix, BackupsTables.VERIFICATION, "All verifications");
+      close(html);
+   }
+
+
+
+   /*******************************************************************************
+    ** Size and duration trend per plan, as numbers next to each other.
+    *******************************************************************************/
+   private static void trend(StringBuilder html, BackupsHomeData data)
+   {
+      open(html, "Size and duration trend");
+      boolean any = false;
+      for(QRecord plan : data.plans())
+      {
+         List<QRecord> runs = data.successfulRuns(plan.getValueInteger("id"));
+         if(runs.isEmpty())
+         {
+            continue;
+         }
+         any = true;
+         QRecord first = runs.get(0);
+         QRecord last  = runs.get(runs.size() - 1);
+         html.append("<dl style=\"").append(LIST).append("\">");
+         row(html, "Plan", plan.getValueString("name"));
+         row(html, "Successful runs", String.valueOf(runs.size()));
+         row(html, "Stored (first, latest)", bytes(first.getValueLong("bytesStored")) + ", " + bytes(last.getValueLong("bytesStored")));
+         row(html, "Rows (first, latest)", number(first.getValueLong("rowsTotal")) + ", " + number(last.getValueLong("rowsTotal")));
+         row(html, "Duration (latest)", last.getValueLong("durationMs") == null ? "Not recorded" : last.getValueLong("durationMs") / 1000 + " s");
+         html.append("</dl>");
+      }
+      if(!any)
+      {
+         empty(html, "Trends appear after the first successful run.");
+      }
+      close(html);
+   }
+
+
+
+   /*******************************************************************************
+    ** Recent activity: the last runs and restores.
+    *******************************************************************************/
+   private static void recentActivity(StringBuilder html, BackupsHomeData data, String prefix)
+   {
+      open(html, "Recent activity");
+      if(data.recentRuns().isEmpty() && data.restores().isEmpty() && data.verifications().isEmpty())
+      {
+         empty(html, "Nothing has run yet.");
+      }
+      else
+      {
+         html.append("<ul style=\"").append(ITEMS).append("\">");
+         data.recentRuns().stream().limit(6).forEach(run ->
+         {
+            Instant start = run.getValueInstant("startDate");
+            html.append("<li>Backup <a href=\"").append(esc(prefix + "/" + BackupsTables.RUN + "/" + run.getValue("id"))).append("\">").append(esc(run.getValueString("runKey")))
+               .append("</a>: ").append(esc(label(run.getValueString("status")))).append(start == null ? "" : ", " + WHEN.format(start)).append("</li>");
+         });
+         data.restores().stream().limit(2).forEach(restore -> html.append("<li>Restore <a href=\"").append(esc(prefix + "/" + BackupsTables.RESTORE_RUN + "/" + restore.getValue("id")))
+            .append("\">#").append(esc(String.valueOf(restore.getValue("id")))).append("</a>: ").append(esc(label(restore.getValueString("status")))).append("</li>"));
+         data.verifications().stream().limit(2).forEach(v -> html.append("<li>Verify <a href=\"").append(esc(prefix + "/" + BackupsTables.VERIFICATION + "/" + v.getValue("id")))
+            .append("\">").append(esc(label(v.getValueString("level")))).append("</a>: ").append(esc(label(v.getValueString("status")))).append("</li>"));
+         html.append("</ul>");
+      }
+      link(html, prefix, BackupsTables.RUN, "All runs");
+      close(html);
+   }
+
+
+
+   /*******************************************************************************
+    ** Retention: runs kept per class and the oldest restorable point.
+    *******************************************************************************/
+   private static void retention(StringBuilder html, BackupsHomeData data)
+   {
+      open(html, "Retention");
+      boolean any = false;
+      for(QRecord plan : data.plans())
+      {
+         Map<String, Integer> counts = data.retentionCounts(plan.getValueInteger("id"));
+         if(counts.isEmpty())
+         {
+            continue;
+         }
+         any = true;
+         html.append("<dl style=\"").append(LIST).append("\">");
+         row(html, "Plan", plan.getValueString("name"));
+         row(html, "Kept", counts.entrySet().stream().map(e -> e.getValue() + " " + label(e.getKey()).toLowerCase()).reduce((a, b) -> a + ", " + b).orElse(""));
+         List<QRecord> runs = data.successfulRuns(plan.getValueInteger("id"));
+         Instant oldest = runs.isEmpty() ? null : runs.get(0).getValueInstant("startDate");
+         row(html, "Oldest restorable", oldest == null ? "Not known" : WHEN.format(oldest));
+         html.append("</dl>");
+      }
+      if(!any)
+      {
+         empty(html, "Retention starts once runs exist.");
+      }
+      close(html);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void open(StringBuilder html, String title)
+   {
+      html.append("<section style=\"").append(CARD).append("\"><h3 style=\"margin:0 0 .75rem;font-size:1rem;\">").append(esc(title)).append("</h3>");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void close(StringBuilder html)
+   {
+      html.append("</section>");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void empty(StringBuilder html, String text)
+   {
+      html.append("<p style=\"").append(MUTED).append("margin:0 0 .75rem;\">").append(esc(text)).append("</p>");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void row(StringBuilder html, String name, String value)
+   {
+      html.append("<dt style=\"").append(MUTED).append("\">").append(esc(name)).append("</dt><dd style=\"margin:0;\">").append(esc(value == null ? "" : value)).append("</dd>");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void link(StringBuilder html, String prefix, String table, String text)
+   {
+      html.append("<a href=\"").append(esc(prefix + "/" + table)).append("\">").append(esc(text)).append("</a>");
+   }
+
+
+
+   /*******************************************************************************
+    ** SOME_STATUS to Some status.
+    *******************************************************************************/
+   static String label(String status)
+   {
+      if(status == null || status.isEmpty())
+      {
+         return ("Unknown");
+      }
+      String s = status.replace('_', ' ').toLowerCase();
+      return (Character.toUpperCase(s.charAt(0)) + s.substring(1));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static String bytes(Long value)
+   {
+      if(value == null)
+      {
+         return ("Not recorded");
+      }
+      double v = value;
+      String[] units = { "B", "KB", "MB", "GB", "TB" };
+      int      i     = 0;
+      while(v >= 1024 && i < units.length - 1)
+      {
+         v /= 1024;
+         i++;
+      }
+      return (i == 0 ? String.format("%d B", value) : String.format("%.1f %s", v, units[i]));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static String number(Long value)
+   {
+      return (value == null ? "Not recorded" : String.format("%,d", value));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static String esc(String s)
+   {
+      return (s == null ? "" : s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace("\"", "&quot;"));
+   }
+}

src/main/java/com/kingsrook/qbits/backups/codec/ArtifactTransform.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.codec;
+
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+
+/*******************************************************************************
+ ** A reversible transform applied to the bytes of an artifact after compression,
+ ** before they reach storage.  The only implementation is client-side encryption
+ ** (the streaming cipher of the QQQ crypto package); IDENTITY is the default.
+ *******************************************************************************/
+public interface ArtifactTransform
+{
+   /** No transform. */
+   ArtifactTransform IDENTITY = new ArtifactTransform()
+   {
+      /*******************************************************************************
+       ** {@inheritDoc}
+       *******************************************************************************/
+      @Override
+      public OutputStream wrap(OutputStream out)
+      {
+         return (out);
+      }
+
+
+
+      /*******************************************************************************
+       ** {@inheritDoc}
+       *******************************************************************************/
+      @Override
+      public InputStream unwrap(InputStream in)
+      {
+         return (in);
+      }
+   };
+
+
+
+   /***************************************************************************
+    ** Wrap the stream going to storage.
+    ***************************************************************************/
+   OutputStream wrap(OutputStream out) throws IOException;
+
+
+   /***************************************************************************
+    ** Wrap a stream read from storage, to undo the transform.
+    ***************************************************************************/
+   InputStream unwrap(InputStream in) throws IOException;
+
+
+   /***************************************************************************
+    ** Whether this transform encrypts.
+    ***************************************************************************/
+   default boolean encrypts()
+   {
+      return (false);
+   }
+
+
+   /***************************************************************************
+    ** File name suffix added to artifact keys, e.g. ".tink".
+    ***************************************************************************/
+   default String keySuffix()
+   {
+      return ("");
+   }
+}

src/main/java/com/kingsrook/qbits/backups/codec/CsvRowCodec.java

@@ -0,0 +1,277 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.codec;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeSet;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
+/*******************************************************************************
+ ** The CSV row format (spec 8.3): RFC 4180, UTF-8 without a BOM, a header of
+ ** field names, and the same canonical value text as JSONL.  Null is written as
+ ** \N (unquoted) so it can be told from the empty string; a string that is
+ ** literally \N is quoted.  A multi-line value is one record in a quoted field,
+ ** so the codec works on records (a record may span lines): encode() returns a
+ ** text that may contain embedded newlines, and the part writer appends one
+ ** newline after each.
+ *******************************************************************************/
+public class CsvRowCodec implements RowCodec
+{
+   public static final String NULL_TOKEN = "\\N";
+
+   private final QTableMetaData table;
+   private final List<String>   fieldNames;
+
+
+
+   /*******************************************************************************
+    ** Constructor.
+    *******************************************************************************/
+   public CsvRowCodec(QTableMetaData table, java.util.Collection<String> excludedFields)
+   {
+      this.table = table;
+      TreeSet<String> names = new TreeSet<>(table.getFields().keySet());
+      if(excludedFields != null)
+      {
+         names.removeAll(excludedFields);
+      }
+      this.fieldNames = new ArrayList<>(names);
+   }
+
+
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   public List<String> getFieldNames()
+   {
+      return (fieldNames);
+   }
+
+
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   public String header()
+   {
+      return (String.join(",", fieldNames.stream().map(CsvRowCodec::quoteIfNeeded).toList()));
+   }
+
+
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   public String encode(QRecord record)
+   {
+      StringBuilder sb = new StringBuilder();
+      for(int i = 0; i < fieldNames.size(); i++)
+      {
+         if(i > 0)
+         {
+            sb.append(',');
+         }
+         String       name  = fieldNames.get(i);
+         Serializable value = record.getValue(name);
+         if(value == null)
+         {
+            sb.append(NULL_TOKEN);
+         }
+         else
+         {
+            String text = FieldValueCodec.toCanonicalText(table.getField(name).getType(), value);
+            sb.append(NULL_TOKEN.equals(text) ? "\"" + text + "\"" : quoteIfNeeded(text));
+         }
+      }
+      return (sb.toString());
+   }
+
+
+
+   /*******************************************************************************
+    ** Parse one CSV record (as returned by CsvRecordReader, header excluded) to a record.
+    ** The line may contain embedded newlines inside quoted fields.
+    *******************************************************************************/
+   @Override
+   public QRecord decode(String line) throws QException
+   {
+      List<String> cells = splitCells(line);
+      return (toRecord(cells, fieldNames));
+   }
+
+
+
+   /*******************************************************************************
+    ** Build a record from already-split cells and the header names.
+    *******************************************************************************/
+   public QRecord toRecord(List<String> cells, List<String> header) throws QException
+   {
+      if(cells.size() != header.size())
+      {
+         throw (new QException("CSV row has " + cells.size() + " cells but the header has " + header.size() + " for table " + table.getName()));
+      }
+      Map<String, Serializable> values = new LinkedHashMap<>();
+      for(int i = 0; i < header.size(); i++)
+      {
+         String         cell  = cells.get(i);
+         QFieldMetaData field = table.getFields().get(header.get(i));
+         if(cell == null)
+         {
+            values.put(header.get(i), null);
+         }
+         else if(field == null)
+         {
+            values.put(header.get(i), cell);
+         }
+         else
+         {
+            values.put(header.get(i), FieldValueCodec.fromCanonicalText(field.getType(), cell));
+         }
+      }
+      QRecord record = new QRecord();
+      record.setTableName(table.getName());
+      record.setValues(values);
+      return (record);
+   }
+
+
+
+   /*******************************************************************************
+    ** Quote a cell when it contains a comma, quote, CR or LF (or is empty after
+    ** being a header, which never happens), doubling embedded quotes.
+    *******************************************************************************/
+   static String quoteIfNeeded(String s)
+   {
+      if(s.indexOf(',') >= 0 || s.indexOf('"') >= 0 || s.indexOf('\n') >= 0 || s.indexOf('\r') >= 0)
+      {
+         return ("\"" + s.replace("\"", "\"\"") + "\"");
+      }
+      return (s);
+   }
+
+
+
+   /*******************************************************************************
+    ** Split one CSV record into cells.  Returns null for an unquoted \N cell (a null).
+    *******************************************************************************/
+   public static List<String> splitCells(String line)
+   {
+      List<String>  cells  = new ArrayList<>();
+      StringBuilder cell   = new StringBuilder();
+      boolean       quoted = false;
+      boolean       wasQuoted = false;
+      for(int i = 0; i < line.length(); i++)
+      {
+         char c = line.charAt(i);
+         if(quoted)
+         {
+            if(c == '"')
+            {
+               if(i + 1 < line.length() && line.charAt(i + 1) == '"')
+               {
+                  cell.append('"');
+                  i++;
+               }
+               else
+               {
+                  quoted = false;
+               }
+            }
+            else
+            {
+               cell.append(c);
+            }
+         }
+         else if(c == '"')
+         {
+            quoted = true;
+            wasQuoted = true;
+         }
+         else if(c == ',')
+         {
+            cells.add(finishCell(cell, wasQuoted));
+            cell = new StringBuilder();
+            wasQuoted = false;
+         }
+         else
+         {
+            cell.append(c);
+         }
+      }
+      cells.add(finishCell(cell, wasQuoted));
+      return (cells);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static String finishCell(StringBuilder cell, boolean wasQuoted)
+   {
+      String s = cell.toString();
+      return (!wasQuoted && NULL_TOKEN.equals(s) ? null : s);
+   }
+
+
+
+   /*******************************************************************************
+    ** Split a whole CSV text into records, honoring newlines inside quoted fields.
+    *******************************************************************************/
+   public static List<String> splitRecords(String text)
+   {
+      List<String>  records = new ArrayList<>();
+      StringBuilder current = new StringBuilder();
+      boolean       quoted  = false;
+      for(int i = 0; i < text.length(); i++)
+      {
+         char c = text.charAt(i);
+         if(c == '"')
+         {
+            quoted = !quoted;
+            current.append(c);
+         }
+         else if(c == '\n' && !quoted)
+         {
+            records.add(current.toString());
+            current = new StringBuilder();
+         }
+         else
+         {
+            current.append(c);
+         }
+      }
+      if(current.length() > 0)
+      {
+         records.add(current.toString());
+      }
+      return (records);
+   }
+}

src/main/java/com/kingsrook/qbits/backups/codec/FieldValueCodec.java

@@ -0,0 +1,172 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.codec;
+
+
+import java.io.Serializable;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.Base64;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+
+
+/*******************************************************************************
+ ** The canonical, lossless text encoding of a field value, by QFieldType
+ ** (spec 8.2).  The same canonical text is used by JSONL and CSV, so a value
+ ** means the same thing in both, and by the content hash.
+ **
+ **  STRING, TEXT, HTML, PASSWORD: string
+ **  INTEGER, LONG: number
+ **  DECIMAL: string (BigDecimal.toPlainString), so there is no float rounding
+ **  BOOLEAN: true or false
+ **  DATE: yyyy-MM-dd
+ **  TIME: HH:mm:ss, with .SSSSSS only when there are fractional seconds
+ **  DATE_TIME: an ISO-8601 instant in UTC, with .SSSSSS only when there are fractional seconds
+ **  BLOB: base64
+ *******************************************************************************/
+public final class FieldValueCodec
+{
+   private static final DateTimeFormatter TIME_FORMAT          = DateTimeFormatter.ofPattern("HH:mm:ss");
+   private static final DateTimeFormatter TIME_FORMAT_MICROS   = DateTimeFormatter.ofPattern("HH:mm:ss.SSSSSS");
+   private static final DateTimeFormatter INSTANT_FORMAT       = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'").withZone(ZoneOffset.UTC);
+   private static final DateTimeFormatter INSTANT_FORMAT_MICRO = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'").withZone(ZoneOffset.UTC);
+
+
+
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private FieldValueCodec()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the canonical JSON form of a type is a bare number or boolean (not quoted).
+    *******************************************************************************/
+   public static boolean isBareJson(QFieldType type)
+   {
+      return (type == QFieldType.INTEGER || type == QFieldType.LONG || type == QFieldType.BOOLEAN);
+   }
+
+
+
+   /*******************************************************************************
+    ** The canonical text of a non-null value.
+    *******************************************************************************/
+   public static String toCanonicalText(QFieldType type, Serializable value)
+   {
+      switch(type)
+      {
+         case INTEGER, LONG:
+            return (String.valueOf(ValueUtils.getValueAsLong(value)));
+         case DECIMAL:
+            return (ValueUtils.getValueAsBigDecimal(value).toPlainString());
+         case BOOLEAN:
+            return (String.valueOf(ValueUtils.getValueAsBoolean(value)));
+         case DATE:
+            return (ValueUtils.getValueAsLocalDate(value).toString());
+         case TIME:
+         {
+            LocalTime time = ValueUtils.getValueAsLocalTime(value);
+            return (time.getNano() == 0 ? TIME_FORMAT.format(time) : TIME_FORMAT_MICROS.format(time));
+         }
+         case DATE_TIME:
+         {
+            Instant instant = ValueUtils.getValueAsInstant(value);
+            return (instant.getNano() == 0 ? INSTANT_FORMAT.format(instant) : INSTANT_FORMAT_MICRO.format(instant));
+         }
+         case BLOB:
+            return (Base64.getEncoder().encodeToString(blobBytes(value)));
+         default:
+            return (String.valueOf(value));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static byte[] blobBytes(Serializable value)
+   {
+      if(value instanceof byte[] bytes)
+      {
+         return (bytes);
+      }
+      return (String.valueOf(value).getBytes(java.nio.charset.StandardCharsets.UTF_8));
+   }
+
+
+
+   /*******************************************************************************
+    ** Parse canonical text back to a Java value.
+    *******************************************************************************/
+   public static Serializable fromCanonicalText(QFieldType type, String text)
+   {
+      if(text == null)
+      {
+         return (null);
+      }
+      switch(type)
+      {
+         case INTEGER:
+            return (Integer.valueOf(text));
+         case LONG:
+            return (Long.valueOf(text));
+         case DECIMAL:
+            return (new BigDecimal(text));
+         case BOOLEAN:
+            return (Boolean.valueOf(text));
+         case DATE:
+            return (LocalDate.parse(text));
+         case TIME:
+            return (LocalTime.parse(text));
+         case DATE_TIME:
+            return (Instant.parse(text));
+         case BLOB:
+            return (Base64.getDecoder().decode(text));
+         default:
+            return (text);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Convert a parsed JSON value to a Java value.
+    *******************************************************************************/
+   public static Serializable fromJson(QFieldType type, JsonNode node)
+   {
+      if(node == null || node.isNull())
+      {
+         return (null);
+      }
+      if(type == QFieldType.BOOLEAN)
+      {
+         return (node.asBoolean());
+      }
+      return (fromCanonicalText(type, node.isNumber() ? node.asText() : node.textValue()));
+   }
+}

src/main/java/com/kingsrook/qbits/backups/codec/HashingOutputStream.java

@@ -0,0 +1,147 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.codec;
+
+
+import java.io.FilterOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.HexFormat;
+
+
+/*******************************************************************************
+ ** An output stream that counts and SHA-256 hashes the bytes passing through.
+ ** Closing it closes the stream underneath.  The hash is available after close()
+ ** (or any time, as the hash of the bytes so far).
+ *******************************************************************************/
+public class HashingOutputStream extends FilterOutputStream
+{
+   private final MessageDigest digest;
+   private long                count = 0;
+   private String              hex;
+
+
+
+   /*******************************************************************************
+    ** Constructor.
+    *******************************************************************************/
+   public HashingOutputStream(OutputStream out)
+   {
+      super(out);
+      this.digest = newSha256();
+   }
+
+
+
+   /*******************************************************************************
+    ** A new SHA-256 digest.
+    *******************************************************************************/
+   public static MessageDigest newSha256()
+   {
+      try
+      {
+         return (MessageDigest.getInstance("SHA-256"));
+      }
+      catch(NoSuchAlgorithmException e)
+      {
+         throw (new IllegalStateException("SHA-256 is not available", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Hex of a digest result.
+    *******************************************************************************/
+   public static String hex(byte[] bytes)
+   {
+      return (HexFormat.of().formatHex(bytes));
+   }
+
+
+
+   /*******************************************************************************
+    ** SHA-256 of a byte array, as hex.
+    *******************************************************************************/
+   public static String sha256Hex(byte[] bytes)
+   {
+      return (hex(newSha256().digest(bytes)));
+   }
+
+
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   public void write(int b) throws IOException
+   {
+      out.write(b);
+      digest.update((byte) b);
+      count++;
+   }
+
+
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   public void write(byte[] b, int off, int len) throws IOException
+   {
+      out.write(b, off, len);
+      digest.update(b, off, len);
+      count += len;
+   }
+
+
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   public void close() throws IOException
+   {
+      super.close();
+      if(hex == null)
+      {
+         hex = hex(digest.digest());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The SHA-256 (hex) of everything written; valid after close().
+    *******************************************************************************/
+   public String getSha256()
+   {
+      return (hex);
+   }
+
+
+
+   /*******************************************************************************
+    ** The number of bytes written.
+    *******************************************************************************/
+   public long getCount()
+   {
+      return (count);
+   }
+}

src/main/java/com/kingsrook/qbits/backups/codec/JsonlRowCodec.java

@@ -0,0 +1,200 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.codec;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeSet;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
+/*******************************************************************************
+ ** The JSON Lines row format (spec 8.2): one JSON object per line, keys are field
+ ** names (sorted), values are the raw canonical encodings, never display values.
+ ** The text of a line is canonical, so the same rows always give the same bytes
+ ** (which is what the rowsHash is computed over).
+ *******************************************************************************/
+public class JsonlRowCodec implements RowCodec
+{
+   private static final ObjectMapper MAPPER = new ObjectMapper();
+
+   private final QTableMetaData   table;
+   private final List<String>     fieldNames;
+
+
+
+   /*******************************************************************************
+    ** Constructor.  The fields are the table fields, minus any excluded, sorted by name.
+    *******************************************************************************/
+   public JsonlRowCodec(QTableMetaData table, java.util.Collection<String> excludedFields)
+   {
+      this.table = table;
+      TreeSet<String> names = new TreeSet<>(table.getFields().keySet());
+      if(excludedFields != null)
+      {
+         names.removeAll(excludedFields);
+      }
+      this.fieldNames = new ArrayList<>(names);
+   }
+
+
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   public List<String> getFieldNames()
+   {
+      return (fieldNames);
+   }
+
+
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   public String header()
+   {
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    ** The canonical line (no trailing newline) for a record.
+    *******************************************************************************/
+   @Override
+   public String encode(QRecord record)
+   {
+      StringBuilder sb    = new StringBuilder("{");
+      boolean       first = true;
+      for(String name : fieldNames)
+      {
+         QFieldType type  = table.getField(name).getType();
+         Serializable value = record.getValue(name);
+         if(!first)
+         {
+            sb.append(',');
+         }
+         first = false;
+         appendString(sb, name);
+         sb.append(':');
+         if(value == null)
+         {
+            sb.append("null");
+         }
+         else
+         {
+            String text = FieldValueCodec.toCanonicalText(type, value);
+            if(FieldValueCodec.isBareJson(type))
+            {
+               sb.append(text);
+            }
+            else
+            {
+               appendString(sb, text);
+            }
+         }
+      }
+      return (sb.append('}').toString());
+   }
+
+
+
+   /*******************************************************************************
+    ** Parse one line into a record.  Unknown keys are kept as strings so that
+    ** restore can report them as unmapped.
+    *******************************************************************************/
+   @Override
+   public QRecord decode(String line) throws QException
+   {
+      try
+      {
+         JsonNode              node   = MAPPER.readTree(line);
+         Map<String, Serializable> values = new LinkedHashMap<>();
+         for(var it = node.fields(); it.hasNext(); )
+         {
+            var            entry = it.next();
+            QFieldMetaData field = table.getFields().get(entry.getKey());
+            if(field == null)
+            {
+               values.put(entry.getKey(), entry.getValue().isNull() ? null : entry.getValue().asText());
+            }
+            else
+            {
+               values.put(entry.getKey(), FieldValueCodec.fromJson(field.getType(), entry.getValue()));
+            }
+         }
+         QRecord record = new QRecord();
+         record.setTableName(table.getName());
+         record.setValues(values);
+         return (record);
+      }
+      catch(Exception e)
+      {
+         throw (new QException("Could not read a backup row for table " + table.getName() + ": " + e.getMessage()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Append a JSON string with escapes.  Control characters, quotes and backslashes
+    ** are escaped; everything else (including non-ASCII) is written as is, in UTF-8.
+    *******************************************************************************/
+   static void appendString(StringBuilder sb, String s)
+   {
+      sb.append('"');
+      for(int i = 0; i < s.length(); i++)
+      {
+         char c = s.charAt(i);
+         switch(c)
+         {
+            case '"' -> sb.append("\\\"");
+            case '\\' -> sb.append("\\\\");
+            case '\n' -> sb.append("\\n");
+            case '\r' -> sb.append("\\r");
+            case '\t' -> sb.append("\\t");
+            case '\b' -> sb.append("\\b");
+            case '\f' -> sb.append("\\f");
+            default ->
+            {
+               if(c < 0x20)
+               {
+                  sb.append(String.format("\\u%04x", (int) c));
+               }
+               else
+               {
+                  sb.append(c);
+               }
+            }
+         }
+      }
+      sb.append('"');
+   }
+}

src/main/java/com/kingsrook/qbits/backups/codec/RowCodec.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.codec;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** Encodes a record to one line of text, and decodes it back.
+ *******************************************************************************/
+public interface RowCodec
+{
+
+   /***************************************************************************
+    ** The field names written, in order.
+    ***************************************************************************/
+   List<String> getFieldNames();
+
+
+   /***************************************************************************
+    ** A header line to write at the start of each part, or null.
+    ***************************************************************************/
+   String header();
+
+
+   /***************************************************************************
+    ** One line, without the trailing newline.
+    ***************************************************************************/
+   String encode(QRecord record);
+
+
+   /***************************************************************************
+    ** Read one line back into a record.
+    ***************************************************************************/
+   QRecord decode(String line) throws QException;
+}

src/main/java/com/kingsrook/qbits/backups/codec/TableArtifactWriter.java

@@ -0,0 +1,274 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.codec;
+
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.zip.GZIPOutputStream;
+import com.kingsrook.qbits.backups.storage.BackupStorage;
+import com.kingsrook.qbits.backups.storage.WriteOptions;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
+/*******************************************************************************
+ ** Streams the rows of one table into numbered parts in storage.  Each row goes
+ ** through the row codec, then (per part) into a hash of the raw bytes, gzip,
+ ** the artifact transform (encryption) and a hash of the stored bytes, and out
+ ** through BackupStorage.openWrite.  Memory is O(row); nothing touches a temp disk.
+ **
+ ** Whatever the output format, a canonical JSONL hash of every row (in the order
+ ** written, which is primary key order) is kept: that is the rowsHash a verifier
+ ** recomputes (spec 8.2), independent of compression, encryption and part sizes.
+ *******************************************************************************/
+public class TableArtifactWriter implements Closeable
+{
+   private final BackupStorage     storage;
+   private final String            keyPrefix;
+   private final QTableMetaData    table;
+   private final RowCodec          codec;
+   private final JsonlRowCodec     canonical;
+   private final boolean           csv;
+   private final boolean           gzip;
+   private final long              targetPartBytes;
+   private final long              maxRowsPerPart;
+   private final ArtifactTransform transform;
+   private final String            runKey;
+
+   private final MessageDigest rowsDigest = HashingOutputStream.newSha256();
+   private final List<PartResult> parts   = new ArrayList<>();
+
+   private int                 partNumber = 0;
+   private PartState           current;
+   private long                totalRows  = 0;
+   private String              firstKeyOfTable;
+   private String              lastKeyOfTable;
+
+
+
+   /***************************************************************************
+    ** The outcome of one written part.
+    ***************************************************************************/
+   public record PartResult(int partNumber, String key, long rows, long bytesRaw, long bytesStored, String sha256Raw, String sha256Stored,
+                            String firstKey, String lastKey, boolean encrypted, String format, String compression)
+   {
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static class PartState
+   {
+      String               key;
+      HashingOutputStream  raw;
+      HashingOutputStream  stored;
+      long                 rows;
+      String               firstKey;
+      String               lastKey;
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor.
+    **
+    ** @param keyPrefix the key prefix of the table directory, ending in a slash
+    ** @param format JSONL or CSV
+    ** @param compression GZIP or NONE
+    *******************************************************************************/
+   public TableArtifactWriter(BackupStorage storage, String keyPrefix, QTableMetaData table, Collection<String> excludedFields, String format,
+                              String compression, long targetPartBytes, long maxRowsPerPart, ArtifactTransform transform, String runKey)
+   {
+      this.storage = storage;
+      this.keyPrefix = keyPrefix;
+      this.table = table;
+      this.csv = "CSV".equals(format);
+      this.canonical = new JsonlRowCodec(table, excludedFields);
+      this.codec = csv ? new CsvRowCodec(table, excludedFields) : canonical;
+      this.gzip = "GZIP".equals(compression);
+      this.targetPartBytes = targetPartBytes;
+      this.maxRowsPerPart = maxRowsPerPart;
+      this.transform = transform == null ? ArtifactTransform.IDENTITY : transform;
+      this.runKey = runKey;
+
+      if(!gzip && !"NONE".equals(compression))
+      {
+         throw (new IllegalArgumentException("Unsupported compression: " + compression));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Write one record.
+    *******************************************************************************/
+   public void write(QRecord record) throws IOException
+   {
+      if(current == null)
+      {
+         openPart();
+      }
+
+      String       pk   = String.valueOf(record.getValue(table.getPrimaryKeyField()));
+      byte[]       line = (codec.encode(record) + "\n").getBytes(StandardCharsets.UTF_8);
+      current.raw.write(line);
+      if(current.firstKey == null)
+      {
+         current.firstKey = pk;
+      }
+      current.lastKey = pk;
+      current.rows++;
+      totalRows++;
+      if(firstKeyOfTable == null)
+      {
+         firstKeyOfTable = pk;
+      }
+      lastKeyOfTable = pk;
+
+      rowsDigest.update((canonical.encode(record) + "\n").getBytes(StandardCharsets.UTF_8));
+
+      if(current.raw.getCount() >= targetPartBytes || current.rows >= maxRowsPerPart)
+      {
+         closePart();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Finish: close the last part (writing an empty part if there were no rows at all).
+    *******************************************************************************/
+   public List<PartResult> finish() throws IOException
+   {
+      if(current == null && parts.isEmpty())
+      {
+         openPart();
+      }
+      if(current != null)
+      {
+         closePart();
+      }
+      return (parts);
+   }
+
+
+
+   /*******************************************************************************
+    ** The canonical content hash of all rows written, as sha256:hex.  Valid after finish().
+    *******************************************************************************/
+   public String rowsHash()
+   {
+      return ("sha256:" + HashingOutputStream.hex(rowsDigest.digest()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Total rows written.
+    *******************************************************************************/
+   public long getTotalRows()
+   {
+      return (totalRows);
+   }
+
+
+
+   /*******************************************************************************
+    ** First primary key written.
+    *******************************************************************************/
+   public String getFirstKey()
+   {
+      return (firstKeyOfTable);
+   }
+
+
+
+   /*******************************************************************************
+    ** Last primary key written.
+    *******************************************************************************/
+   public String getLastKey()
+   {
+      return (lastKeyOfTable);
+   }
+
+
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   public void close() throws IOException
+   {
+      if(current != null)
+      {
+         // an abandoned part is never committed to storage
+         current.raw.close();
+         current = null;
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void openPart() throws IOException
+   {
+      partNumber++;
+      current = new PartState();
+      String extension = csv ? "csv" : "jsonl";
+      current.key = keyPrefix + String.format("part-%05d.%s%s%s", partNumber, extension, gzip ? ".gz" : "", transform.keySuffix());
+
+      WriteOptions options = WriteOptions.create().withContentType("application/octet-stream")
+         .withMetadata("qqq-backup-run", runKey)
+         .withMetadata("qqq-backup-table", table.getName());
+      OutputStream toStorage = storage.openWrite(current.key, options);
+      current.stored = new HashingOutputStream(toStorage);
+      OutputStream transformed = transform.wrap(current.stored);
+      OutputStream compressed  = gzip ? new GZIPOutputStream(transformed) : transformed;
+      current.raw = new HashingOutputStream(compressed);
+
+      String header = codec.header();
+      if(header != null)
+      {
+         current.raw.write((header + "\n").getBytes(StandardCharsets.UTF_8));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void closePart() throws IOException
+   {
+      current.raw.close();
+      parts.add(new PartResult(partNumber, current.key, current.rows, current.raw.getCount(), current.stored.getCount(),
+         current.raw.getSha256(), current.stored.getSha256(), current.firstKey, current.lastKey, transform.encrypts(),
+         csv ? "CSV" : "JSONL", gzip ? "GZIP" : "NONE"));
+      current = null;
+   }
+}

src/main/java/com/kingsrook/qbits/backups/files/FileCopier.java

@@ -0,0 +1,223 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.files;
+
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Optional;
+import java.util.function.Function;
+import com.kingsrook.qbits.backups.codec.ArtifactTransform;
+import com.kingsrook.qbits.backups.codec.HashingOutputStream;
+import com.kingsrook.qbits.backups.storage.BackupStorage;
+import com.kingsrook.qbits.backups.storage.StoredObjectInfo;
+import com.kingsrook.qbits.backups.storage.WriteOptions;
+
+
+/*******************************************************************************
+ ** Makes sure each file that a record references is in the content-addressed
+ ** store (spec 8.5), and says what it did.
+ **
+ ** For each reference: head the source (missing is reported, never silently
+ ** skipped); find its SHA-256 (known from the reference provider, else the object
+ ** is read once and hashed); if the store already has that hash the file is
+ ** DEDUPLICATED and nothing is copied; otherwise it is copied (server-side when
+ ** the source and the destination are the same storage and the plan does not
+ ** encrypt, else streamed through the transform) and the hash observed while
+ ** copying must equal the hash the key was derived from.  If the source changed
+ ** in between, the file is hashed again and copied once more.
+ *******************************************************************************/
+public class FileCopier
+{
+   public static final String COPIED         = "COPIED";
+   public static final String DEDUPLICATED   = "DEDUPLICATED";
+   public static final String MISSING        = "MISSING";
+   public static final String REFERENCE_ONLY = "REFERENCE_ONLY";
+
+   private final Function<String, BackupStorage> sourceResolver;
+   private final String                          destinationAlias;
+   private final BackupStorage                   destination;
+   private final String                          casPrefix;
+   private final ArtifactTransform               transform;
+   private final FileStoreRegistry               registry;
+   private final boolean                         referenceOnly;
+   private final String                          runKey;
+
+
+
+   /***************************************************************************
+    ** The catalog side of the file store: which hashes are already stored.
+    ***************************************************************************/
+   public interface FileStoreRegistry
+   {
+      /***************************************************************************
+       ** Whether the store knows this hash (and the object is expected to be there).
+       ***************************************************************************/
+      boolean isKnown(String sha256);
+
+
+      /***************************************************************************
+       ** Record a newly stored file.
+       ***************************************************************************/
+      void register(String sha256, long sizeBytes, String contentType, String casKey, boolean encrypted);
+   }
+
+
+
+   /***************************************************************************
+    ** One line of the per-run file index.
+    **
+    ** @param tableName the referencing table
+    ** @param recordKey the referencing record primary key
+    ** @param field the referencing field
+    ** @param sourceBucketAlias where the file was
+    ** @param sourceKey the source key
+    ** @param sha256 content hash (null when missing or reference-only without a known hash)
+    ** @param size size in bytes (-1 when unknown)
+    ** @param status COPIED, DEDUPLICATED, MISSING or REFERENCE_ONLY
+    ** @param casKey the content-addressed key in the destination, if stored
+    ***************************************************************************/
+   public record IndexEntry(String tableName, String recordKey, String field, String sourceBucketAlias, String sourceKey,
+                            String sha256, long size, String status, String casKey)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor.
+    **
+    ** @param casPrefix prefix of the store, ending in a slash, e.g. qqq-backups/app/_files/sha256/
+    *******************************************************************************/
+   public FileCopier(Function<String, BackupStorage> sourceResolver, String destinationAlias, BackupStorage destination, String casPrefix,
+                     ArtifactTransform transform, FileStoreRegistry registry, boolean referenceOnly, String runKey)
+   {
+      this.sourceResolver = sourceResolver;
+      this.destinationAlias = destinationAlias;
+      this.destination = destination;
+      this.casPrefix = casPrefix;
+      this.transform = transform == null ? ArtifactTransform.IDENTITY : transform;
+      this.registry = registry;
+      this.referenceOnly = referenceOnly;
+      this.runKey = runKey;
+   }
+
+
+
+   /*******************************************************************************
+    ** The content-addressed key for a hash: prefix/ab/cd/sha256.
+    *******************************************************************************/
+   public static String casKey(String casPrefix, String sha256)
+   {
+      return (casPrefix + sha256.substring(0, 2) + "/" + sha256.substring(2, 4) + "/" + sha256);
+   }
+
+
+
+   /*******************************************************************************
+    ** Process one reference.
+    *******************************************************************************/
+   public IndexEntry process(FileReference ref) throws IOException
+   {
+      BackupStorage source = sourceResolver.apply(ref.bucketAlias());
+      Optional<StoredObjectInfo> head = source == null ? Optional.empty() : source.head(ref.key());
+      if(head.isEmpty())
+      {
+         return (entry(ref, null, -1, MISSING, null));
+      }
+      long size = head.get().size();
+
+      if(referenceOnly)
+      {
+         return (entry(ref, ref.knownSha256(), size, REFERENCE_ONLY, null));
+      }
+
+      String sha = ref.knownSha256() != null ? ref.knownSha256() : hashOf(source, ref.key());
+      for(int attempt = 0; attempt < 2; attempt++)
+      {
+         String key = casKey(casPrefix, sha);
+         if(registry.isKnown(sha) && destination.head(key).isPresent())
+         {
+            return (entry(ref, sha, size, DEDUPLICATED, key));
+         }
+
+         String observed = copy(source, ref, key);
+         if(observed == null || observed.equals(sha))
+         {
+            registry.register(sha, size, ref.contentType(), key, transform.encrypts());
+            return (entry(ref, sha, size, COPIED, key));
+         }
+         // the source changed while it was being copied: the object we wrote is under the wrong key
+         destination.deletePermanent(key);
+         sha = observed;
+      }
+      throw (new IOException("The file " + ref.key() + " kept changing while it was being copied"));
+   }
+
+
+
+   /*******************************************************************************
+    ** Copy the file to the store; returns the hash observed while streaming, or null
+    ** for a server-side copy (which cannot observe the bytes).
+    *******************************************************************************/
+   private String copy(BackupStorage source, FileReference ref, String key) throws IOException
+   {
+      if(source == destination && !transform.encrypts())
+      {
+         destination.copy(ref.key(), key);
+         return (null);
+      }
+
+      WriteOptions options = WriteOptions.create().withContentType(ref.contentType() == null ? "application/octet-stream" : ref.contentType())
+         .withMetadata("qqq-backup-run", runKey);
+      try(InputStream in = source.openRead(ref.key());
+          HashingOutputStream plain = new HashingOutputStream(transform.wrap(destination.openWrite(key, options))))
+      {
+         in.transferTo(plain);
+         plain.flush();
+         plain.close();
+         return (plain.getSha256());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Read an object once, to find its SHA-256.
+    *******************************************************************************/
+   private String hashOf(BackupStorage source, String key) throws IOException
+   {
+      try(InputStream in = source.openRead(key);
+          HashingOutputStream sink = new HashingOutputStream(OutputStream.nullOutputStream()))
+      {
+         in.transferTo(sink);
+         sink.close();
+         return (sink.getSha256());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private IndexEntry entry(FileReference ref, String sha, long size, String status, String casKey)
+   {
+      return (new IndexEntry(ref.tableName(), ref.recordKey(), ref.field(), ref.bucketAlias(), ref.key(), sha, size, status, casKey));
+   }
+}

src/main/java/com/kingsrook/qbits/backups/files/FileReference.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.files;
+
+
+/*******************************************************************************
+ ** A reference from a record field to a file in object storage.
+ **
+ ** @param tableName the referencing table
+ ** @param recordKey the referencing record primary key (as text)
+ ** @param field the referencing field
+ ** @param bucketAlias the storage bucket alias the file lives in
+ ** @param key the object key
+ ** @param knownSha256 the content hash when the reference provider knows it (a content-addressed key or a stored checksum), else null
+ ** @param contentType the content type when known, else null
+ *******************************************************************************/
+public record FileReference(String tableName, String recordKey, String field, String bucketAlias, String key, String knownSha256, String contentType)
+{
+}

src/main/java/com/kingsrook/qbits/backups/files/FileReferenceProvider.java

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.files;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qbits.backups.scope.BackupTableMetaData;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
+/*******************************************************************************
+ ** Finds the files a record references.  Apps declare their file references once
+ ** (the object-storage qbit StorageReferenceProvider registrations, which an
+ ** adapter exposes here); the default provider reads the fileReferenceFields of
+ ** the table BackupTableMetaData.
+ *******************************************************************************/
+public interface FileReferenceProvider
+{
+
+   /***************************************************************************
+    ** The files this record references (empty when none).
+    ***************************************************************************/
+   List<FileReference> referencesIn(QTableMetaData table, QRecord record);
+
+
+
+   /***************************************************************************
+    ** Reads BackupTableMetaData.fileReferenceFields: each named field holds a key
+    ** in the table fileBucketAlias bucket.
+    ***************************************************************************/
+   class FromTableMetaData implements FileReferenceProvider
+   {
+      /*******************************************************************************
+       ** {@inheritDoc}
+       *******************************************************************************/
+      @Override
+      public List<FileReference> referencesIn(QTableMetaData table, QRecord record)
+      {
+         BackupTableMetaData meta = BackupTableMetaData.of(table);
+         if(meta == null || meta.getFileReferenceFields().isEmpty())
+         {
+            return (List.of());
+         }
+         List<FileReference> rs = new ArrayList<>();
+         for(String field : meta.getFileReferenceFields())
+         {
+            String key = record.getValueString(field);
+            if(key != null && !key.isBlank())
+            {
+               rs.add(new FileReference(table.getName(), String.valueOf(record.getValue(table.getPrimaryKeyField())), field, meta.getFileBucketAlias(), key, null, null));
+            }
+         }
+         return (rs);
+      }
+   }
+}

src/main/java/com/kingsrook/qbits/backups/manifest/BackupManifest.java

@@ -0,0 +1,219 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.manifest;
+
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import java.util.Map;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+
+
+/*******************************************************************************
+ ** The self-describing manifest of a run (spec 8.7), written LAST: it is the
+ ** commit marker.  A run without a manifest did not complete.  It holds no
+ ** secrets: only key references and fingerprints, never key material.
+ **
+ ** Unknown properties are ignored when reading, so a newer manifest is still
+ ** readable by an older qbit (and by the offline tools).
+ *******************************************************************************/
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public record BackupManifest(
+   int manifestVersion,
+   String runKey,
+   PlanInfo plan,
+   InstanceInfo instance,
+   String startedAt,
+   String finishedAt,
+   String requestedBy,
+   String securityScope,
+   ConsistencyInfo consistency,
+   EncryptionInfo encryption,
+   List<TableEntry> tables,
+   List<String> restoreOrder,
+   FilesInfo files,
+   List<String> warnings,
+   Totals totals)
+{
+   public static final int CURRENT_VERSION = 1;
+
+   private static final ObjectMapper MAPPER = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT);
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @JsonIgnoreProperties(ignoreUnknown = true)
+   @JsonInclude(JsonInclude.Include.NON_NULL)
+   public record PlanInfo(String planKey, String name, Map<String, java.io.Serializable> settings)
+   {
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @JsonIgnoreProperties(ignoreUnknown = true)
+   @JsonInclude(JsonInclude.Include.NON_NULL)
+   public record InstanceInfo(String name, String appVersion, String qqqVersion, List<Map<String, String>> qbits)
+   {
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @JsonIgnoreProperties(ignoreUnknown = true)
+   @JsonInclude(JsonInclude.Include.NON_NULL)
+   public record ConsistencyInfo(String achieved, List<Map<String, String>> perBackend)
+   {
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @JsonIgnoreProperties(ignoreUnknown = true)
+   @JsonInclude(JsonInclude.Include.NON_NULL)
+   public record EncryptionInfo(String mode, String scheme, String keyRef, String keyFingerprint)
+   {
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @JsonIgnoreProperties(ignoreUnknown = true)
+   @JsonInclude(JsonInclude.Include.NON_NULL)
+   public record ExcludedField(String name, String reason)
+   {
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @JsonIgnoreProperties(ignoreUnknown = true)
+   @JsonInclude(JsonInclude.Include.NON_NULL)
+   public record Part(String key, long rows, long bytesRaw, long bytesStored, String sha256Stored, String sha256Raw, String format, String compression, boolean encrypted)
+   {
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @JsonIgnoreProperties(ignoreUnknown = true)
+   @JsonInclude(JsonInclude.Include.NON_NULL)
+   public record TableEntry(
+      String name,
+      String role,
+      String backend,
+      String primaryKey,
+      String fingerprint,
+      Map<String, Object> schema,
+      String filter,
+      List<ExcludedField> excludedFields,
+      long rowCount,
+      String rowsHash,
+      Map<String, String> keyRange,
+      List<Part> parts,
+      String consistency)
+   {
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @JsonIgnoreProperties(ignoreUnknown = true)
+   @JsonInclude(JsonInclude.Include.NON_NULL)
+   public record FilesInfo(String policy, List<String> indexKeys, long objects, long bytes, long missing)
+   {
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @JsonIgnoreProperties(ignoreUnknown = true)
+   @JsonInclude(JsonInclude.Include.NON_NULL)
+   public record Totals(int tables, long rows, long bytesStored)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** The manifest as pretty JSON bytes.
+    *******************************************************************************/
+   public byte[] toJsonBytes()
+   {
+      try
+      {
+         return (MAPPER.writeValueAsBytes(this));
+      }
+      catch(IOException e)
+      {
+         throw (new IllegalStateException("Could not write the manifest", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Read a manifest.
+    *******************************************************************************/
+   public static BackupManifest read(InputStream in) throws IOException
+   {
+      return (MAPPER.readValue(in, BackupManifest.class));
+   }
+
+
+
+   /*******************************************************************************
+    ** Read a manifest from bytes.
+    *******************************************************************************/
+   public static BackupManifest read(byte[] bytes) throws IOException
+   {
+      return (MAPPER.readValue(bytes, BackupManifest.class));
+   }
+
+
+
+   /*******************************************************************************
+    ** The table entry with a name, or null.
+    *******************************************************************************/
+   public TableEntry table(String name)
+   {
+      return (tables == null ? null : tables.stream().filter(t -> t.name().equals(name)).findFirst().orElse(null));
+   }
+}

src/main/java/com/kingsrook/qbits/backups/manifest/TableFingerprints.java

@@ -0,0 +1,184 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.manifest;
+
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import com.kingsrook.qbits.backups.codec.HashingOutputStream;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+
+
+/*******************************************************************************
+ ** A stable schema descriptor and SHA-256 fingerprint of a table (spec 5.5).
+ **
+ ** This is an internal stand-in for the QQQ core TableSchemaFingerprint (core PR
+ ** feature/schema-fingerprint).  When that ships in the next 4.1.x RC, this class
+ ** delegates to it and the descriptor below is kept only for reading old manifests.
+ ** Labels, adornments and help text are excluded, so cosmetic changes do not count.
+ *******************************************************************************/
+public final class TableFingerprints
+{
+
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private TableFingerprints()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** The descriptor of a table (sorted fields; plain maps and lists, so it is also
+    ** what goes in the manifest schema).
+    *******************************************************************************/
+   public static Map<String, Object> descriptor(QTableMetaData table, String backendType)
+   {
+      Map<String, Object> d = new LinkedHashMap<>();
+      d.put("name", table.getName());
+      d.put("primaryKey", table.getPrimaryKeyField());
+      d.put("backendType", backendType);
+
+      Map<String, Object> fields = new TreeMap<>();
+      for(QFieldMetaData field : new TreeMap<>(table.getFields()).values())
+      {
+         Map<String, Object> f = new LinkedHashMap<>();
+         f.put("type", field.getType().name());
+         f.put("required", field.getIsRequired());
+         f.put("maxLength", field.getMaxLength());
+         f.put("pvs", field.getPossibleValueSourceName());
+         f.put("heavy", field.getIsHeavy());
+         f.put("sensitive", field.getIsSensitive());
+         fields.put(field.getName(), f);
+      }
+      d.put("fields", fields);
+
+      List<String> uniqueKeys = new ArrayList<>();
+      if(table.getUniqueKeys() != null)
+      {
+         for(UniqueKey key : table.getUniqueKeys())
+         {
+            uniqueKeys.add(String.join("+", key.getFieldNames().stream().sorted().toList()));
+         }
+      }
+      uniqueKeys.sort(String::compareTo);
+      d.put("uniqueKeys", uniqueKeys);
+
+      List<String> associations = new ArrayList<>();
+      if(table.getAssociations() != null)
+      {
+         for(Association association : table.getAssociations())
+         {
+            associations.add(association.getName() + "->" + association.getAssociatedTableName());
+         }
+      }
+      associations.sort(String::compareTo);
+      d.put("associations", associations);
+      return (d);
+   }
+
+
+
+   /*******************************************************************************
+    ** The fingerprint of a table: sha256:hex of the canonical descriptor.
+    *******************************************************************************/
+   public static String of(QTableMetaData table, String backendType)
+   {
+      return (fingerprintOf(descriptor(table, backendType)));
+   }
+
+
+
+   /*******************************************************************************
+    ** The fingerprint of a descriptor (maps sorted by key at every level, no whitespace).
+    *******************************************************************************/
+   public static String fingerprintOf(Map<String, Object> descriptor)
+   {
+      StringBuilder sb = new StringBuilder();
+      canonical(descriptor, sb);
+      return ("sha256:" + HashingOutputStream.sha256Hex(sb.toString().getBytes(StandardCharsets.UTF_8)));
+   }
+
+
+
+   /*******************************************************************************
+    ** The fingerprint of a whole instance's tables: a hash of the table fingerprints.
+    *******************************************************************************/
+   public static String combine(List<String> tableFingerprints)
+   {
+      return ("sha256:" + HashingOutputStream.sha256Hex(String.join("\n", tableFingerprints).getBytes(StandardCharsets.UTF_8)));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("unchecked")
+   private static void canonical(Object o, StringBuilder sb)
+   {
+      if(o == null)
+      {
+         sb.append("null");
+      }
+      else if(o instanceof Map<?, ?> map)
+      {
+         sb.append('{');
+         boolean first = true;
+         for(Map.Entry<String, Object> e : new TreeMap<>((Map<String, Object>) map).entrySet())
+         {
+            if(!first)
+            {
+               sb.append(',');
+            }
+            first = false;
+            sb.append('"').append(e.getKey()).append("\":");
+            canonical(e.getValue(), sb);
+         }
+         sb.append('}');
+      }
+      else if(o instanceof List<?> list)
+      {
+         sb.append('[');
+         for(int i = 0; i < list.size(); i++)
+         {
+            if(i > 0)
+            {
+               sb.append(',');
+            }
+            canonical(list.get(i), sb);
+         }
+         sb.append(']');
+      }
+      else if(o instanceof Number || o instanceof Boolean)
+      {
+         sb.append(o);
+      }
+      else
+      {
+         sb.append('"').append(String.valueOf(o).replace("\\", "\\\\").replace("\"", "\\\"")).append('"');
+      }
+   }
+}

src/main/java/com/kingsrook/qbits/backups/model/BackupsTables.java

@@ -0,0 +1,572 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.model;
+
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qbits.backups.BackupsQBitConfig;
+import com.kingsrook.qbits.backups.permissions.BackupsPermissionChecker;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ ** The qbit tables (design spec section 4).  All are standard visible QQQ
+ ** tables, prefixed backups_, in lowercase snake_case.  Tables written by
+ ** processes are read-only in the UI.  No table ever holds a key, password or
+ ** credential: only references such as a key name or a bucket alias.
+ *******************************************************************************/
+public final class BackupsTables
+{
+   public static final String PLAN              = "backups_plan";
+   public static final String PLAN_SCOPE        = "backups_plan_scope";
+   public static final String RUN               = "backups_run";
+   public static final String ARTIFACT          = "backups_artifact";
+   public static final String VERIFICATION      = "backups_verification";
+   public static final String RESTORE_RUN       = "backups_restore_run";
+   public static final String RESTORE_RUN_TABLE = "backups_restore_run_table";
+   public static final String FILE_OBJECT       = "backups_file_object";
+   public static final String ID_MAP            = "backups_id_map";
+
+   /** Tables that only processes write (read-only in the UI; no delete). */
+   public static final List<String> PROCESS_WRITTEN = List.of(RUN, ARTIFACT, VERIFICATION, RESTORE_RUN, RESTORE_RUN_TABLE, FILE_OBJECT, ID_MAP);
+
+   /** The tables shown in the nav (sub-tables appear as relations on their parents). */
+   public static final List<String> NAV_TABLES = List.of(PLAN, RUN, RESTORE_RUN, VERIFICATION, FILE_OBJECT);
+
+   public static final List<String> ALL = List.of(PLAN, PLAN_SCOPE, RUN, ARTIFACT, VERIFICATION, RESTORE_RUN, RESTORE_RUN_TABLE, FILE_OBJECT, ID_MAP);
+
+   public static final String PVS_PREFIX = "backups_";
+
+   private static final Map<String, List<String>> ENUMS = new LinkedHashMap<>();
+
+   static
+   {
+      ENUMS.put("scope_type", List.of("INSTANCE", "APPS", "QBITS", "TABLES"));
+      ENUMS.put("table_role", List.of("DATA", "AUDIT", "DERIVED", "EPHEMERAL", "SECRET", "BACKUPS", "EXTERNAL"));
+      ENUMS.put("scope_action", List.of("INCLUDE", "EXCLUDE"));
+      ENUMS.put("selector_type", List.of("TABLE", "APP", "QBIT", "ROLE"));
+      ENUMS.put("format", List.of("JSONL", "CSV", "JSONL_AND_CSV", "PARQUET"));
+      ENUMS.put("compression", List.of("GZIP", "ZSTD", "NONE"));
+      ENUMS.put("encryption_mode", List.of("NONE", "CLIENT_AES256_GCM"));
+      ENUMS.put("sensitive_policy", List.of("EXCLUDE", "INCLUDE_ENCRYPTED_ONLY", "INCLUDE"));
+      ENUMS.put("heavy_policy", List.of("INLINE", "SEPARATE_OBJECTS", "EXCLUDE"));
+      ENUMS.put("file_policy", List.of("COPY", "REFERENCE_ONLY"));
+      ENUMS.put("file_missing_policy", List.of("WARN", "FAIL"));
+      ENUMS.put("consistency", List.of("SNAPSHOT_REQUIRED", "SNAPSHOT_PREFERRED", "PER_TABLE"));
+      ENUMS.put("consistency_achieved", List.of("INSTANCE_SNAPSHOT", "BACKEND_SNAPSHOT", "PER_TABLE", "BEST_EFFORT"));
+      ENUMS.put("incremental_mode", List.of("FULL", "CHANGED_SINCE_LAST"));
+      ENUMS.put("native_dump", List.of("NONE", "PG_DUMP_CUSTOM"));
+      ENUMS.put("lock_mode", List.of("NONE", "GOVERNANCE", "COMPLIANCE"));
+      ENUMS.put("sla_status", List.of("OK", "LATE", "FAILING", "MISSING_IN_STORAGE", "UNVERIFIED", "ABANDONED", "OWNER_INVALID", "UNKNOWN"));
+      ENUMS.put("run_trigger", List.of("SCHEDULED", "MANUAL", "PRE_RESTORE", "RESUME", "IMPORTED"));
+      ENUMS.put("run_status", List.of("QUEUED", "RUNNING", "SUCCEEDED", "SUCCEEDED_WITH_WARNINGS", "FAILED", "TIMED_OUT", "CANCELLED", "ABANDONED"));
+      ENUMS.put("retention_class", List.of("DAILY", "WEEKLY", "MONTHLY", "YEARLY", "NONE"));
+      ENUMS.put("verification_status", List.of("UNVERIFIED", "STORAGE_OK", "CONTENT_OK", "TEST_RESTORE_OK", "FAILED"));
+      ENUMS.put("artifact_type", List.of("TABLE_DATA", "TABLE_BLOBS", "FILE_OBJECTS_INDEX", "NATIVE_DUMP", "MANIFEST", "CHECKSUMS"));
+      ENUMS.put("artifact_status", List.of("WRITING", "COMPLETE", "FAILED"));
+      ENUMS.put("verify_level", List.of("STORAGE", "CONTENT", "TEST_RESTORE"));
+      ENUMS.put("verify_result", List.of("PASSED", "PASSED_WITH_WARNINGS", "FAILED", "RUNNING", "ERROR"));
+      ENUMS.put("restore_status", List.of("PREVIEWED", "QUEUED", "RUNNING", "SUCCEEDED", "SUCCEEDED_WITH_WARNINGS", "FAILED", "CANCELLED"));
+      ENUMS.put("restore_mode", List.of("INSERT_NEW", "UPSERT", "REPLACE"));
+      ENUMS.put("id_strategy", List.of("PRESERVE", "REMAP_NEW_IDS"));
+      ENUMS.put("schema_status", List.of("MATCH", "COMPATIBLE", "MISMATCH"));
+   }
+
+
+
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private BackupsTables()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** The name of the possible value source for an enum key.
+    *******************************************************************************/
+   public static String pvs(String enumKey)
+   {
+      return (PVS_PREFIX + enumKey);
+   }
+
+
+
+   /*******************************************************************************
+    ** The values of an enum, by key.
+    *******************************************************************************/
+   public static List<String> enumValues(String enumKey)
+   {
+      return (ENUMS.get(enumKey));
+   }
+
+
+
+   /*******************************************************************************
+    ** Permission rules shared by every qbit table, app, widget and process.
+    *******************************************************************************/
+   public static QPermissionRules permissionRules()
+   {
+      return (new QPermissionRules()
+         .withLevel(PermissionLevel.HAS_ACCESS_PERMISSION)
+         .withPermissionBaseName(BackupsQBitConfig.PERMISSION_BASE_NAME)
+         .withCustomPermissionChecker(new QCodeReference(BackupsPermissionChecker.class)));
+   }
+
+
+
+   /*******************************************************************************
+    ** The possible-value sources: enums and table references.
+    *******************************************************************************/
+   public static List<QPossibleValueSource> possibleValueSources()
+   {
+      List<QPossibleValueSource> rs = new ArrayList<>();
+      for(Map.Entry<String, List<String>> entry : ENUMS.entrySet())
+      {
+         List<QPossibleValue<?>> values = new ArrayList<>();
+         for(String value : entry.getValue())
+         {
+            values.add(new QPossibleValue<>(value, StringUtils.allCapsToMixedCase(value)));
+         }
+         rs.add(new QPossibleValueSource().withName(pvs(entry.getKey())).withType(QPossibleValueSourceType.ENUM).withEnumValues(values));
+      }
+      rs.add(QPossibleValueSource.newForTable(PLAN));
+      rs.add(QPossibleValueSource.newForTable(RUN));
+      rs.add(QPossibleValueSource.newForTable(RESTORE_RUN));
+      return (rs);
+   }
+
+
+
+   /*******************************************************************************
+    ** The joins that back the parent-to-child associations (child lists on parents).
+    *******************************************************************************/
+   public static List<QJoinMetaData> joins()
+   {
+      return (List.of(
+         oneToMany("backupsPlanScopeJoinPlan", PLAN, PLAN_SCOPE, "backupPlanId"),
+         oneToMany("backupsArtifactJoinRun", RUN, ARTIFACT, "backupRunId"),
+         oneToMany("backupsVerificationJoinRun", RUN, VERIFICATION, "backupRunId"),
+         oneToMany("backupsRestoreRunJoinRun", RUN, RESTORE_RUN, "sourceBackupRunId"),
+         oneToMany("backupsRunJoinPlan", PLAN, RUN, "backupPlanId"),
+         oneToMany("backupsRestoreRunTableJoinRestoreRun", RESTORE_RUN, RESTORE_RUN_TABLE, "backupRestoreRunId")));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static QJoinMetaData oneToMany(String name, String parent, String child, String childField)
+   {
+      return (new QJoinMetaData().withName(name).withType(JoinType.ONE_TO_MANY).withLeftTable(parent).withRightTable(child).withJoinOn(new JoinOn("id", childField)));
+   }
+
+
+
+   /*******************************************************************************
+    ** All tables.
+    *******************************************************************************/
+   public static List<QTableMetaData> tables(BackupsQBitConfig config)
+   {
+      return (List.of(plan(config), planScope(config), run(config), artifact(config), verification(config),
+         restoreRun(config), restoreRunTable(config), fileObject(config), idMap(config)));
+   }
+
+
+
+   /*******************************************************************************
+    ** A table with the common id/createDate/modifyDate fields.
+    *******************************************************************************/
+   private static QTableMetaData base(BackupsQBitConfig config, String name, String label, String icon)
+   {
+      return (new QTableMetaData()
+         .withName(name)
+         .withLabel(label)
+         .withIcon(new QIcon(icon))
+         .withBackendName(config.getBackendName())
+         .withPrimaryKeyField("id")
+         .withPermissionRules(permissionRules())
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
+         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)));
+   }
+
+
+
+   /*******************************************************************************
+    ** A read-only field.
+    *******************************************************************************/
+   private static QFieldMetaData ro(String name, QFieldType type)
+   {
+      return (new QFieldMetaData(name, type).withIsEditable(false));
+   }
+
+
+
+   /*******************************************************************************
+    ** A read-only field backed by an enum possible value source.
+    *******************************************************************************/
+   private static QFieldMetaData roEnum(String name, String enumKey)
+   {
+      return (ro(name, QFieldType.STRING).withPossibleValueSourceName(pvs(enumKey)));
+   }
+
+
+
+   /*******************************************************************************
+    ** An editable enum field with a default.
+    *******************************************************************************/
+   private static QFieldMetaData enumField(String name, String enumKey, String defaultValue)
+   {
+      QFieldMetaData field = new QFieldMetaData(name, QFieldType.STRING).withPossibleValueSourceName(pvs(enumKey));
+      if(defaultValue != null)
+      {
+         field.withDefaultValue(defaultValue);
+      }
+      return (field);
+   }
+
+
+
+   /*******************************************************************************
+    ** A read-only integer count.
+    *******************************************************************************/
+   private static QFieldMetaData roInt(String name)
+   {
+      return (ro(name, QFieldType.INTEGER));
+   }
+
+
+
+   /*******************************************************************************
+    ** A read-only long (byte counts, millis).
+    *******************************************************************************/
+   private static QFieldMetaData roLong(String name)
+   {
+      return (ro(name, QFieldType.LONG));
+   }
+
+
+
+   /*******************************************************************************
+    ** backups_plan: what, how, where, when and how long (spec 4.1).
+    *******************************************************************************/
+   static QTableMetaData plan(BackupsQBitConfig config)
+   {
+      return (base(config, PLAN, "Backup Plan", "backup")
+         .withRecordLabelFormatAndFields("%s", "name")
+         .withUniqueKey(new UniqueKey("name"))
+         .withUniqueKey(new UniqueKey("planKey"))
+         .withAssociation(new Association().withName("scopes").withAssociatedTableName(PLAN_SCOPE).withJoinName("backupsPlanScopeJoinPlan"))
+         .withField(new QFieldMetaData("name", QFieldType.STRING).withIsRequired(true).withMaxLength(100))
+         .withField(new QFieldMetaData("description", QFieldType.TEXT))
+         .withField(new QFieldMetaData("planKey", QFieldType.STRING).withIsRequired(true).withMaxLength(60))
+         .withField(new QFieldMetaData("isActive", QFieldType.BOOLEAN).withDefaultValue(true))
+         .withField(enumField("scopeType", "scope_type", "INSTANCE").withIsRequired(true))
+         .withField(new QFieldMetaData("includeRoles", QFieldType.STRING).withDefaultValue("DATA,AUDIT").withMaxLength(100))
+         .withField(new QFieldMetaData("storageBucketAlias", QFieldType.STRING).withIsRequired(true).withMaxLength(100))
+         .withField(new QFieldMetaData("pruneBucketAlias", QFieldType.STRING).withMaxLength(100))
+         .withField(new QFieldMetaData("prefix", QFieldType.STRING).withMaxLength(255))
+         .withField(new QFieldMetaData("critical", QFieldType.BOOLEAN).withDefaultValue(false))
+         .withField(enumField("format", "format", "JSONL"))
+         .withField(enumField("compression", "compression", "GZIP"))
+         .withField(enumField("encryptionMode", "encryption_mode", "NONE"))
+         .withField(new QFieldMetaData("encryptionKeyRef", QFieldType.STRING).withMaxLength(100))
+         .withField(enumField("sensitiveFieldPolicy", "sensitive_policy", "EXCLUDE"))
+         .withField(enumField("heavyFieldPolicy", "heavy_policy", "INLINE"))
+         .withField(enumField("fileObjectPolicy", "file_policy", "COPY"))
+         .withField(enumField("fileMissingPolicy", "file_missing_policy", "WARN"))
+         .withField(enumField("consistency", "consistency", "SNAPSHOT_PREFERRED"))
+         .withField(new QFieldMetaData("useReadReplica", QFieldType.BOOLEAN).withDefaultValue(false))
+         .withField(enumField("incrementalMode", "incremental_mode", "FULL"))
+         .withField(enumField("nativeDump", "native_dump", "NONE"))
+         .withField(new QFieldMetaData("cronExpression", QFieldType.STRING).withMaxLength(100))
+         .withField(new QFieldMetaData("cronTimeZoneId", QFieldType.STRING).withMaxLength(60))
+         .withField(roInt("scheduledJobId"))
+         .withField(new QFieldMetaData("slaHours", QFieldType.INTEGER).withDefaultValue(26))
+         .withField(new QFieldMetaData("verifySlaDays", QFieldType.INTEGER).withDefaultValue(8))
+         .withField(new QFieldMetaData("verifyCronExpression", QFieldType.STRING).withMaxLength(100))
+         .withField(enumField("verifyLevel", "verify_level", "CONTENT"))
+         .withField(new QFieldMetaData("retainDaily", QFieldType.INTEGER).withDefaultValue(7))
+         .withField(new QFieldMetaData("retainWeekly", QFieldType.INTEGER).withDefaultValue(4))
+         .withField(new QFieldMetaData("retainMonthly", QFieldType.INTEGER).withDefaultValue(12))
+         .withField(new QFieldMetaData("retainYearly", QFieldType.INTEGER).withDefaultValue(0))
+         .withField(new QFieldMetaData("retainMinimum", QFieldType.INTEGER).withDefaultValue(3))
+         .withField(enumField("objectLockMode", "lock_mode", "NONE"))
+         .withField(new QFieldMetaData("objectLockDays", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("maxDurationMinutes", QFieldType.INTEGER).withDefaultValue(240))
+         .withField(new QFieldMetaData("pageSize", QFieldType.INTEGER).withDefaultValue(5000))
+         .withField(new QFieldMetaData("targetPartBytes", QFieldType.LONG).withDefaultValue(268435456L))
+         .withField(new QFieldMetaData("ownerUserId", QFieldType.STRING).withMaxLength(100))
+         .withField(ro("lastRunId", QFieldType.INTEGER).withPossibleValueSourceName(RUN))
+         .withField(ro("lastSuccessRunId", QFieldType.INTEGER).withPossibleValueSourceName(RUN))
+         .withField(ro("lastSuccessDate", QFieldType.DATE_TIME))
+         .withField(ro("lastVerifiedRunId", QFieldType.INTEGER).withPossibleValueSourceName(RUN))
+         .withField(ro("lastVerifiedDate", QFieldType.DATE_TIME))
+         .withField(roEnum("slaStatus", "sla_status")));
+   }
+
+
+
+   /*******************************************************************************
+    ** backups_plan_scope: one row per inclusion or exclusion (spec 4.2).
+    *******************************************************************************/
+   static QTableMetaData planScope(BackupsQBitConfig config)
+   {
+      return (base(config, PLAN_SCOPE, "Backup Plan Scope", "filter_list")
+         .withRecordLabelFormatAndFields("%s %s %s", "action", "selectorType", "selectorValue")
+         .withField(new QFieldMetaData("backupPlanId", QFieldType.INTEGER).withIsRequired(true).withPossibleValueSourceName(PLAN))
+         .withField(enumField("action", "scope_action", "INCLUDE").withIsRequired(true))
+         .withField(enumField("selectorType", "selector_type", "TABLE").withIsRequired(true))
+         .withField(new QFieldMetaData("selectorValue", QFieldType.STRING).withIsRequired(true).withMaxLength(200))
+         .withField(new QFieldMetaData("filterJson", QFieldType.TEXT))
+         .withField(new QFieldMetaData("includeAssociatedChildren", QFieldType.BOOLEAN).withDefaultValue(true))
+         .withField(new QFieldMetaData("fieldExclusions", QFieldType.STRING).withMaxLength(1000)));
+   }
+
+
+
+   /*******************************************************************************
+    ** backups_run: one execution of a plan, or an ad-hoc export (spec 4.3).
+    *******************************************************************************/
+   static QTableMetaData run(BackupsQBitConfig config)
+   {
+      return (base(config, RUN, "Backup Run", "history")
+         .withRecordLabelFormatAndFields("%s", "runKey")
+         .withUniqueKey(new UniqueKey("runKey"))
+         .withAssociation(new Association().withName("artifacts").withAssociatedTableName(ARTIFACT).withJoinName("backupsArtifactJoinRun"))
+         .withAssociation(new Association().withName("verifications").withAssociatedTableName(VERIFICATION).withJoinName("backupsVerificationJoinRun"))
+         .withField(ro("backupPlanId", QFieldType.INTEGER).withPossibleValueSourceName(PLAN))
+         .withField(ro("runKey", QFieldType.STRING))
+         .withField(roEnum("trigger", "run_trigger"))
+         .withField(ro("requestedByUserId", QFieldType.STRING))
+         .withField(roEnum("status", "run_status"))
+         .withField(ro("startDate", QFieldType.DATE_TIME))
+         .withField(ro("endDate", QFieldType.DATE_TIME))
+         .withField(roLong("durationMs"))
+         .withField(ro("heartbeatDate", QFieldType.DATE_TIME))
+         .withField(roInt("tablesPlanned"))
+         .withField(roInt("tablesDone"))
+         .withField(roLong("rowsTotal"))
+         .withField(roLong("bytesRaw"))
+         .withField(roLong("bytesStored"))
+         .withField(roInt("filesReferenced"))
+         .withField(roInt("filesCopied"))
+         .withField(roInt("filesDeduplicated"))
+         .withField(roInt("filesMissing"))
+         .withField(roLong("fileBytesCopied"))
+         .withField(roLong("fileBytesReferenced"))
+         .withField(roEnum("consistencyAchieved", "consistency_achieved"))
+         .withField(ro("snapshotId", QFieldType.STRING))
+         .withField(ro("storageBucketAlias", QFieldType.STRING))
+         .withField(ro("runPrefix", QFieldType.STRING))
+         .withField(ro("manifestKey", QFieldType.STRING))
+         .withField(ro("manifestSha256", QFieldType.STRING))
+         .withField(ro("qqqVersion", QFieldType.STRING))
+         .withField(ro("appVersion", QFieldType.STRING))
+         .withField(ro("instanceName", QFieldType.STRING))
+         .withField(ro("schemaFingerprint", QFieldType.STRING))
+         .withField(ro("securityScope", QFieldType.STRING))
+         .withField(ro("encryptionKeyRef", QFieldType.STRING))
+         .withField(ro("encryptionKeyFingerprint", QFieldType.STRING))
+         .withField(roEnum("retentionClass", "retention_class"))
+         .withField(ro("lockedUntil", QFieldType.DATE_TIME))
+         .withField(ro("prunedDate", QFieldType.DATE_TIME))
+         .withField(ro("prunedByUserId", QFieldType.STRING))
+         .withField(roEnum("verificationStatus", "verification_status"))
+         .withField(roInt("warningCount"))
+         .withField(ro("errorMessage", QFieldType.TEXT))
+         .withField(ro("errorDetail", QFieldType.TEXT)));
+   }
+
+
+
+   /*******************************************************************************
+    ** backups_artifact: one stored object or part (spec 4.4).
+    *******************************************************************************/
+   static QTableMetaData artifact(BackupsQBitConfig config)
+   {
+      return (base(config, ARTIFACT, "Backup Artifact", "inventory_2")
+         .withRecordLabelFormatAndFields("%s", "objectKey")
+         .withField(ro("backupRunId", QFieldType.INTEGER).withPossibleValueSourceName(RUN))
+         .withField(roEnum("artifactType", "artifact_type"))
+         .withField(ro("tableName", QFieldType.STRING))
+         .withField(roInt("partNumber"))
+         .withField(ro("objectKey", QFieldType.STRING))
+         .withField(ro("format", QFieldType.STRING))
+         .withField(ro("compression", QFieldType.STRING))
+         .withField(ro("encrypted", QFieldType.BOOLEAN))
+         .withField(roLong("rowCount"))
+         .withField(roLong("bytesRaw"))
+         .withField(roLong("bytesStored"))
+         .withField(ro("sha256Stored", QFieldType.STRING))
+         .withField(ro("sha256Raw", QFieldType.STRING))
+         .withField(ro("rowsHash", QFieldType.STRING))
+         .withField(ro("firstKey", QFieldType.STRING))
+         .withField(ro("lastKey", QFieldType.STRING))
+         .withField(ro("tableFingerprint", QFieldType.STRING))
+         .withField(roEnum("status", "artifact_status"))
+         .withField(ro("storageVersionId", QFieldType.STRING)));
+   }
+
+
+
+   /*******************************************************************************
+    ** backups_verification: the result of a verify (spec 4.5).
+    *******************************************************************************/
+   static QTableMetaData verification(BackupsQBitConfig config)
+   {
+      return (base(config, VERIFICATION, "Backup Verification", "verified")
+         .withRecordLabelFormatAndFields("%s %s", "level", "status")
+         .withField(ro("backupRunId", QFieldType.INTEGER).withPossibleValueSourceName(RUN))
+         .withField(roEnum("level", "verify_level"))
+         .withField(roEnum("trigger", "run_trigger"))
+         .withField(ro("requestedByUserId", QFieldType.STRING))
+         .withField(ro("startDate", QFieldType.DATE_TIME))
+         .withField(ro("endDate", QFieldType.DATE_TIME))
+         .withField(roEnum("status", "verify_result"))
+         .withField(roInt("tablesChecked"))
+         .withField(roLong("rowsChecked"))
+         .withField(roInt("mismatches"))
+         .withField(ro("scratchTarget", QFieldType.STRING))
+         .withField(ro("detailJson", QFieldType.TEXT)));
+   }
+
+
+
+   /*******************************************************************************
+    ** backups_restore_run: one restore (spec 4.6).
+    *******************************************************************************/
+   static QTableMetaData restoreRun(BackupsQBitConfig config)
+   {
+      return (base(config, RESTORE_RUN, "Backup Restore", "restore")
+         .withRecordLabelFormatAndFields("Restore %s", "id")
+         .withAssociation(new Association().withName("tables").withAssociatedTableName(RESTORE_RUN_TABLE).withJoinName("backupsRestoreRunTableJoinRestoreRun"))
+         .withField(ro("sourceBackupRunId", QFieldType.INTEGER).withPossibleValueSourceName(RUN))
+         .withField(ro("requestedByUserId", QFieldType.STRING))
+         .withField(ro("startDate", QFieldType.DATE_TIME))
+         .withField(ro("endDate", QFieldType.DATE_TIME))
+         .withField(roEnum("status", "restore_status"))
+         .withField(roEnum("mode", "restore_mode"))
+         .withField(roEnum("idStrategy", "id_strategy"))
+         .withField(ro("matchOn", QFieldType.STRING))
+         .withField(ro("targetInstanceName", QFieldType.STRING))
+         .withField(ro("scopeSummary", QFieldType.TEXT))
+         .withField(ro("preRestoreBackupRunId", QFieldType.INTEGER).withPossibleValueSourceName(RUN))
+         .withField(roLong("rowsInserted"))
+         .withField(roLong("rowsUpdated"))
+         .withField(roLong("rowsUnchanged"))
+         .withField(roLong("rowsDeleted"))
+         .withField(roLong("rowsSkipped"))
+         .withField(roLong("rowsErrored"))
+         .withField(roInt("filesRestored"))
+         .withField(roInt("filesAlreadyPresent"))
+         .withField(roInt("filesConflicting"))
+         .withField(roInt("filesMissingFromBackup"))
+         .withField(roLong("fileBytesRestored"))
+         .withField(roInt("sequencesResynced"))
+         .withField(ro("automationsSuppressed", QFieldType.BOOLEAN))
+         .withField(ro("esbSuppressed", QFieldType.BOOLEAN))
+         .withField(ro("confirmationText", QFieldType.STRING))
+         .withField(ro("errorMessage", QFieldType.TEXT)));
+   }
+
+
+
+   /*******************************************************************************
+    ** backups_restore_run_table: per-table restore results (spec 4.7).
+    *******************************************************************************/
+   static QTableMetaData restoreRunTable(BackupsQBitConfig config)
+   {
+      return (base(config, RESTORE_RUN_TABLE, "Backup Restore Table", "table_rows")
+         .withRecordLabelFormatAndFields("%s", "tableName")
+         .withField(ro("backupRestoreRunId", QFieldType.INTEGER).withPossibleValueSourceName(RESTORE_RUN))
+         .withField(ro("tableName", QFieldType.STRING))
+         .withField(ro("targetTableName", QFieldType.STRING))
+         .withField(roLong("rowsInserted"))
+         .withField(roLong("rowsUpdated"))
+         .withField(roLong("rowsUnchanged"))
+         .withField(roLong("rowsDeleted"))
+         .withField(roLong("rowsSkipped"))
+         .withField(roLong("rowsErrored"))
+         .withField(roEnum("schemaStatus", "schema_status"))
+         .withField(ro("unmappedFields", QFieldType.STRING))
+         .withField(ro("sequenceResynced", QFieldType.BOOLEAN))
+         .withField(ro("status", QFieldType.STRING)));
+   }
+
+
+
+   /*******************************************************************************
+    ** backups_file_object: the content-addressed file store (spec 4.8).
+    *******************************************************************************/
+   static QTableMetaData fileObject(BackupsQBitConfig config)
+   {
+      return (base(config, FILE_OBJECT, "Backup File", "attach_file")
+         .withRecordLabelFormatAndFields("%s", "sha256")
+         .withUniqueKey(new UniqueKey("sha256"))
+         .withField(ro("sha256", QFieldType.STRING))
+         .withField(roLong("sizeBytes"))
+         .withField(ro("contentType", QFieldType.STRING))
+         .withField(ro("storageBucketAlias", QFieldType.STRING))
+         .withField(ro("objectKey", QFieldType.STRING))
+         .withField(ro("encrypted", QFieldType.BOOLEAN))
+         .withField(ro("encryptionKeyFingerprint", QFieldType.STRING))
+         .withField(ro("firstRunId", QFieldType.INTEGER).withPossibleValueSourceName(RUN))
+         .withField(ro("lastReferencedRunId", QFieldType.INTEGER).withPossibleValueSourceName(RUN))
+         .withField(roInt("referenceCount"))
+         .withField(ro("lastVerifiedDate", QFieldType.DATE_TIME))
+         .withField(ro("lastVerifyStatus", QFieldType.STRING))
+         .withField(ro("lockedUntil", QFieldType.DATE_TIME))
+         .withField(ro("deletedDate", QFieldType.DATE_TIME)));
+   }
+
+
+
+   /*******************************************************************************
+    ** backups_id_map: transient old-to-new ids for REMAP_NEW_IDS restores (spec 4.9).
+    *******************************************************************************/
+   static QTableMetaData idMap(BackupsQBitConfig config)
+   {
+      return (base(config, ID_MAP, "Backup ID Map", "swap_horiz")
+         .withRecordLabelFormatAndFields("%s %s", "tableName", "oldId")
+         .withField(ro("restoreRunId", QFieldType.INTEGER).withPossibleValueSourceName(RESTORE_RUN))
+         .withField(ro("tableName", QFieldType.STRING))
+         .withField(ro("oldId", QFieldType.STRING))
+         .withField(ro("newId", QFieldType.STRING)));
+   }
+}

src/main/java/com/kingsrook/qbits/backups/permissions/BackupsPermissionChecker.java

@@ -0,0 +1,103 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.permissions;
+
+
+import java.util.Map;
+import com.kingsrook.qbits.backups.BackupsRuntime;
+import com.kingsrook.qbits.backups.model.BackupsTables;
+import com.kingsrook.qqq.backend.core.actions.permissions.CustomPermissionChecker;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+
+
+/*******************************************************************************
+ ** Permission rules for every qbit table, app, widget and process (spec 11.4):
+ ** view reads everything; plans are edited with backups.admin; the tables that
+ ** processes write are read-only in the UI; each process names its permission.
+ *******************************************************************************/
+public class BackupsPermissionChecker implements CustomPermissionChecker
+{
+   /*******************************************************************************
+    ** Check, throwing when denied.
+    *******************************************************************************/
+   @Override
+   public void checkPermissionsThrowing(AbstractActionInput actionInput, MetaDataWithPermissionRules metaData) throws QPermissionDeniedException
+   {
+      String needed = required(actionInput, metaData);
+      if(needed == null)
+      {
+         throw (new QPermissionDeniedException("This Backups table is written only by the Backups qbit."));
+      }
+      if(!has(needed))
+      {
+         throw (new QPermissionDeniedException("This needs the " + needed + " permission."));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The permission an action needs, or null when nobody may do it.
+    *******************************************************************************/
+   static String required(AbstractActionInput actionInput, MetaDataWithPermissionRules metaData)
+   {
+      boolean write = actionInput instanceof InsertInput || actionInput instanceof UpdateInput || actionInput instanceof DeleteInput;
+      if(metaData instanceof QTableMetaData table)
+      {
+         if(!write)
+         {
+            return (BackupsPermissions.VIEW);
+         }
+         return (BackupsTables.PROCESS_WRITTEN.contains(table.getName()) ? null : BackupsPermissions.ADMIN);
+      }
+      if(metaData instanceof QProcessMetaData process)
+      {
+         Map<String, String> byProcess = BackupsRuntime.processPermissions();
+         return (byProcess.getOrDefault(process.getName(), BackupsPermissions.ADMIN));
+      }
+      return (BackupsPermissions.VIEW);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the current session holds a permission (or the instance is fully
+    ** anonymous and the config allows that, for local development).
+    *******************************************************************************/
+   public static boolean has(String permission)
+   {
+      QInstance instance = QContext.getQInstance();
+      QSession  session  = QContext.getQSession();
+      if(session != null && session.hasPermission(permission))
+      {
+         return (true);
+      }
+      return (BackupsRuntime.isOpenWhenAnonymous() && instance != null && instance.getAuthentication() != null
+         && QAuthenticationType.FULLY_ANONYMOUS.equals(instance.getAuthentication().getType()));
+   }
+}

src/main/java/com/kingsrook/qbits/backups/permissions/BackupsPermissions.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.permissions;
+
+
+/*******************************************************************************
+ ** Permission names (design spec section 11.4).
+ *******************************************************************************/
+public final class BackupsPermissions
+{
+   /** The app, every backups table (read-only), the home. */
+   public static final String VIEW    = "backups.view";
+   /** Run backup now, back up records, verify, cancel and resume. */
+   public static final String RUN     = "backups.run";
+   /** Restore (insert-new and upsert), download, cancel restore. */
+   public static final String RESTORE = "backups.restore";
+   /** Edit plans, prune, replace restores, import catalog, test destination. */
+   public static final String ADMIN   = "backups.admin";
+
+
+
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private BackupsPermissions()
+   {
+   }
+}

src/main/java/com/kingsrook/qbits/backups/processes/BackupsProcesses.java

@@ -0,0 +1,263 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.processes;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qbits.backups.BackupsRuntime;
+import com.kingsrook.qbits.backups.model.BackupsTables;
+import com.kingsrook.qbits.backups.permissions.BackupsPermissions;
+import com.kingsrook.qbits.backups.run.BackupEngine;
+import com.kingsrook.qbits.backups.run.BackupRunFactory;
+import com.kingsrook.qbits.backups.run.BackupRunRequest;
+import com.kingsrook.qbits.backups.run.BackupRunResult;
+import com.kingsrook.qbits.backups.scope.ScopeResolver;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+
+
+/*******************************************************************************
+ ** The qbit processes (spec 9).  Each is a standard streamed ETL with the standard
+ ** Review and Result summary lines; none is in the nav; labels are plain.  They
+ ** appear as record actions on the table that has the records they act on.
+ *******************************************************************************/
+public final class BackupsProcesses
+{
+   public static final String RUN_PLAN = "backupRunPlan";
+
+
+
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private BackupsProcesses()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** All processes.
+    *******************************************************************************/
+   public static List<QProcessMetaData> processes()
+   {
+      List<QProcessMetaData> processes = new ArrayList<>();
+      processes.add(permission(etl(RUN_PLAN, "Run backup now", BackupsTables.PLAN, RunPlanTransform.class, RunPlanLoad.class,
+         "This is a preview of what the backup will include. Nothing is written until you continue."), BackupsPermissions.RUN));
+      return (processes);
+   }
+
+
+
+   /*******************************************************************************
+    ** Record the permission a process needs.
+    *******************************************************************************/
+   public static QProcessMetaData permission(QProcessMetaData process, String permission)
+   {
+      BackupsRuntime.processPermissions().put(process.getName(), permission);
+      return (process);
+   }
+
+
+
+   /*******************************************************************************
+    ** A standard streamed ETL process with full validation (Review is the dry run).
+    *******************************************************************************/
+   public static QProcessMetaData etl(String name, String label, String table, Class<? extends AbstractTransformStep> transform,
+                                      Class<? extends AbstractLoadStep> load, String preview)
+   {
+      Map<String, Serializable> values = new HashMap<>();
+      values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, table);
+      values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table);
+      values.put(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION, true);
+      values.put(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true);
+      values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, preview);
+      QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(ExtractViaQueryStep.class, transform, load, values);
+      process.setName(name);
+      process.setLabel(label);
+      process.setTableName(table);
+      process.setPermissionRules(BackupsTables.permissionRules());
+      return (process);
+   }
+
+
+
+   /*******************************************************************************
+    ** A summary line with future (Review) and past (Result) wording.
+    *******************************************************************************/
+   public static ProcessSummaryLine line(Status status, String singularFuture, String pluralFuture, String singularPast, String pluralPast)
+   {
+      ProcessSummaryLine line = new ProcessSummaryLine(status, 0, null);
+      line.setSingularFutureMessage(singularFuture);
+      line.setPluralFutureMessage(pluralFuture);
+      line.setSingularPastMessage(singularPast);
+      line.setPluralPastMessage(pluralPast);
+      return (line);
+   }
+
+
+
+   /*******************************************************************************
+    ** Run backup now, review: what each selected plan will back up.
+    *******************************************************************************/
+   public static class RunPlanTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface
+   {
+      private final ProcessSummaryLine plans    = line(Status.OK, "plan will run", "plans will run", "plan was run", "plans were run");
+      private final ProcessSummaryLine tables   = line(Status.INFO, "table will be backed up", "tables will be backed up", "table was backed up", "tables were backed up");
+      private final ProcessSummaryLine excluded = line(Status.INFO, "table is left out by its role or scope", "tables are left out by their role or scope",
+         "table was left out by its role or scope", "tables were left out by their role or scope");
+      private final ProcessSummaryLine inactive = line(Status.WARNING, "plan is not active (it can still be run by hand)", "plans are not active (they can still be run by hand)",
+         "plan was not active", "plans were not active");
+      private final ProcessSummaryLine invalid  = line(Status.ERROR, "plan cannot run", "plans cannot run", "plan could not run", "plans could not run");
+      private final List<String>       samples  = new ArrayList<>();
+
+
+
+      /*******************************************************************************
+       ** Resolve each plan scope without writing anything.
+       *******************************************************************************/
+      @Override
+      public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
+      {
+         for(QRecord plan : input.getRecords())
+         {
+            try
+            {
+               String alias = plan.getValueString("storageBucketAlias");
+               if(alias == null)
+               {
+                  throw (new QException("no storage bucket alias"));
+               }
+               BackupsRuntime.storageFor(alias);
+               ScopeResolver.ResolvedScope scope = BackupRunFactory.resolveScope(plan);
+               plans.incrementCountAndAddPrimaryKey(plan.getValue("id"));
+               tables.incrementCount(scope.tables().size());
+               excluded.incrementCount(scope.excluded().size());
+               if(!Boolean.TRUE.equals(plan.getValueBoolean("isActive")))
+               {
+                  inactive.incrementCountAndAddPrimaryKey(plan.getValue("id"));
+               }
+               for(ScopeResolver.ResolvedTable table : scope.tables().stream().limit(5).toList())
+               {
+                  samples.add(table.name());
+               }
+               output.getRecords().add(plan);
+            }
+            catch(Exception e)
+            {
+               invalid.incrementCountAndAddPrimaryKey(plan.getValue("id"));
+               invalid.setMessageSuffix(": " + e.getMessage());
+            }
+         }
+      }
+
+
+
+      /*******************************************************************************
+       ** Review lines.
+       *******************************************************************************/
+      @Override
+      public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
+      {
+         ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
+         plans.addSelfToListIfAnyCount(lines);
+         tables.addSelfToListIfAnyCount(lines);
+         excluded.addSelfToListIfAnyCount(lines);
+         inactive.addSelfToListIfAnyCount(lines);
+         invalid.addSelfToListIfAnyCount(lines);
+         return (lines);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Run backup now, execute: run the engine for each plan.
+    *******************************************************************************/
+   public static class RunPlanLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
+   {
+      private final ProcessSummaryLine succeeded = line(Status.OK, "backup will succeed", "backups succeeded", "backup succeeded", "backups succeeded");
+      private final ProcessSummaryLine warnings  = line(Status.WARNING, "backup finished with warnings", "backups finished with warnings",
+         "backup finished with warnings", "backups finished with warnings");
+      private final ProcessSummaryLine failed    = line(Status.ERROR, "backup failed", "backups failed", "backup failed", "backups failed");
+      private final ProcessSummaryLine rows      = line(Status.INFO, "row was written", "rows were written", "row was written", "rows were written");
+      private final ProcessSummaryLine files     = line(Status.INFO, "referenced file was copied or already stored", "referenced files were copied or already stored",
+         "referenced file was copied or already stored", "referenced files were copied or already stored");
+      private final List<BackupRunResult> results = new ArrayList<>();
+
+
+
+      /*******************************************************************************
+       ** Run the plans.
+       *******************************************************************************/
+      @Override
+      public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
+      {
+         for(QRecord plan : input.getRecords())
+         {
+            BackupRunRequest request = BackupRunFactory.forPlan(plan, "MANUAL");
+            BackupRunResult  result  = new BackupEngine().execute(request);
+            results.add(result);
+            switch(result.getStatus())
+            {
+               case "SUCCEEDED" -> succeeded.incrementCountAndAddPrimaryKey(result.getRunId());
+               case "SUCCEEDED_WITH_WARNINGS" -> warnings.incrementCountAndAddPrimaryKey(result.getRunId());
+               default ->
+               {
+                  failed.incrementCountAndAddPrimaryKey(result.getRunId());
+                  failed.setMessageSuffix(result.getErrorMessage() == null ? null : ": " + result.getErrorMessage());
+               }
+            }
+            rows.incrementCount((int) Math.min(Integer.MAX_VALUE, result.getRows()));
+            files.incrementCount(result.getFilesCopied() + result.getFilesDeduplicated());
+            output.getRecords().add(plan);
+         }
+      }
+
+
+
+      /*******************************************************************************
+       ** Result lines, with the run in the primary keys so it links to the run.
+       *******************************************************************************/
+      @Override
+      public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
+      {
+         ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
+         succeeded.addSelfToListIfAnyCount(lines);
+         warnings.addSelfToListIfAnyCount(lines);
+         failed.addSelfToListIfAnyCount(lines);
+         rows.addSelfToListIfAnyCount(lines);
+         files.addSelfToListIfAnyCount(lines);
+         return (lines);
+      }
+   }
+}

src/main/java/com/kingsrook/qbits/backups/run/BackupCatalog.java

@@ -0,0 +1,254 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.run;
+
+
+import java.io.Serializable;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import com.kingsrook.qbits.backups.files.FileCopier;
+import com.kingsrook.qbits.backups.model.BackupsTables;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** Reads and writes the qbit own tables (the catalog) through the standard QQQ
+ ** actions.  Every table is a normal, visible QQQ table; this class only wraps
+ ** the calls the processes make, under a permission bypass (the callers have
+ ** already checked backups.* permissions).
+ *******************************************************************************/
+public class BackupCatalog
+{
+
+   /*******************************************************************************
+    ** Insert one row, returning its id.
+    *******************************************************************************/
+   public Integer insert(String table, Map<String, Serializable> values) throws QException
+   {
+      QRecord record = new QRecord();
+      record.setValues(new LinkedHashMap<>(values));
+      InsertInput input = new InsertInput(table);
+      input.setRecords(List.of(record));
+      input.withPermissionBypass("backups catalog");
+      List<QRecord> rs = new InsertAction().execute(input).getRecords();
+      if(rs.get(0).getErrors() != null && !rs.get(0).getErrors().isEmpty())
+      {
+         throw (new QException("Could not write to " + table + ": " + rs.get(0).getErrors().get(0).getMessage()));
+      }
+      return (rs.get(0).getValueInteger("id"));
+   }
+
+
+
+   /*******************************************************************************
+    ** Update some fields of one row.
+    *******************************************************************************/
+   public void update(String table, Integer id, Map<String, Serializable> values) throws QException
+   {
+      Map<String, Serializable> all = new LinkedHashMap<>(values);
+      all.put("id", id);
+      QRecord record = new QRecord();
+      record.setValues(all);
+      UpdateInput input = new UpdateInput(table);
+      input.setRecords(List.of(record));
+      input.withPermissionBypass("backups catalog");
+      List<QRecord> rs = new UpdateAction().execute(input).getRecords();
+      if(rs.get(0).getErrors() != null && !rs.get(0).getErrors().isEmpty())
+      {
+         throw (new QException("Could not update " + table + " " + id + ": " + rs.get(0).getErrors().get(0).getMessage()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** One row by id.
+    *******************************************************************************/
+   public Optional<QRecord> get(String table, Integer id) throws QException
+   {
+      return (query(table, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, id))).stream().findFirst());
+   }
+
+
+
+   /*******************************************************************************
+    ** Query a table (catalog tables are small; no paging).
+    *******************************************************************************/
+   public List<QRecord> query(String table, QQueryFilter filter) throws QException
+   {
+      QueryInput input = new QueryInput(table);
+      input.setFilter(filter);
+      input.withPermissionBypass("backups catalog");
+      return (new QueryAction().execute(input).getRecords());
+   }
+
+
+
+   /*******************************************************************************
+    ** Query by one equals criterion, newest first by id.
+    *******************************************************************************/
+   public List<QRecord> where(String table, String field, Serializable value) throws QException
+   {
+      return (query(table, new QQueryFilter(new QFilterCriteria(field, QCriteriaOperator.EQUALS, value)).withOrderBy(new QFilterOrderBy("id", false))));
+   }
+
+
+
+   /*******************************************************************************
+    ** Insert one artifact row.
+    *******************************************************************************/
+   public Integer insertArtifact(Integer runId, String type, String tableName, Integer partNumber, String objectKey, String format, String compression,
+                                 boolean encrypted, long rows, long bytesRaw, long bytesStored, String sha256Stored, String sha256Raw, String rowsHash,
+                                 String firstKey, String lastKey, String tableFingerprint) throws QException
+   {
+      Map<String, Serializable> v = new LinkedHashMap<>();
+      v.put("backupRunId", runId);
+      v.put("artifactType", type);
+      v.put("tableName", tableName);
+      v.put("partNumber", partNumber);
+      v.put("objectKey", objectKey);
+      v.put("format", format);
+      v.put("compression", compression);
+      v.put("encrypted", encrypted);
+      v.put("rowCount", rows);
+      v.put("bytesRaw", bytesRaw);
+      v.put("bytesStored", bytesStored);
+      v.put("sha256Stored", sha256Stored);
+      v.put("sha256Raw", sha256Raw);
+      v.put("rowsHash", rowsHash);
+      v.put("firstKey", firstKey);
+      v.put("lastKey", lastKey);
+      v.put("tableFingerprint", tableFingerprint);
+      v.put("status", "COMPLETE");
+      return (insert(BackupsTables.ARTIFACT, v));
+   }
+
+
+
+   /*******************************************************************************
+    ** The file store registry over backups_file_object: known hashes are those with
+    ** a live (not deleted) row.
+    *******************************************************************************/
+   public FileCopier.FileStoreRegistry fileStoreRegistry(String bucketAlias, Integer runId, String keyFingerprint)
+   {
+      return (new FileCopier.FileStoreRegistry()
+      {
+         /*******************************************************************************
+          ** {@inheritDoc}
+          *******************************************************************************/
+         @Override
+         public boolean isKnown(String sha256)
+         {
+            try
+            {
+               return (existing(sha256).isPresent());
+            }
+            catch(QException e)
+            {
+               throw (new IllegalStateException("Could not look up a stored file: " + e.getMessage(), e));
+            }
+         }
+
+
+
+         /*******************************************************************************
+          ** {@inheritDoc}
+          *******************************************************************************/
+         @Override
+         public void register(String sha256, long sizeBytes, String contentType, String casKey, boolean encrypted)
+         {
+            try
+            {
+               Optional<QRecord> existing = existing(sha256);
+               if(existing.isPresent())
+               {
+                  return;
+               }
+               List<QRecord> deleted = where(BackupsTables.FILE_OBJECT, "sha256", sha256);
+               Map<String, Serializable> v = new LinkedHashMap<>();
+               v.put("sha256", sha256);
+               v.put("sizeBytes", sizeBytes);
+               v.put("contentType", contentType);
+               v.put("storageBucketAlias", bucketAlias);
+               v.put("objectKey", casKey);
+               v.put("encrypted", encrypted);
+               v.put("encryptionKeyFingerprint", keyFingerprint);
+               v.put("firstRunId", runId);
+               v.put("lastReferencedRunId", runId);
+               v.put("referenceCount", 0);
+               v.put("deletedDate", null);
+               if(deleted.isEmpty())
+               {
+                  insert(BackupsTables.FILE_OBJECT, v);
+               }
+               else
+               {
+                  update(BackupsTables.FILE_OBJECT, deleted.get(0).getValueInteger("id"), v);
+               }
+            }
+            catch(QException e)
+            {
+               throw (new IllegalStateException("Could not record a stored file: " + e.getMessage(), e));
+            }
+         }
+
+
+
+         /*******************************************************************************
+          **
+          *******************************************************************************/
+         private Optional<QRecord> existing(String sha256) throws QException
+         {
+            return (where(BackupsTables.FILE_OBJECT, "sha256", sha256).stream().filter(r -> r.getValue("deletedDate") == null).findFirst());
+         }
+      });
+   }
+
+
+
+   /*******************************************************************************
+    ** The runs that are RUNNING for a plan, with a heartbeat newer than the cutoff.
+    *******************************************************************************/
+   public List<QRecord> liveRunsForPlan(Integer planId, Instant heartbeatCutoff) throws QException
+   {
+      List<QRecord> live = new ArrayList<>();
+      for(QRecord run : where(BackupsTables.RUN, "backupPlanId", planId))
+      {
+         Instant heartbeat = run.getValueInstant("heartbeatDate");
+         if("RUNNING".equals(run.getValueString("status")) && heartbeat != null && heartbeat.isAfter(heartbeatCutoff))
+         {
+            live.add(run);
+         }
+      }
+      return (live);
+   }
+}

src/main/java/com/kingsrook/qbits/backups/run/BackupEngine.java

@@ -0,0 +1,768 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.run;
+
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.Serializable;
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.function.Function;
+import java.util.zip.GZIPOutputStream;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.kingsrook.qbits.backups.BackupsRuntime;
+import com.kingsrook.qbits.backups.alerts.BackupAlertSink;
+import com.kingsrook.qbits.backups.codec.HashingOutputStream;
+import com.kingsrook.qbits.backups.codec.TableArtifactWriter;
+import com.kingsrook.qbits.backups.files.FileCopier;
+import com.kingsrook.qbits.backups.files.FileReference;
+import com.kingsrook.qbits.backups.files.FileReferenceProvider;
+import com.kingsrook.qbits.backups.manifest.BackupManifest;
+import com.kingsrook.qbits.backups.manifest.TableFingerprints;
+import com.kingsrook.qbits.backups.model.BackupsTables;
+import com.kingsrook.qbits.backups.scope.ScopeResolver;
+import com.kingsrook.qbits.backups.storage.BackupStorage;
+import com.kingsrook.qbits.backups.storage.WriteOptions;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Executes one backup run (spec 8 and 9.2).
+ **
+ ** The order is the contract: the run row is created RUNNING; every table is
+ ** streamed in primary key order into numbered parts (with their SHA-256 and the
+ ** rowsHash); the files the rows reference are copied into the deduplicated
+ ** content-addressed store; checksums.sha256 is written; and the manifest is
+ ** written LAST, then checked with a head.  A run with no manifest did not
+ ** complete.  The run row is updated every page (heartbeat), and a failure marks
+ ** the run FAILED with a scrubbed message (never row values).
+ *******************************************************************************/
+public class BackupEngine
+{
+   private static final QLogger LOG = QLogger.getLogger(BackupEngine.class);
+
+   private static final ObjectMapper MAPPER = new ObjectMapper();
+
+   private static final DateTimeFormatter RUN_KEY_FORMAT = DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'").withZone(ZoneOffset.UTC);
+   private static final DateTimeFormatter DATE_PATH      = DateTimeFormatter.ofPattern("yyyy/MM/dd").withZone(ZoneOffset.UTC);
+
+   private static final long MAX_ROWS_PER_PART = 1_000_000L;
+
+   private final BackupCatalog catalog = new BackupCatalog();
+
+
+
+   /*******************************************************************************
+    ** Run a backup.  Never throws for a backup failure: the result carries the
+    ** status and message (and the run row is FAILED).
+    *******************************************************************************/
+   public BackupRunResult execute(BackupRunRequest request)
+   {
+      BackupRunResult result  = new BackupRunResult();
+      Instant         started = request.getClock().instant();
+      String          runKey  = RUN_KEY_FORMAT.format(started) + "-" + UUID.randomUUID().toString().substring(0, 4);
+      String          planPrefix = request.getSettings().string("prefix", "qqq-backups");
+      String          instancePrefix = trimSlashes(planPrefix) + "/" + request.getInstanceName() + "/";
+      String          runPrefix  = instancePrefix + request.getSettings().planKey() + "/" + DATE_PATH.format(started) + "/" + runKey + "/";
+      result.setRunKey(runKey);
+      result.setRunPrefix(runPrefix);
+
+      try
+      {
+         Map<String, Serializable> runValues = new LinkedHashMap<>();
+         runValues.put("backupPlanId", request.getSettings().getPlanId());
+         runValues.put("runKey", runKey);
+         runValues.put("trigger", request.getTrigger());
+         runValues.put("requestedByUserId", request.getRequestedBy());
+         runValues.put("status", "RUNNING");
+         runValues.put("startDate", started);
+         runValues.put("heartbeatDate", started);
+         runValues.put("tablesPlanned", request.getScope().tables().size());
+         runValues.put("storageBucketAlias", request.getBucketAlias());
+         runValues.put("runPrefix", runPrefix);
+         runValues.put("instanceName", request.getInstanceName());
+         runValues.put("appVersion", request.getAppVersion());
+         runValues.put("securityScope", request.getSecurityScope());
+         runValues.put("encryptionKeyRef", request.getEncryptionKeyRef());
+         runValues.put("encryptionKeyFingerprint", request.getEncryptionKeyFingerprint());
+         runValues.put("verificationStatus", "UNVERIFIED");
+         result.setRunId(catalog.insert(BackupsTables.RUN, runValues));
+      }
+      catch(Exception e)
+      {
+         result.setStatus("FAILED");
+         result.setErrorMessage("Could not create the run record: " + e.getMessage());
+         return (result);
+      }
+
+      try
+      {
+         doRun(request, result, runPrefix, instancePrefix, started);
+      }
+      catch(CancelledException e)
+      {
+         finish(request, result, "CANCELLED", "Cancelled", started);
+      }
+      catch(TimedOutException e)
+      {
+         finish(request, result, "TIMED_OUT", e.getMessage(), started);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Backup run failed", e, logPair("runKey", runKey));
+         finish(request, result, "FAILED", scrub(e), started);
+      }
+      return (result);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void doRun(BackupRunRequest request, BackupRunResult result, String runPrefix, String instancePrefix, Instant started) throws Exception
+   {
+      QInstance        qInstance = QContext.getQInstance();
+      BackupStorage    storage   = request.getStorage();
+      var              settings  = request.getSettings();
+      String           format    = settings.format();
+      String           compression = settings.compression();
+      boolean          copyFiles = "COPY".equals(settings.fileObjectPolicy());
+      long             maxMillis = settings.integer("maxDurationMinutes", 240) * 60_000L;
+
+      if("ZSTD".equals(compression) || "PARQUET".equals(format))
+      {
+         throw (new IllegalStateException("The " + ("ZSTD".equals(compression) ? "ZSTD compression" : "PARQUET format") + " is not available yet; use GZIP and JSONL or CSV."));
+      }
+      String sensitivePolicy = settings.string("sensitiveFieldPolicy", "EXCLUDE");
+      if("INCLUDE_ENCRYPTED_ONLY".equals(sensitivePolicy) && !request.getTransform().encrypts())
+      {
+         throw (new IllegalStateException("The sensitive field policy INCLUDE_ENCRYPTED_ONLY requires client-side encryption."));
+      }
+      String consistency = consistencyFor(settings.string("consistency", "SNAPSHOT_PREFERRED"), result);
+
+      Function<String, BackupStorage> sourceResolver = request.getSourceResolver() != null ? request.getSourceResolver() : alias -> storage;
+      FileCopier copier = new FileCopier(sourceResolver, request.getBucketAlias(), storage, instancePrefix + "_files/sha256/", request.getTransform(),
+         catalog.fileStoreRegistry(request.getBucketAlias(), result.getRunId(), request.getEncryptionKeyFingerprint()), !copyFiles, result.getRunKey());
+
+      List<BackupManifest.TableEntry> manifestTables = new ArrayList<>();
+      List<String>                    checksums      = new ArrayList<>();
+      List<String>                    fingerprints   = new ArrayList<>();
+      FileIndexWriter                 indexWriter    = null;
+      long                            rows           = 0;
+      long                            bytesStored    = 0;
+      long                            bytesRaw       = 0;
+      int                             done           = 0;
+      int                             referenced     = 0;
+      int                             copied         = 0;
+      int                             deduplicated   = 0;
+      int                             missing        = 0;
+      long                            fileBytes      = 0;
+      long                            fileBytesRef   = 0;
+
+      try
+      {
+         for(ScopeResolver.ResolvedTable resolved : request.getScope().tables())
+         {
+            QTableMetaData table = qInstance.getTable(resolved.name());
+            String         backendType = qInstance.getBackend(table.getBackendName()) == null ? null : qInstance.getBackend(table.getBackendName()).getBackendType();
+
+            List<BackupManifest.ExcludedField> excluded = excludedFields(table, resolved, sensitivePolicy, settings.string("heavyFieldPolicy", "INLINE"));
+            List<String> excludedNames = excluded.stream().map(BackupManifest.ExcludedField::name).toList();
+
+            List<TableArtifactWriter> writers = new ArrayList<>();
+            String tablePrefix = runPrefix + "tables/" + table.getName() + "/";
+            for(String f : "JSONL_AND_CSV".equals(format) ? List.of("JSONL", "CSV") : List.of("CSV".equals(format) ? "CSV" : "JSONL"))
+            {
+               writers.add(new TableArtifactWriter(storage, tablePrefix, table, excludedNames, f, compression, settings.targetPartBytes(), MAX_ROWS_PER_PART,
+                  request.getTransform(), result.getRunKey()));
+            }
+
+            List<FileReference> references = new ArrayList<>();
+            long[]              counter    = { 0 };
+            Instant             deadline   = started.plusMillis(maxMillis);
+            Throwable[]         writeError = { null };
+
+            request.getReader().read(table, resolved.filterJson(), null, settings.pageSize(), null, record ->
+            {
+               try
+               {
+                  for(TableArtifactWriter writer : writers)
+                  {
+                     writer.write(record);
+                  }
+               }
+               catch(IOException e)
+               {
+                  throw (new IllegalStateException("Could not write a row of " + table.getName() + ": " + e.getMessage(), e));
+               }
+               if(copyFiles || "REFERENCE_ONLY".equals(settings.fileObjectPolicy()))
+               {
+                  for(FileReferenceProvider provider : request.getFileProviders())
+                  {
+                     references.addAll(provider.referencesIn(table, record));
+                  }
+               }
+               counter[0]++;
+               if(counter[0] % settings.pageSize() == 0)
+               {
+                  heartbeat(request, result, deadline, table.getName(), counter[0]);
+               }
+            });
+
+            String       rowsHash = null;
+            List<BackupManifest.Part> parts = new ArrayList<>();
+            String fingerprint = TableFingerprints.of(table, backendType);
+            fingerprints.add(fingerprint);
+            for(TableArtifactWriter writer : writers)
+            {
+               List<TableArtifactWriter.PartResult> results = writer.finish();
+               rowsHash = writer.rowsHash();
+               for(TableArtifactWriter.PartResult part : results)
+               {
+                  String relative = part.key().substring(runPrefix.length());
+                  parts.add(new BackupManifest.Part(relative, part.rows(), part.bytesRaw(), part.bytesStored(), part.sha256Stored(), part.sha256Raw(), part.format(), part.compression(), part.encrypted()));
+                  checksums.add(part.sha256Stored() + "  " + relative);
+                  bytesStored += part.bytesStored();
+                  bytesRaw += part.bytesRaw();
+                  catalog.insertArtifact(result.getRunId(), "TABLE_DATA", table.getName(), part.partNumber(), relative, part.format(), part.compression(),
+                     part.encrypted(), part.rows(), part.bytesRaw(), part.bytesStored(), part.sha256Stored(), part.sha256Raw(),
+                     part.partNumber() == results.size() ? rowsHash : null, part.firstKey(), part.lastKey(), fingerprint);
+               }
+            }
+            rows += counter[0];
+
+            for(FileReference ref : references)
+            {
+               if(indexWriter == null)
+               {
+                  indexWriter = new FileIndexWriter(storage, runPrefix + "files/index-00001.jsonl.gz", result.getRunKey());
+               }
+               FileCopier.IndexEntry entry = copier.process(ref);
+               indexWriter.write(entry);
+               referenced++;
+               switch(entry.status())
+               {
+                  case FileCopier.COPIED ->
+                  {
+                     copied++;
+                     fileBytes += entry.size();
+                  }
+                  case FileCopier.DEDUPLICATED -> deduplicated++;
+                  case FileCopier.MISSING -> missing++;
+                  default -> fileBytesRef += Math.max(0, entry.size());
+               }
+               if(request.getCancelRequested().getAsBoolean())
+               {
+                  throw (new CancelledException());
+               }
+            }
+
+            Map<String, String> keyRange = new LinkedHashMap<>();
+            keyRange.put("min", writers.get(0).getFirstKey());
+            keyRange.put("max", writers.get(0).getLastKey());
+            manifestTables.add(new BackupManifest.TableEntry(table.getName(), resolved.role().name(), table.getBackendName(), table.getPrimaryKeyField(),
+               fingerprint, TableFingerprints.descriptor(table, backendType), resolved.filterJson(), excluded, counter[0], rowsHash, keyRange, parts, consistency));
+            done++;
+            catalogProgress(result, done, rows, bytesRaw, bytesStored);
+            if(request.getCancelRequested().getAsBoolean())
+            {
+               throw (new CancelledException());
+            }
+         }
+      }
+      finally
+      {
+         if(indexWriter != null)
+         {
+            indexWriter.close();
+         }
+      }
+
+      List<String> indexKeys = new ArrayList<>();
+      if(indexWriter != null)
+      {
+         indexKeys.add("files/index-00001.jsonl.gz");
+         checksums.add(indexWriter.sha256Stored() + "  files/index-00001.jsonl.gz");
+         catalog.insertArtifact(result.getRunId(), "FILE_OBJECTS_INDEX", null, 1, "files/index-00001.jsonl.gz", "JSONL", "GZIP", request.getTransform().encrypts(),
+            indexWriter.rows(), indexWriter.bytesRaw(), indexWriter.bytesStored(), indexWriter.sha256Stored(), indexWriter.sha256Raw(), null, null, null, null);
+         bytesStored += indexWriter.bytesStored();
+      }
+
+      if(missing > 0)
+      {
+         if("FAIL".equals(settings.string("fileMissingPolicy", "WARN")))
+         {
+            throw (new IllegalStateException(missing + " referenced files are missing at the source"));
+         }
+         result.getWarnings().add(missing + " referenced files were missing at the source");
+      }
+
+      /////////////////////////////////////////////////////////
+      // checksums, then the manifest LAST (the commit mark) //
+      /////////////////////////////////////////////////////////
+      byte[] checksumBytes = (String.join("\n", checksums) + "\n").getBytes(StandardCharsets.UTF_8);
+      putBytes(storage, runPrefix + "checksums.sha256", checksumBytes, result.getRunKey());
+      catalog.insertArtifact(result.getRunId(), "CHECKSUMS", null, 1, "checksums.sha256", "TEXT", "NONE", false, checksums.size(), checksumBytes.length,
+         checksumBytes.length, HashingOutputStream.sha256Hex(checksumBytes), HashingOutputStream.sha256Hex(checksumBytes), null, null, null, null);
+
+      Instant finished = request.getClock().instant();
+      List<String> order = request.getScope().names();
+      BackupManifest manifest = new BackupManifest(BackupManifest.CURRENT_VERSION, result.getRunKey(),
+         new BackupManifest.PlanInfo(settings.planKey(), request.getPlanName(), settings.manifestSettings()),
+         new BackupManifest.InstanceInfo(request.getInstanceName(), request.getAppVersion(), qqqVersion(), List.of()),
+         started.toString(), finished.toString(), request.getRequestedBy(), request.getSecurityScope(),
+         new BackupManifest.ConsistencyInfo(result.getConsistencyAchieved(), List.of()),
+         request.getTransform().encrypts()
+            ? new BackupManifest.EncryptionInfo("CLIENT_AES256_GCM", "tink-streaming-aead-AES256_GCM_HKDF_1MB", request.getEncryptionKeyRef(), request.getEncryptionKeyFingerprint())
+            : new BackupManifest.EncryptionInfo("NONE", null, null, null),
+         manifestTables, order,
+         new BackupManifest.FilesInfo(settings.fileObjectPolicy(), indexKeys, copied + deduplicated, fileBytes + fileBytesRef, missing),
+         result.getWarnings(),
+         new BackupManifest.Totals(manifestTables.size(), rows, bytesStored));
+      byte[] manifestBytes = manifest.toJsonBytes();
+      String manifestKey   = runPrefix + "manifest.json";
+      putBytes(storage, manifestKey, manifestBytes, result.getRunKey());
+      if(storage.head(manifestKey).isEmpty())
+      {
+         throw (new IllegalStateException("The manifest was written but is not visible in the bucket"));
+      }
+      catalog.insertArtifact(result.getRunId(), "MANIFEST", null, 1, "manifest.json", "JSON", "NONE", false, manifestTables.size(), manifestBytes.length,
+         manifestBytes.length, HashingOutputStream.sha256Hex(manifestBytes), HashingOutputStream.sha256Hex(manifestBytes), null, null, null, null);
+
+      result.setManifestKey(manifestKey);
+      result.setManifestSha256(HashingOutputStream.sha256Hex(manifestBytes));
+      result.setTables(manifestTables.size());
+      result.setRows(rows);
+      result.setBytesRaw(bytesRaw);
+      result.setBytesStored(bytesStored);
+      result.setFilesReferenced(referenced);
+      result.setFilesCopied(copied);
+      result.setFilesDeduplicated(deduplicated);
+      result.setFilesMissing(missing);
+      result.setFileBytesCopied(fileBytes);
+
+      Map<String, Serializable> extra = new LinkedHashMap<>();
+      extra.put("manifestKey", manifestKey);
+      extra.put("manifestSha256", result.getManifestSha256());
+      extra.put("schemaFingerprint", TableFingerprints.combine(fingerprints));
+      extra.put("qqqVersion", qqqVersion());
+      extra.put("consistencyAchieved", result.getConsistencyAchieved());
+      extra.put("filesReferenced", referenced);
+      extra.put("filesCopied", copied);
+      extra.put("filesDeduplicated", deduplicated);
+      extra.put("filesMissing", missing);
+      extra.put("fileBytesCopied", fileBytes);
+      extra.put("fileBytesReferenced", fileBytesRef);
+      extra.put("tablesDone", manifestTables.size());
+      extra.put("rowsTotal", rows);
+      extra.put("bytesRaw", bytesRaw);
+      extra.put("bytesStored", bytesStored);
+      extra.put("warningCount", result.getWarnings().size());
+      finishWith(request, result, result.getWarnings().isEmpty() ? "SUCCEEDED" : "SUCCEEDED_WITH_WARNINGS", null, started, extra);
+   }
+
+
+
+   /*******************************************************************************
+    ** Work out the consistency achieved.  Until the core consistent-read hook ships
+    ** (QQQ core PR feature/consistent-read-transactions, next 4.1.x RC), no backend
+    ** reports a snapshot, so: SNAPSHOT_REQUIRED fails before writing anything,
+    ** SNAPSHOT_PREFERRED records BEST_EFFORT with a warning, PER_TABLE records PER_TABLE.
+    *******************************************************************************/
+   private String consistencyFor(String requested, BackupRunResult result)
+   {
+      switch(requested)
+      {
+         case "SNAPSHOT_REQUIRED":
+            throw (new IllegalStateException("The plan requires a consistent snapshot, which this version of QQQ cannot open yet. Use SNAPSHOT_PREFERRED or PER_TABLE."));
+         case "PER_TABLE":
+            result.setConsistencyAchieved("PER_TABLE");
+            return ("PER_TABLE");
+         default:
+            result.setConsistencyAchieved("BEST_EFFORT");
+            result.getWarnings().add("No consistent snapshot was available: every table was read on its own (BEST_EFFORT)");
+            return ("BEST_EFFORT");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The fields left out of a table, by policy.  The primary key is never left out.
+    *******************************************************************************/
+   private List<BackupManifest.ExcludedField> excludedFields(QTableMetaData table, ScopeResolver.ResolvedTable resolved, String sensitivePolicy, String heavyPolicy)
+   {
+      Map<String, String> out = new LinkedHashMap<>();
+      for(QFieldMetaData field : table.getFields().values())
+      {
+         if(field.getName().equals(table.getPrimaryKeyField()))
+         {
+            continue;
+         }
+         if("EXCLUDE".equals(sensitivePolicy) && (field.getIsSensitive() || field.getType() == QFieldType.PASSWORD))
+         {
+            out.put(field.getName(), "SENSITIVE_POLICY");
+         }
+         else if("EXCLUDE".equals(heavyPolicy) && field.getIsHeavy())
+         {
+            out.put(field.getName(), "HEAVY_POLICY");
+         }
+      }
+      for(String name : resolved.excludedFields())
+      {
+         if(table.getFields().containsKey(name) && !name.equals(table.getPrimaryKeyField()))
+         {
+            out.put(name, "SCOPE");
+         }
+      }
+      return (out.entrySet().stream().map(e -> new BackupManifest.ExcludedField(e.getKey(), e.getValue())).toList());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void heartbeat(BackupRunRequest request, BackupRunResult result, Instant deadline, String tableName, long rowsSoFar)
+   {
+      Instant now = request.getClock().instant();
+      if(request.getCancelRequested().getAsBoolean())
+      {
+         throw (new CancelledException());
+      }
+      if(now.isAfter(deadline))
+      {
+         throw (new TimedOutException("The run exceeded its maximum duration"));
+      }
+      try
+      {
+         catalog.update(BackupsTables.RUN, result.getRunId(), Map.of("heartbeatDate", now));
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Could not record a heartbeat", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void catalogProgress(BackupRunResult result, int done, long rows, long bytesRaw, long bytesStored)
+   {
+      try
+      {
+         Map<String, Serializable> v = new LinkedHashMap<>();
+         v.put("tablesDone", done);
+         v.put("rowsTotal", rows);
+         v.put("bytesRaw", bytesRaw);
+         v.put("bytesStored", bytesStored);
+         catalog.update(BackupsTables.RUN, result.getRunId(), v);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Could not record progress", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void finish(BackupRunRequest request, BackupRunResult result, String status, String message, Instant started)
+   {
+      try
+      {
+         finishWith(request, result, status, message, started, new LinkedHashMap<>());
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Could not finish the run record", e);
+         result.setStatus(status);
+         result.setErrorMessage(message);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void finishWith(BackupRunRequest request, BackupRunResult result, String status, String message, Instant started, Map<String, Serializable> extra) throws Exception
+   {
+      Instant end = request.getClock().instant();
+      Map<String, Serializable> v = new LinkedHashMap<>(extra);
+      v.put("status", status);
+      v.put("endDate", end);
+      v.put("durationMs", end.toEpochMilli() - started.toEpochMilli());
+      v.put("heartbeatDate", end);
+      v.put("errorMessage", message);
+      v.put("warningCount", result.getWarnings().size());
+      catalog.update(BackupsTables.RUN, result.getRunId(), v);
+      result.setStatus(status);
+      result.setErrorMessage(message);
+      signal(request, result, status, message);
+   }
+
+
+
+   /*******************************************************************************
+    ** Update the plan denormalized fields, check in the heartbeat, and raise or resolve alerts.
+    *******************************************************************************/
+   private void signal(BackupRunRequest request, BackupRunResult result, String status, String message) throws Exception
+   {
+      Integer planId = request.getSettings().getPlanId();
+      if(planId == null)
+      {
+         return;
+      }
+      boolean ok = "SUCCEEDED".equals(status) || "SUCCEEDED_WITH_WARNINGS".equals(status);
+      Map<String, Serializable> plan = new LinkedHashMap<>();
+      plan.put("lastRunId", result.getRunId());
+      if(ok)
+      {
+         plan.put("lastSuccessRunId", result.getRunId());
+         plan.put("lastSuccessDate", request.getClock().instant());
+         plan.put("slaStatus", "OK");
+      }
+      catalog.update(BackupsTables.PLAN, planId, plan);
+
+      BackupAlertSink sink = BackupsRuntime.getAlertSink();
+      String planKey = request.getSettings().planKey();
+      if(ok)
+      {
+         sink.resolve("backups.run.failed", "backups.run.failed:" + planKey);
+         sink.heartbeat("backups.plan:" + planKey, request.getSettings().integer("slaHours", 26) * 3600L, 0L);
+         if(result.getFilesMissing() > 0)
+         {
+            sink.raise("backups.files.missing", "backups.files.missing:" + planKey, BackupAlertSink.Severity.WARNING, "Referenced files are missing",
+               Map.of("plan", planKey, "missing", String.valueOf(result.getFilesMissing()), "run", result.getRunKey()));
+         }
+         else
+         {
+            sink.resolve("backups.files.missing", "backups.files.missing:" + planKey);
+         }
+      }
+      else if(!"CANCELLED".equals(status))
+      {
+         sink.raise("backups.run.failed", "backups.run.failed:" + planKey, BackupAlertSink.Severity.WARNING, "A backup run failed",
+            Map.of("plan", planKey, "status", status, "run", result.getRunKey()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** A message safe to store: the exception type and message, never row values (the
+    ** engine never puts values in messages), truncated.
+    *******************************************************************************/
+   private static String scrub(Exception e)
+   {
+      String message = e.getClass().getSimpleName() + ": " + e.getMessage();
+      return (message.length() > 2000 ? message.substring(0, 2000) : message);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void putBytes(BackupStorage storage, String key, byte[] bytes, String runKey) throws IOException
+   {
+      try(OutputStream out = storage.openWrite(key, WriteOptions.create().withMetadata("qqq-backup-run", runKey)))
+      {
+         out.write(bytes);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static String trimSlashes(String s)
+   {
+      String r = s;
+      while(r.startsWith("/"))
+      {
+         r = r.substring(1);
+      }
+      while(r.endsWith("/"))
+      {
+         r = r.substring(0, r.length() - 1);
+      }
+      return (r);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static String qqqVersion()
+   {
+      String v = QInstance.class.getPackage().getImplementationVersion();
+      return (v == null ? "unknown" : v);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   static class CancelledException extends RuntimeException
+   {
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   static class TimedOutException extends RuntimeException
+   {
+      /*******************************************************************************
+       ** Constructor.
+       *******************************************************************************/
+      TimedOutException(String message)
+      {
+         super(message);
+      }
+   }
+
+
+
+   /***************************************************************************
+    ** The per-run file index, written as it goes: one JSON line per reference.
+    ***************************************************************************/
+   static class FileIndexWriter
+   {
+      private final HashingOutputStream raw;
+      private final HashingOutputStream stored;
+      private long                      rows = 0;
+
+
+
+      /*******************************************************************************
+       ** Constructor.
+       *******************************************************************************/
+      FileIndexWriter(BackupStorage storage, String key, String runKey) throws IOException
+      {
+         stored = new HashingOutputStream(storage.openWrite(key, WriteOptions.create().withMetadata("qqq-backup-run", runKey)));
+         raw = new HashingOutputStream(new GZIPOutputStream(stored));
+      }
+
+
+
+      /*******************************************************************************
+       ** Write one entry.
+       *******************************************************************************/
+      void write(FileCopier.IndexEntry entry) throws IOException
+      {
+         Map<String, Object> line = new LinkedHashMap<>();
+         line.put("table", entry.tableName());
+         line.put("recordKey", entry.recordKey());
+         line.put("field", entry.field());
+         line.put("sourceBucket", entry.sourceBucketAlias());
+         line.put("sourceKey", entry.sourceKey());
+         line.put("sha256", entry.sha256());
+         line.put("size", entry.size());
+         line.put("status", entry.status());
+         line.put("casKey", entry.casKey());
+         raw.write((MAPPER.writeValueAsString(line) + "\n").getBytes(StandardCharsets.UTF_8));
+         rows++;
+      }
+
+
+
+      /*******************************************************************************
+       ** Close the stream.
+       *******************************************************************************/
+      void close() throws IOException
+      {
+         raw.close();
+      }
+
+
+
+      /*******************************************************************************
+       ** Rows written.
+       *******************************************************************************/
+      long rows()
+      {
+         return (rows);
+      }
+
+
+
+      /*******************************************************************************
+       ** Raw byte count.
+       *******************************************************************************/
+      long bytesRaw()
+      {
+         return (raw.getCount());
+      }
+
+
+
+      /*******************************************************************************
+       ** Stored byte count.
+       *******************************************************************************/
+      long bytesStored()
+      {
+         return (stored.getCount());
+      }
+
+
+
+      /*******************************************************************************
+       ** Raw hash.
+       *******************************************************************************/
+      String sha256Raw()
+      {
+         return (raw.getSha256());
+      }
+
+
+
+      /*******************************************************************************
+       ** Stored hash.
+       *******************************************************************************/
+      String sha256Stored()
+      {
+         return (stored.getSha256());
+      }
+   }
+}

src/main/java/com/kingsrook/qbits/backups/run/BackupRunFactory.java

@@ -0,0 +1,98 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.run;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qbits.backups.BackupsQBitConfig;
+import com.kingsrook.qbits.backups.BackupsRuntime;
+import com.kingsrook.qbits.backups.model.BackupsTables;
+import com.kingsrook.qbits.backups.scope.PlanSettings;
+import com.kingsrook.qbits.backups.scope.ScopeResolver;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+
+
+/*******************************************************************************
+ ** Builds the BackupRunRequest for a plan record: loads the scope rows, resolves
+ ** the tables, and picks the storage by the plan bucket alias.
+ *******************************************************************************/
+public final class BackupRunFactory
+{
+
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private BackupRunFactory()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** The scope rules of a plan.
+    *******************************************************************************/
+   public static List<ScopeResolver.ScopeRule> scopeRules(Integer planId) throws QException
+   {
+      List<ScopeResolver.ScopeRule> rules = new ArrayList<>();
+      for(QRecord row : new BackupCatalog().where(BackupsTables.PLAN_SCOPE, "backupPlanId", planId))
+      {
+         rules.add(new ScopeResolver.ScopeRule("INCLUDE".equals(row.getValueString("action")), row.getValueString("selectorType"), row.getValueString("selectorValue"),
+            row.getValueString("filterJson"), !Boolean.FALSE.equals(row.getValueBoolean("includeAssociatedChildren")), row.getValueString("fieldExclusions")));
+      }
+      return (rules);
+   }
+
+
+
+   /*******************************************************************************
+    ** Resolve the scope of a plan record.
+    *******************************************************************************/
+   public static ScopeResolver.ResolvedScope resolveScope(QRecord plan) throws QException
+   {
+      return (new ScopeResolver(QContext.getQInstance()).resolve(PlanSettings.of(plan), scopeRules(plan.getValueInteger("id"))));
+   }
+
+
+
+   /*******************************************************************************
+    ** The request to run a plan.
+    *******************************************************************************/
+   public static BackupRunRequest forPlan(QRecord plan, String trigger) throws QException
+   {
+      BackupsQBitConfig config   = BackupsRuntime.getConfig();
+      PlanSettings      settings = PlanSettings.of(plan);
+      QSession          session  = QContext.getQSession();
+      String            user     = session == null || session.getUser() == null ? "system:scheduler" : session.getUser().getIdReference();
+
+      return (new BackupRunRequest()
+         .withSettings(settings)
+         .withScope(resolveScope(plan))
+         .withTrigger(trigger)
+         .withRequestedBy("SCHEDULED".equals(trigger) ? "system:scheduler" : user)
+         .withSecurityScope("SCHEDULED".equals(trigger) ? "FULL" : "FULL")
+         .withBucketAlias(settings.string("storageBucketAlias", null))
+         .withStorage(BackupsRuntime.storageFor(settings.string("storageBucketAlias", null)))
+         .withSourceResolver(BackupsRuntime::storageFor)
+         .withInstanceName(config.getInstanceName() != null ? config.getInstanceName() : "qqq")
+         .withPlanName(plan.getValueString("name"))
+         .withClock(BackupsRuntime.getClock()));
+   }
+}

src/main/java/com/kingsrook/qbits/backups/run/BackupRunRequest.java

@@ -0,0 +1,413 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.run;
+
+
+import java.time.Clock;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.BooleanSupplier;
+import com.kingsrook.qbits.backups.codec.ArtifactTransform;
+import com.kingsrook.qbits.backups.files.FileReferenceProvider;
+import com.kingsrook.qbits.backups.scope.PlanSettings;
+import com.kingsrook.qbits.backups.scope.ScopeResolver;
+import com.kingsrook.qbits.backups.storage.BackupStorage;
+
+
+/*******************************************************************************
+ ** Everything a backup run needs, resolved by the caller (the process step, the
+ ** scheduler or a test).
+ *******************************************************************************/
+public class BackupRunRequest
+{
+   private PlanSettings               settings;
+   private ScopeResolver.ResolvedScope scope;
+   private String                     trigger         = "MANUAL";
+   private String                     requestedBy     = "system";
+   private String                     securityScope   = "FULL";
+   private BackupStorage              storage;
+   private String                     bucketAlias;
+   private String                     instanceName    = "qqq";
+   private String                     appVersion;
+   private String                     planName;
+   private ArtifactTransform          transform       = ArtifactTransform.IDENTITY;
+   private String                     encryptionKeyRef;
+   private String                     encryptionKeyFingerprint;
+   private List<FileReferenceProvider> fileProviders   = new ArrayList<>(List.of(new FileReferenceProvider.FromTableMetaData()));
+   private TableReader                reader          = new TableReader.Keyset();
+   private Clock                      clock           = Clock.systemUTC();
+   private BooleanSupplier            cancelRequested = () -> false;
+   private java.util.function.Function<String, BackupStorage> sourceResolver;
+
+
+
+   /*******************************************************************************
+    ** Getter for settings
+    *******************************************************************************/
+   public PlanSettings getSettings()
+   {
+      return (settings);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for settings
+    *******************************************************************************/
+   public BackupRunRequest withSettings(PlanSettings settings)
+   {
+      this.settings = settings;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for scope
+    *******************************************************************************/
+   public ScopeResolver.ResolvedScope getScope()
+   {
+      return (scope);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for scope
+    *******************************************************************************/
+   public BackupRunRequest withScope(ScopeResolver.ResolvedScope scope)
+   {
+      this.scope = scope;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for trigger
+    *******************************************************************************/
+   public String getTrigger()
+   {
+      return (trigger);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for trigger
+    *******************************************************************************/
+   public BackupRunRequest withTrigger(String trigger)
+   {
+      this.trigger = trigger;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for requestedBy
+    *******************************************************************************/
+   public String getRequestedBy()
+   {
+      return (requestedBy);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for requestedBy
+    *******************************************************************************/
+   public BackupRunRequest withRequestedBy(String requestedBy)
+   {
+      this.requestedBy = requestedBy;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for securityScope (FULL, or USER when the run saw only what a user may see)
+    *******************************************************************************/
+   public String getSecurityScope()
+   {
+      return (securityScope);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for securityScope
+    *******************************************************************************/
+   public BackupRunRequest withSecurityScope(String securityScope)
+   {
+      this.securityScope = securityScope;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for storage
+    *******************************************************************************/
+   public BackupStorage getStorage()
+   {
+      return (storage);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for storage
+    *******************************************************************************/
+   public BackupRunRequest withStorage(BackupStorage storage)
+   {
+      this.storage = storage;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for bucketAlias
+    *******************************************************************************/
+   public String getBucketAlias()
+   {
+      return (bucketAlias);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for bucketAlias
+    *******************************************************************************/
+   public BackupRunRequest withBucketAlias(String bucketAlias)
+   {
+      this.bucketAlias = bucketAlias;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for instanceName
+    *******************************************************************************/
+   public String getInstanceName()
+   {
+      return (instanceName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for instanceName
+    *******************************************************************************/
+   public BackupRunRequest withInstanceName(String instanceName)
+   {
+      this.instanceName = instanceName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for appVersion
+    *******************************************************************************/
+   public String getAppVersion()
+   {
+      return (appVersion);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for appVersion
+    *******************************************************************************/
+   public BackupRunRequest withAppVersion(String appVersion)
+   {
+      this.appVersion = appVersion;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for planName
+    *******************************************************************************/
+   public String getPlanName()
+   {
+      return (planName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for planName
+    *******************************************************************************/
+   public BackupRunRequest withPlanName(String planName)
+   {
+      this.planName = planName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for transform
+    *******************************************************************************/
+   public ArtifactTransform getTransform()
+   {
+      return (transform);
+   }
+
+
+
+   /*******************************************************************************
+    ** Set the artifact transform (encryption) with the key reference and fingerprint to record.
+    *******************************************************************************/
+   public BackupRunRequest withTransform(ArtifactTransform transform, String keyRef, String keyFingerprint)
+   {
+      this.transform = transform;
+      this.encryptionKeyRef = keyRef;
+      this.encryptionKeyFingerprint = keyFingerprint;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for encryptionKeyRef
+    *******************************************************************************/
+   public String getEncryptionKeyRef()
+   {
+      return (encryptionKeyRef);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for encryptionKeyFingerprint
+    *******************************************************************************/
+   public String getEncryptionKeyFingerprint()
+   {
+      return (encryptionKeyFingerprint);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for fileProviders
+    *******************************************************************************/
+   public List<FileReferenceProvider> getFileProviders()
+   {
+      return (fileProviders);
+   }
+
+
+
+   /*******************************************************************************
+    ** Add a file reference provider.
+    *******************************************************************************/
+   public BackupRunRequest withFileProvider(FileReferenceProvider provider)
+   {
+      this.fileProviders.add(provider);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for reader
+    *******************************************************************************/
+   public TableReader getReader()
+   {
+      return (reader);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for reader
+    *******************************************************************************/
+   public BackupRunRequest withReader(TableReader reader)
+   {
+      this.reader = reader;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for clock
+    *******************************************************************************/
+   public Clock getClock()
+   {
+      return (clock);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for clock
+    *******************************************************************************/
+   public BackupRunRequest withClock(Clock clock)
+   {
+      this.clock = clock;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for cancelRequested
+    *******************************************************************************/
+   public BooleanSupplier getCancelRequested()
+   {
+      return (cancelRequested);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for cancelRequested
+    *******************************************************************************/
+   public BackupRunRequest withCancelRequested(BooleanSupplier cancelRequested)
+   {
+      this.cancelRequested = cancelRequested;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for sourceResolver (bucket alias of the referencing files to storage); defaults to the plan storage
+    *******************************************************************************/
+   public java.util.function.Function<String, BackupStorage> getSourceResolver()
+   {
+      return (sourceResolver);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for sourceResolver
+    *******************************************************************************/
+   public BackupRunRequest withSourceResolver(java.util.function.Function<String, BackupStorage> sourceResolver)
+   {
+      this.sourceResolver = sourceResolver;
+      return (this);
+   }
+}

src/main/java/com/kingsrook/qbits/backups/run/BackupRunResult.java

@@ -0,0 +1,407 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.run;
+
+
+import java.util.ArrayList;
+import java.util.List;
+
+
+/*******************************************************************************
+ ** What a backup run did.
+ *******************************************************************************/
+public class BackupRunResult
+{
+   private Integer      runId;
+   private String       runKey;
+   private String       runPrefix;
+   private String       status;
+   private String       manifestKey;
+   private String       manifestSha256;
+   private String       consistencyAchieved;
+   private int          tables;
+   private long         rows;
+   private long         bytesStored;
+   private long         bytesRaw;
+   private int          filesReferenced;
+   private int          filesCopied;
+   private int          filesDeduplicated;
+   private int          filesMissing;
+   private long         fileBytesCopied;
+   private String       errorMessage;
+   private List<String> warnings = new ArrayList<>();
+
+
+
+   /*******************************************************************************
+    ** Whether the run produced a usable backup (has a manifest).
+    *******************************************************************************/
+   public boolean isUsable()
+   {
+      return (manifestKey != null && ("SUCCEEDED".equals(status) || "SUCCEEDED_WITH_WARNINGS".equals(status)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for runId
+    *******************************************************************************/
+   public Integer getRunId()
+   {
+      return (runId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for runId
+    *******************************************************************************/
+   public void setRunId(Integer runId)
+   {
+      this.runId = runId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for runKey
+    *******************************************************************************/
+   public String getRunKey()
+   {
+      return (runKey);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for runKey
+    *******************************************************************************/
+   public void setRunKey(String runKey)
+   {
+      this.runKey = runKey;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for runPrefix
+    *******************************************************************************/
+   public String getRunPrefix()
+   {
+      return (runPrefix);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for runPrefix
+    *******************************************************************************/
+   public void setRunPrefix(String runPrefix)
+   {
+      this.runPrefix = runPrefix;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for status
+    *******************************************************************************/
+   public String getStatus()
+   {
+      return (status);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for status
+    *******************************************************************************/
+   public void setStatus(String status)
+   {
+      this.status = status;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for manifestKey
+    *******************************************************************************/
+   public String getManifestKey()
+   {
+      return (manifestKey);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for manifestKey
+    *******************************************************************************/
+   public void setManifestKey(String manifestKey)
+   {
+      this.manifestKey = manifestKey;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for manifestSha256
+    *******************************************************************************/
+   public String getManifestSha256()
+   {
+      return (manifestSha256);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for manifestSha256
+    *******************************************************************************/
+   public void setManifestSha256(String manifestSha256)
+   {
+      this.manifestSha256 = manifestSha256;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for consistencyAchieved
+    *******************************************************************************/
+   public String getConsistencyAchieved()
+   {
+      return (consistencyAchieved);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for consistencyAchieved
+    *******************************************************************************/
+   public void setConsistencyAchieved(String consistencyAchieved)
+   {
+      this.consistencyAchieved = consistencyAchieved;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for tables
+    *******************************************************************************/
+   public int getTables()
+   {
+      return (tables);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for tables
+    *******************************************************************************/
+   public void setTables(int tables)
+   {
+      this.tables = tables;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for rows
+    *******************************************************************************/
+   public long getRows()
+   {
+      return (rows);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for rows
+    *******************************************************************************/
+   public void setRows(long rows)
+   {
+      this.rows = rows;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for bytesStored
+    *******************************************************************************/
+   public long getBytesStored()
+   {
+      return (bytesStored);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for bytesStored
+    *******************************************************************************/
+   public void setBytesStored(long bytesStored)
+   {
+      this.bytesStored = bytesStored;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for bytesRaw
+    *******************************************************************************/
+   public long getBytesRaw()
+   {
+      return (bytesRaw);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for bytesRaw
+    *******************************************************************************/
+   public void setBytesRaw(long bytesRaw)
+   {
+      this.bytesRaw = bytesRaw;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for filesReferenced
+    *******************************************************************************/
+   public int getFilesReferenced()
+   {
+      return (filesReferenced);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for filesReferenced
+    *******************************************************************************/
+   public void setFilesReferenced(int filesReferenced)
+   {
+      this.filesReferenced = filesReferenced;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for filesCopied
+    *******************************************************************************/
+   public int getFilesCopied()
+   {
+      return (filesCopied);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for filesCopied
+    *******************************************************************************/
+   public void setFilesCopied(int filesCopied)
+   {
+      this.filesCopied = filesCopied;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for filesDeduplicated
+    *******************************************************************************/
+   public int getFilesDeduplicated()
+   {
+      return (filesDeduplicated);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for filesDeduplicated
+    *******************************************************************************/
+   public void setFilesDeduplicated(int filesDeduplicated)
+   {
+      this.filesDeduplicated = filesDeduplicated;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for filesMissing
+    *******************************************************************************/
+   public int getFilesMissing()
+   {
+      return (filesMissing);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for filesMissing
+    *******************************************************************************/
+   public void setFilesMissing(int filesMissing)
+   {
+      this.filesMissing = filesMissing;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for fileBytesCopied
+    *******************************************************************************/
+   public long getFileBytesCopied()
+   {
+      return (fileBytesCopied);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for fileBytesCopied
+    *******************************************************************************/
+   public void setFileBytesCopied(long fileBytesCopied)
+   {
+      this.fileBytesCopied = fileBytesCopied;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for errorMessage
+    *******************************************************************************/
+   public String getErrorMessage()
+   {
+      return (errorMessage);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for errorMessage
+    *******************************************************************************/
+   public void setErrorMessage(String errorMessage)
+   {
+      this.errorMessage = errorMessage;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for warnings
+    *******************************************************************************/
+   public List<String> getWarnings()
+   {
+      return (warnings);
+   }
+}

src/main/java/com/kingsrook/qbits/backups/run/TableReader.java

@@ -0,0 +1,128 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.run;
+
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.function.Consumer;
+import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+
+
+/*******************************************************************************
+ ** Reads the rows of one table in primary key order, in keyset pages (WHERE pk >
+ ** lastKey ORDER BY pk LIMIT n), so the cost stays flat with depth and a read can
+ ** resume from a key.  A read raw: hidden fields, passwords and heavy fields are
+ ** returned as stored, and possible values are not translated.
+ **
+ ** This is an internal stand-in for the QQQ core KeysetPager and QueryInput
+ ** withRawRead (core PR feature/keyset-pager-raw-read); the run engine only
+ ** depends on this interface, so it is swapped when that ships in the next RC.
+ *******************************************************************************/
+public interface TableReader
+{
+
+   /***************************************************************************
+    ** Read every row of the table matching the optional filter, after startAfterKey
+    ** (exclusive, null for the beginning), calling the sink for each in key order.
+    ***************************************************************************/
+   void read(QTableMetaData table, String filterJson, Serializable startAfterKey, int pageSize, QBackendTransaction transaction, Consumer<QRecord> sink) throws QException;
+
+
+   /***************************************************************************
+    ** The default implementation over QueryAction.
+    ***************************************************************************/
+   class Keyset implements TableReader
+   {
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      private static QQueryFilter parseFilter(String filterJson) throws QException
+      {
+         try
+         {
+            return (filterJson == null || filterJson.isBlank() ? null : JsonUtils.toObject(filterJson, QQueryFilter.class));
+         }
+         catch(java.io.IOException e)
+         {
+            throw (new QException("Could not read a scope filter: " + e.getMessage()));
+         }
+      }
+
+
+
+      /*******************************************************************************
+       ** {@inheritDoc}
+       *******************************************************************************/
+      @Override
+      public void read(QTableMetaData table, String filterJson, Serializable startAfterKey, int pageSize, QBackendTransaction transaction, Consumer<QRecord> sink) throws QException
+      {
+         String       pk      = table.getPrimaryKeyField();
+         Serializable lastKey = startAfterKey;
+         QQueryFilter user    = parseFilter(filterJson);
+
+         while(true)
+         {
+            QQueryFilter filter = new QQueryFilter();
+            if(lastKey != null)
+            {
+               filter.addCriteria(new QFilterCriteria(pk, QCriteriaOperator.GREATER_THAN, List.of(lastKey)));
+            }
+            if(user != null)
+            {
+               filter.addSubFilter(user);
+            }
+            filter.withOrderBy(new QFilterOrderBy(pk, true));
+            filter.setLimit(pageSize);
+
+            QueryInput queryInput = new QueryInput(table.getName());
+            queryInput.setFilter(filter);
+            queryInput.setShouldOmitHiddenFields(false);
+            queryInput.setShouldMaskPasswords(false);
+            queryInput.setShouldFetchHeavyFields(true);
+            queryInput.setShouldTranslatePossibleValues(false);
+            queryInput.setShouldGenerateDisplayValues(false);
+            if(transaction != null)
+            {
+               queryInput.setTransaction(transaction);
+            }
+            queryInput.withPermissionBypass("backups run");
+
+            List<QRecord> page = new QueryAction().execute(queryInput).getRecords();
+            for(QRecord record : page)
+            {
+               sink.accept(record);
+               lastKey = record.getValue(pk);
+            }
+            if(page.size() < pageSize)
+            {
+               return;
+            }
+         }
+      }
+   }
+}

src/main/java/com/kingsrook/qbits/backups/scope/BackupTableMetaData.java

@@ -0,0 +1,147 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.scope;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QSupplementalTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
+/*******************************************************************************
+ ** Supplemental table meta-data: the backup role of a table, and the fields
+ ** that reference files in object storage (when the app does not register them
+ ** through the object-storage reference providers).
+ *******************************************************************************/
+public class BackupTableMetaData extends QSupplementalTableMetaData
+{
+   public static final String TYPE = "backups";
+
+   private TableRole    role;
+   private List<String> fileReferenceFields = new ArrayList<>();
+   private String       fileBucketAlias;
+
+
+
+   /*******************************************************************************
+    ** The BackupTableMetaData on a table, or null.
+    *******************************************************************************/
+   public static BackupTableMetaData of(QTableMetaData table)
+   {
+      return (table == null ? null : QSupplementalTableMetaData.of(table, TYPE));
+   }
+
+
+
+   /*******************************************************************************
+    ** The BackupTableMetaData on a table, adding an empty one if needed.
+    *******************************************************************************/
+   public static BackupTableMetaData ofOrWithNew(QTableMetaData table)
+   {
+      return (QSupplementalTableMetaData.ofOrWithNew(table, TYPE, BackupTableMetaData::new));
+   }
+
+
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   public String getType()
+   {
+      return (TYPE);
+   }
+
+
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   protected QSupplementalTableMetaData finishClone(QSupplementalTableMetaData abstractClone)
+   {
+      BackupTableMetaData clone = (BackupTableMetaData) abstractClone;
+      clone.role = this.role;
+      clone.fileReferenceFields = new ArrayList<>(this.fileReferenceFields);
+      clone.fileBucketAlias = this.fileBucketAlias;
+      return (clone);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for role
+    *******************************************************************************/
+   public TableRole getRole()
+   {
+      return (role);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for role
+    *******************************************************************************/
+   public BackupTableMetaData withRole(TableRole role)
+   {
+      this.role = role;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for fileReferenceFields (fields holding a storage key in the source bucket)
+    *******************************************************************************/
+   public List<String> getFileReferenceFields()
+   {
+      return (fileReferenceFields);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for fileReferenceFields
+    *******************************************************************************/
+   public BackupTableMetaData withFileReferenceFields(List<String> fileReferenceFields)
+   {
+      this.fileReferenceFields = fileReferenceFields;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for fileBucketAlias (the bucket the fileReferenceFields keys live in)
+    *******************************************************************************/
+   public String getFileBucketAlias()
+   {
+      return (fileBucketAlias);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for fileBucketAlias
+    *******************************************************************************/
+   public BackupTableMetaData withFileBucketAlias(String fileBucketAlias)
+   {
+      this.fileBucketAlias = fileBucketAlias;
+      return (this);
+   }
+}

src/main/java/com/kingsrook/qbits/backups/scope/PlanSettings.java

@@ -0,0 +1,229 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.scope;
+
+
+import java.io.Serializable;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+
+
+/*******************************************************************************
+ ** The settings of a backup plan (a backups_plan record), with the defaults of
+ ** spec 4.1 applied for anything not set.  Ad-hoc exports (no plan row) build one
+ ** with the static factory and the with() method.
+ *******************************************************************************/
+public class PlanSettings
+{
+   private final Map<String, Serializable> values = new LinkedHashMap<>();
+
+   private Integer planId;
+
+
+
+   /*******************************************************************************
+    ** From a plan record.
+    *******************************************************************************/
+   public static PlanSettings of(QRecord plan)
+   {
+      PlanSettings settings = new PlanSettings();
+      if(plan != null)
+      {
+         settings.planId = plan.getValueInteger("id");
+         settings.values.putAll(plan.getValues());
+      }
+      return (settings);
+   }
+
+
+
+   /*******************************************************************************
+    ** An ad-hoc (no plan row) settings object with all defaults.
+    *******************************************************************************/
+   public static PlanSettings adHoc()
+   {
+      return (new PlanSettings());
+   }
+
+
+
+   /*******************************************************************************
+    ** Set one value (fluent).
+    *******************************************************************************/
+   public PlanSettings with(String name, Serializable value)
+   {
+      values.put(name, value);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** The plan id, or null for an ad-hoc run.
+    *******************************************************************************/
+   public Integer getPlanId()
+   {
+      return (planId);
+   }
+
+
+
+   /*******************************************************************************
+    ** A string setting.
+    *******************************************************************************/
+   public String string(String name, String defaultValue)
+   {
+      String value = ValueUtils.getValueAsString(values.get(name));
+      return (value == null || value.isEmpty() ? defaultValue : value);
+   }
+
+
+
+   /*******************************************************************************
+    ** An integer setting.
+    *******************************************************************************/
+   public Integer integer(String name, Integer defaultValue)
+   {
+      Integer value = ValueUtils.getValueAsInteger(values.get(name));
+      return (value == null ? defaultValue : value);
+   }
+
+
+
+   /*******************************************************************************
+    ** A long setting.
+    *******************************************************************************/
+   public Long longValue(String name, Long defaultValue)
+   {
+      Long value = ValueUtils.getValueAsLong(values.get(name));
+      return (value == null ? defaultValue : value);
+   }
+
+
+
+   /*******************************************************************************
+    ** A boolean setting.
+    *******************************************************************************/
+   public boolean bool(String name, boolean defaultValue)
+   {
+      Boolean value = ValueUtils.getValueAsBoolean(values.get(name));
+      return (value == null ? defaultValue : value);
+   }
+
+
+
+   /*******************************************************************************
+    ** The roles a plan includes by default (comma separated).
+    *******************************************************************************/
+   public List<TableRole> includeRoles()
+   {
+      String csv = string("includeRoles", "DATA,AUDIT");
+      return (java.util.Arrays.stream(csv.split(",")).map(String::trim).filter(s -> !s.isEmpty()).map(TableRole::valueOf).toList());
+   }
+
+
+
+   /*******************************************************************************
+    ** The plan key (slug used in object keys).
+    *******************************************************************************/
+   public String planKey()
+   {
+      return (string("planKey", "adhoc"));
+   }
+
+
+
+   /*******************************************************************************
+    ** The scope type.
+    *******************************************************************************/
+   public String scopeType()
+   {
+      return (string("scopeType", "INSTANCE"));
+   }
+
+
+
+   /*******************************************************************************
+    ** The format.
+    *******************************************************************************/
+   public String format()
+   {
+      return (string("format", "JSONL"));
+   }
+
+
+
+   /*******************************************************************************
+    ** The compression.
+    *******************************************************************************/
+   public String compression()
+   {
+      return (string("compression", "GZIP"));
+   }
+
+
+
+   /*******************************************************************************
+    ** The file object policy: COPY (default) or REFERENCE_ONLY.
+    *******************************************************************************/
+   public String fileObjectPolicy()
+   {
+      return (string("fileObjectPolicy", "COPY"));
+   }
+
+
+
+   /*******************************************************************************
+    ** The page size.
+    *******************************************************************************/
+   public int pageSize()
+   {
+      return (integer("pageSize", 5000));
+   }
+
+
+
+   /*******************************************************************************
+    ** Target part size in bytes.
+    *******************************************************************************/
+   public long targetPartBytes()
+   {
+      return (longValue("targetPartBytes", 268435456L));
+   }
+
+
+
+   /*******************************************************************************
+    ** The settings as a map (for the manifest; contains no secrets by design).
+    *******************************************************************************/
+   public Map<String, Serializable> manifestSettings()
+   {
+      Map<String, Serializable> map = new LinkedHashMap<>();
+      for(String key : List.of("scopeType", "includeRoles", "format", "compression", "encryptionMode", "sensitiveFieldPolicy", "heavyFieldPolicy",
+         "fileObjectPolicy", "fileMissingPolicy", "consistency", "incrementalMode", "pageSize", "targetPartBytes"))
+      {
+         if(values.get(key) != null)
+         {
+            map.put(key, values.get(key));
+         }
+      }
+      return (map);
+   }
+}

src/main/java/com/kingsrook/qbits/backups/scope/ScopeResolver.java

@@ -0,0 +1,448 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.scope;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
+/*******************************************************************************
+ ** Turns a plan (scope type, included roles, scope rows) into the ordered list of
+ ** tables a run will back up (spec 4.2).
+ **
+ ** Resolution: start from the scope type and roles, apply the INCLUDE rows, then
+ ** the EXCLUDE rows, add associated children, and drop SECRET and BACKUPS tables.
+ ** Tables are ordered parents first, using associations, joins and table-backed
+ ** possible values.  Every table that was left out is reported with its reason.
+ *******************************************************************************/
+public class ScopeResolver
+{
+   private final QInstance qInstance;
+
+
+
+   /*******************************************************************************
+    ** Constructor.
+    *******************************************************************************/
+   public ScopeResolver(QInstance qInstance)
+   {
+      this.qInstance = qInstance;
+   }
+
+
+
+   /***************************************************************************
+    ** One scope row.
+    **
+    ** @param include true for INCLUDE, false for EXCLUDE
+    ** @param selectorType TABLE, APP, QBIT or ROLE
+    ** @param selectorValue the table, app, qbit or role
+    ** @param filterJson optional QQueryFilter json (TABLE only)
+    ** @param includeAssociatedChildren also include tables reachable through associations
+    ** @param fieldExclusions comma separated field names to leave out
+    ***************************************************************************/
+   public record ScopeRule(boolean include, String selectorType, String selectorValue, String filterJson, boolean includeAssociatedChildren, String fieldExclusions)
+   {
+   }
+
+
+
+   /***************************************************************************
+    ** One table that will be backed up.
+    ***************************************************************************/
+   public record ResolvedTable(String name, TableRole role, String filterJson, List<String> excludedFields)
+   {
+   }
+
+
+
+   /***************************************************************************
+    ** A table that was left out, and why.
+    ***************************************************************************/
+   public record ExcludedTable(String name, String reason)
+   {
+   }
+
+
+
+   /***************************************************************************
+    ** The outcome of resolving a scope.
+    **
+    ** @param tables the tables, parents first
+    ** @param excluded tables left out, by reason
+    ** @param cyclic tables that could not be ordered (reference cycle), in name order at the end
+    ***************************************************************************/
+   public record ResolvedScope(List<ResolvedTable> tables, List<ExcludedTable> excluded, List<String> cyclic)
+   {
+      /*******************************************************************************
+       ** The table names, in restore order.
+       *******************************************************************************/
+      public List<String> names()
+      {
+         return (tables.stream().map(ResolvedTable::name).toList());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Resolve a scope.
+    *******************************************************************************/
+   public ResolvedScope resolve(PlanSettings settings, List<ScopeRule> rules)
+   {
+      List<TableRole> roles = settings.includeRoles();
+
+      Set<String>         selected  = new LinkedHashSet<>();
+      Set<String>         explicit  = new LinkedHashSet<>();
+      Map<String, String> filters   = new LinkedHashMap<>();
+      Map<String, String> fieldsOut = new LinkedHashMap<>();
+      Set<String>         expandChildrenFor = new LinkedHashSet<>();
+
+      if("INSTANCE".equals(settings.scopeType()))
+      {
+         for(QTableMetaData table : sortedTables())
+         {
+            if(roles.contains(TableRoles.roleOf(qInstance, table)))
+            {
+               selected.add(table.getName());
+            }
+         }
+      }
+
+      for(ScopeRule rule : rules)
+      {
+         if(!rule.include())
+         {
+            continue;
+         }
+         for(String name : tablesFor(rule, roles))
+         {
+            selected.add(name);
+            if("TABLE".equals(rule.selectorType()))
+            {
+               explicit.add(name);
+               if(rule.filterJson() != null && !rule.filterJson().isBlank())
+               {
+                  filters.put(name, rule.filterJson());
+               }
+               if(rule.fieldExclusions() != null && !rule.fieldExclusions().isBlank())
+               {
+                  fieldsOut.put(name, rule.fieldExclusions());
+               }
+            }
+            if(rule.includeAssociatedChildren())
+            {
+               expandChildrenFor.add(name);
+            }
+         }
+      }
+
+      for(ScopeRule rule : rules)
+      {
+         if(rule.include())
+         {
+            continue;
+         }
+         for(String name : tablesFor(rule, List.of(TableRole.values())))
+         {
+            selected.remove(name);
+            explicit.remove(name);
+         }
+      }
+
+      for(String name : new ArrayList<>(expandChildrenFor))
+      {
+         addAssociatedChildren(name, selected);
+      }
+
+      List<ExcludedTable> excluded = new ArrayList<>();
+      List<String>        kept     = new ArrayList<>();
+      for(String name : selected)
+      {
+         QTableMetaData table = qInstance.getTable(name);
+         if(table == null)
+         {
+            excluded.add(new ExcludedTable(name, "NOT_FOUND"));
+            continue;
+         }
+         TableRole role = TableRoles.roleOf(qInstance, table);
+         if(role == TableRole.SECRET)
+         {
+            excluded.add(new ExcludedTable(name, "SECRET"));
+         }
+         else if(role == TableRole.BACKUPS)
+         {
+            excluded.add(new ExcludedTable(name, "BACKUPS"));
+         }
+         else if((role == TableRole.EPHEMERAL || role == TableRole.DERIVED || role == TableRole.EXTERNAL) && !explicit.contains(name) && !roles.contains(role))
+         {
+            excluded.add(new ExcludedTable(name, role.name()));
+         }
+         else
+         {
+            kept.add(name);
+         }
+      }
+
+      List<String> cyclic  = new ArrayList<>();
+      List<String> ordered = order(kept, cyclic);
+
+      List<ResolvedTable> tables = new ArrayList<>();
+      for(String name : ordered)
+      {
+         List<String> out = new ArrayList<>();
+         if(fieldsOut.get(name) != null)
+         {
+            for(String field : fieldsOut.get(name).split(","))
+            {
+               if(!field.isBlank())
+               {
+                  out.add(field.trim());
+               }
+            }
+         }
+         tables.add(new ResolvedTable(name, TableRoles.roleOf(qInstance, qInstance.getTable(name)), filters.get(name), out));
+      }
+      return (new ResolvedScope(tables, excluded, cyclic));
+   }
+
+
+
+   /*******************************************************************************
+    ** The tables a rule selects.
+    *******************************************************************************/
+   private Collection<String> tablesFor(ScopeRule rule, List<TableRole> rolesForExpansion)
+   {
+      Set<String> names = new LinkedHashSet<>();
+      switch(rule.selectorType())
+      {
+         case "TABLE" -> names.add(rule.selectorValue());
+         case "ROLE" ->
+         {
+            TableRole role = TableRole.valueOf(rule.selectorValue());
+            for(QTableMetaData table : sortedTables())
+            {
+               if(TableRoles.roleOf(qInstance, table) == role)
+               {
+                  names.add(table.getName());
+               }
+            }
+         }
+         case "QBIT" ->
+         {
+            for(QTableMetaData table : sortedTables())
+            {
+               if(rule.selectorValue().equals(table.getSourceQBitName()) && rolesForExpansion.contains(TableRoles.roleOf(qInstance, table)))
+               {
+                  names.add(table.getName());
+               }
+            }
+         }
+         case "APP" ->
+         {
+            QAppMetaData app = qInstance.getApp(rule.selectorValue());
+            if(app != null)
+            {
+               Set<String> inApp = new LinkedHashSet<>();
+               collectAppTables(app, inApp, new LinkedHashSet<>());
+               for(String name : inApp)
+               {
+                  QTableMetaData table = qInstance.getTable(name);
+                  if(table != null && rolesForExpansion.contains(TableRoles.roleOf(qInstance, table)))
+                  {
+                     names.add(name);
+                  }
+               }
+            }
+         }
+         default ->
+         {
+         }
+      }
+      return (names);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void collectAppTables(QAppMetaData app, Set<String> tables, Set<String> seenApps)
+   {
+      if(!seenApps.add(app.getName()))
+      {
+         return;
+      }
+      for(QAppChildMetaData child : app.getChildren() == null ? List.<QAppChildMetaData>of() : app.getChildren())
+      {
+         if(child instanceof QAppMetaData childApp)
+         {
+            collectAppTables(childApp, tables, seenApps);
+         }
+         else if(child instanceof QTableMetaData table)
+         {
+            tables.add(table.getName());
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void addAssociatedChildren(String name, Set<String> selected)
+   {
+      QTableMetaData table = qInstance.getTable(name);
+      if(table == null || table.getAssociations() == null)
+      {
+         return;
+      }
+      for(Association association : table.getAssociations())
+      {
+         if(selected.add(association.getAssociatedTableName()))
+         {
+            addAssociatedChildren(association.getAssociatedTableName(), selected);
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private List<QTableMetaData> sortedTables()
+   {
+      return (new ArrayList<>(new TreeMap<>(qInstance.getTables()).values()));
+   }
+
+
+
+   /*******************************************************************************
+    ** The tables that must be restored before the given table (its parents).
+    *******************************************************************************/
+   Set<String> parentsOf(String name, Set<String> universe)
+   {
+      Set<String>    parents = new LinkedHashSet<>();
+      QTableMetaData table   = qInstance.getTable(name);
+
+      for(QTableMetaData other : qInstance.getTables().values())
+      {
+         if(other.getAssociations() != null)
+         {
+            for(Association association : other.getAssociations())
+            {
+               if(name.equals(association.getAssociatedTableName()))
+               {
+                  parents.add(other.getName());
+               }
+            }
+         }
+      }
+
+      for(QJoinMetaData join : qInstance.getJoins().values())
+      {
+         if(join.getType() == JoinType.MANY_TO_ONE && name.equals(join.getLeftTable()))
+         {
+            parents.add(join.getRightTable());
+         }
+         else if(join.getType() == JoinType.ONE_TO_MANY && name.equals(join.getRightTable()))
+         {
+            parents.add(join.getLeftTable());
+         }
+      }
+
+      for(QFieldMetaData field : table.getFields().values())
+      {
+         if(field.getPossibleValueSourceName() != null)
+         {
+            QPossibleValueSource pvs = qInstance.getPossibleValueSource(field.getPossibleValueSourceName());
+            if(pvs != null && pvs.getType() == QPossibleValueSourceType.TABLE && pvs.getTableName() != null)
+            {
+               parents.add(pvs.getTableName());
+            }
+         }
+      }
+
+      parents.remove(name);
+      parents.retainAll(universe);
+      return (parents);
+   }
+
+
+
+   /*******************************************************************************
+    ** Topological order, parents first, stable by name.  Tables in a reference cycle
+    ** are appended in name order and reported through the cyclic list.
+    *******************************************************************************/
+   List<String> order(List<String> names, List<String> cyclic)
+   {
+      Set<String>              universe  = new LinkedHashSet<>(names);
+      Map<String, Set<String>> remaining = new TreeMap<>();
+      for(String name : names)
+      {
+         remaining.put(name, parentsOf(name, universe));
+      }
+
+      List<String> ordered = new ArrayList<>();
+      while(!remaining.isEmpty())
+      {
+         List<String> ready = new ArrayList<>();
+         for(Map.Entry<String, Set<String>> entry : remaining.entrySet())
+         {
+            if(entry.getValue().isEmpty())
+            {
+               ready.add(entry.getKey());
+            }
+         }
+         if(ready.isEmpty())
+         {
+            cyclic.addAll(remaining.keySet());
+            ordered.addAll(remaining.keySet());
+            break;
+         }
+         for(String name : ready)
+         {
+            ordered.add(name);
+            remaining.remove(name);
+         }
+         for(Set<String> parents : remaining.values())
+         {
+            parents.removeAll(ready);
+         }
+      }
+      return (ordered);
+   }
+}

src/main/java/com/kingsrook/qbits/backups/scope/TableRole.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.scope;
+
+
+/*******************************************************************************
+ ** What a table is for, as far as backups are concerned (spec 3.2).  Plans select
+ ** tables by role.
+ *******************************************************************************/
+public enum TableRole
+{
+   /** Business data (included by default). */
+   DATA,
+   /** Audit tables (included by default, configurable). */
+   AUDIT,
+   /** Rebuildable from other data: search indexes, rollups, caches. Excluded by default. */
+   DERIVED,
+   /** Sessions, idempotency keys, scheduler internals. Excluded by default. */
+   EPHEMERAL,
+   /** Tokens, salts, credentials. NEVER backed up, even when selected explicitly. */
+   SECRET,
+   /** The tables of this qbit: the catalog is rebuilt from manifests. */
+   BACKUPS,
+   /** Tables backed by another system (API). Excluded by default; selectable. */
+   EXTERNAL
+}

src/main/java/com/kingsrook/qbits/backups/scope/TableRoles.java

@@ -0,0 +1,130 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.scope;
+
+
+import java.util.Set;
+import com.kingsrook.qbits.backups.BackupsQBitConfig;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
+/*******************************************************************************
+ ** Works out the backup role of a table: the declared role when there is one,
+ ** else inferred by the rules of spec 3.2.
+ *******************************************************************************/
+public final class TableRoles
+{
+   private static final Set<String> AUDIT_TABLES = Set.of("audit", "auditTable", "auditUser", "auditDetail");
+
+   private static final String[] SECRET_NAME_HINTS = { "refreshToken", "RefreshToken", "oauthToken", "OAuthToken", "apiKey" };
+
+   private static final String[] EPHEMERAL_NAME_PREFIXES = { "qrtz_" };
+   private static final String[] EPHEMERAL_NAME_PARTS    = { "Session", "Idempotency", "RedirectState", "AsyncJob" };
+
+
+
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private TableRoles()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** The role of a table.
+    *******************************************************************************/
+   public static TableRole roleOf(QInstance qInstance, QTableMetaData table)
+   {
+      BackupTableMetaData declared = BackupTableMetaData.of(table);
+      if(declared != null && declared.getRole() != null)
+      {
+         return (declared.getRole());
+      }
+
+      String name = table.getName();
+      if(name.startsWith(BackupsQBitConfig.TABLE_NAME_PREFIX))
+      {
+         return (TableRole.BACKUPS);
+      }
+      for(String hint : SECRET_NAME_HINTS)
+      {
+         if(name.contains(hint))
+         {
+            return (TableRole.SECRET);
+         }
+      }
+      if(isHiddenWithSecretFields(table))
+      {
+         return (TableRole.SECRET);
+      }
+      if(AUDIT_TABLES.contains(name))
+      {
+         return (TableRole.AUDIT);
+      }
+      if(table.getCacheOf() != null)
+      {
+         return (TableRole.DERIVED);
+      }
+      for(String prefix : EPHEMERAL_NAME_PREFIXES)
+      {
+         if(name.startsWith(prefix))
+         {
+            return (TableRole.EPHEMERAL);
+         }
+      }
+      for(String part : EPHEMERAL_NAME_PARTS)
+      {
+         if(name.contains(part) || name.toLowerCase().contains(part.toLowerCase()))
+         {
+            return (TableRole.EPHEMERAL);
+         }
+      }
+      if(qInstance != null && table.getBackendName() != null && qInstance.getBackend(table.getBackendName()) != null
+         && "api".equalsIgnoreCase(qInstance.getBackend(table.getBackendName()).getBackendType()))
+      {
+         return (TableRole.EXTERNAL);
+      }
+      return (TableRole.DATA);
+   }
+
+
+
+   /*******************************************************************************
+    ** A table that is hidden and has a password or sensitive field (other than the
+    ** keys and dates) is treated as a table of secrets.
+    *******************************************************************************/
+   static boolean isHiddenWithSecretFields(QTableMetaData table)
+   {
+      if(!table.getIsHidden())
+      {
+         return (false);
+      }
+      for(QFieldMetaData field : table.getFields().values())
+      {
+         if(field.getType() == QFieldType.PASSWORD || field.getIsSensitive())
+         {
+            return (true);
+         }
+      }
+      return (false);
+   }
+}

src/main/java/com/kingsrook/qbits/backups/storage/BackupStorage.java

@@ -0,0 +1,107 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.storage;
+
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.time.Duration;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+
+/*******************************************************************************
+ ** The narrow storage API that the Backups qbit needs.  It mirrors the
+ ** qbit-object-storage bucket API (put, get, head, list, copy, delete, presign),
+ ** so the production implementation is a thin adapter over a StorageBucket, and
+ ** tests use InMemoryBackupStorage.  Keys are relative to the bucket.
+ **
+ ** All methods throw BackupStorageException (a RuntimeException) on failure.
+ *******************************************************************************/
+public interface BackupStorage
+{
+
+   /***************************************************************************
+    ** Open a stream to write an object of unknown length.  The object becomes
+    ** visible when the stream is closed successfully.
+    ***************************************************************************/
+   OutputStream openWrite(String key, WriteOptions options);
+
+
+   /***************************************************************************
+    ** Open a stream over an object.
+    ***************************************************************************/
+   InputStream openRead(String key);
+
+
+   /***************************************************************************
+    ** Existence and size.
+    ***************************************************************************/
+   Optional<StoredObjectInfo> head(String key);
+
+
+   /***************************************************************************
+    ** All objects with the given key prefix (recursive, in key order).
+    ***************************************************************************/
+   List<StoredObjectInfo> list(String prefix);
+
+
+   /***************************************************************************
+    ** Server-side copy within the same bucket (or connection).
+    ***************************************************************************/
+   void copy(String sourceKey, String destinationKey);
+
+
+   /***************************************************************************
+    ** Permanently delete an object, every version.  Object-locked objects are
+    ** not deleted (DeleteResult.LOCKED); a missing object is NOT_FOUND.
+    ***************************************************************************/
+   DeleteResult deletePermanent(String key);
+
+
+   /***************************************************************************
+    ** A time-limited download link.
+    ***************************************************************************/
+   String presignGet(String key, Duration ttl, String downloadName);
+
+
+   /***************************************************************************
+    ** What the bucket can do (object lock, server-side copy, ...).
+    ***************************************************************************/
+   Set<StorageCapability> capabilities();
+
+
+
+   /***************************************************************************
+    ** Result of a permanent delete.
+    ***************************************************************************/
+   enum DeleteResult
+   {
+      DELETED, NOT_FOUND, LOCKED
+   }
+
+
+
+   /***************************************************************************
+    ** Capabilities, as probed by the storage qbit.
+    ***************************************************************************/
+   enum StorageCapability
+   {
+      OBJECT_LOCK, VERSIONING, SERVER_SIDE_COPY, MULTIPART
+   }
+}

src/main/java/com/kingsrook/qbits/backups/storage/BackupStorageException.java

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.storage;
+
+
+/*******************************************************************************
+ ** A storage failure, with a reason that mirrors the StorageException reasons
+ ** of the object-storage qbit.
+ *******************************************************************************/
+public class BackupStorageException extends RuntimeException
+{
+   private final Reason reason;
+
+
+
+   /*******************************************************************************
+    ** Constructor.
+    *******************************************************************************/
+   public BackupStorageException(Reason reason, String message)
+   {
+      super(message);
+      this.reason = reason;
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor with a cause.
+    *******************************************************************************/
+   public BackupStorageException(Reason reason, String message, Throwable cause)
+   {
+      super(message, cause);
+      this.reason = reason;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for reason
+    *******************************************************************************/
+   public Reason getReason()
+   {
+      return (reason);
+   }
+
+
+
+   /***************************************************************************
+    ** Failure reasons.
+    ***************************************************************************/
+   public enum Reason
+   {
+      NOT_FOUND, ACCESS_DENIED, UNSUPPORTED, PRECONDITION_FAILED, THROTTLED, PROVIDER_ERROR
+   }
+}

src/main/java/com/kingsrook/qbits/backups/storage/InMemoryBackupStorage.java

@@ -0,0 +1,300 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.storage;
+
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.function.Predicate;
+
+
+/*******************************************************************************
+ ** A BackupStorage held in memory.  It is the test implementation and a
+ ** stand-in for local development.  It supports object lock, and has hooks for
+ ** failure injection (a predicate over keys that makes a write fail).
+ *******************************************************************************/
+public class InMemoryBackupStorage implements BackupStorage
+{
+   private final TreeMap<String, Entry> objects = new TreeMap<>();
+
+   private Clock                  clock = Clock.systemUTC();
+   private Predicate<String>      failWritesFor;
+   private Set<StorageCapability> capabilities = EnumSet.of(StorageCapability.OBJECT_LOCK, StorageCapability.SERVER_SIDE_COPY, StorageCapability.MULTIPART);
+
+   private record Entry(byte[] bytes, Map<String, String> metadata, Instant lastModified, Instant lockedUntil)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Use a fixed clock (for object lock and last modified in tests).
+    *******************************************************************************/
+   public InMemoryBackupStorage withClock(Clock clock)
+   {
+      this.clock = clock;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Make writes of matching keys fail with a provider error (failure injection).
+    *******************************************************************************/
+   public InMemoryBackupStorage withFailWritesFor(Predicate<String> failWritesFor)
+   {
+      this.failWritesFor = failWritesFor;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Set the capabilities this storage reports.
+    *******************************************************************************/
+   public InMemoryBackupStorage withCapabilities(Set<StorageCapability> capabilities)
+   {
+      this.capabilities = capabilities;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Test helper: remove everything.
+    *******************************************************************************/
+   public synchronized void clear()
+   {
+      objects.clear();
+      failWritesFor = null;
+   }
+
+
+
+   /*******************************************************************************
+    ** Test helper: put bytes directly.
+    *******************************************************************************/
+   public synchronized void putBytes(String key, byte[] bytes)
+   {
+      objects.put(key, new Entry(bytes, new LinkedHashMap<>(), clock.instant(), null));
+   }
+
+
+
+   /*******************************************************************************
+    ** Test helper: read bytes directly.
+    *******************************************************************************/
+   public synchronized byte[] getBytes(String key)
+   {
+      Entry entry = objects.get(key);
+      return (entry == null ? null : entry.bytes());
+   }
+
+
+
+   /*******************************************************************************
+    ** Test helper: remove an object ignoring any lock (simulating an out-of-band delete).
+    *******************************************************************************/
+   public synchronized void removeIgnoringLock(String key)
+   {
+      objects.remove(key);
+   }
+
+
+
+   /*******************************************************************************
+    ** Test helper: replace an object bytes (simulating corruption).
+    *******************************************************************************/
+   public synchronized void corrupt(String key, byte[] bytes)
+   {
+      Entry old = objects.get(key);
+      objects.put(key, new Entry(bytes, old == null ? new LinkedHashMap<>() : old.metadata(), clock.instant(), old == null ? null : old.lockedUntil()));
+   }
+
+
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   public OutputStream openWrite(String key, WriteOptions options)
+   {
+      if(failWritesFor != null && failWritesFor.test(key))
+      {
+         throw (new BackupStorageException(BackupStorageException.Reason.PROVIDER_ERROR, "injected write failure for " + key));
+      }
+      if(options.getObjectLockMode() != null && !capabilities.contains(StorageCapability.OBJECT_LOCK))
+      {
+         throw (new BackupStorageException(BackupStorageException.Reason.UNSUPPORTED, "object lock is not supported"));
+      }
+
+      return (new ByteArrayOutputStream()
+      {
+         private boolean closed = false;
+
+
+
+         /*******************************************************************************
+          ** commit on close
+          *******************************************************************************/
+         @Override
+         public void close() throws IOException
+         {
+            super.close();
+            if(!closed)
+            {
+               closed = true;
+               synchronized(InMemoryBackupStorage.this)
+               {
+                  objects.put(key, new Entry(toByteArray(), new LinkedHashMap<>(options.getMetadata()), clock.instant(), options.getRetainUntil()));
+               }
+            }
+         }
+      });
+   }
+
+
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   public synchronized InputStream openRead(String key)
+   {
+      Entry entry = objects.get(key);
+      if(entry == null)
+      {
+         throw (new BackupStorageException(BackupStorageException.Reason.NOT_FOUND, "no such object: " + key));
+      }
+      return (new ByteArrayInputStream(entry.bytes()));
+   }
+
+
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   public synchronized Optional<StoredObjectInfo> head(String key)
+   {
+      Entry entry = objects.get(key);
+      return (entry == null ? Optional.empty() : Optional.of(info(key, entry)));
+   }
+
+
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   public synchronized List<StoredObjectInfo> list(String prefix)
+   {
+      List<StoredObjectInfo> rs = new ArrayList<>();
+      for(Map.Entry<String, Entry> e : objects.tailMap(prefix == null ? "" : prefix).entrySet())
+      {
+         if(prefix != null && !e.getKey().startsWith(prefix))
+         {
+            break;
+         }
+         rs.add(info(e.getKey(), e.getValue()));
+      }
+      return (rs);
+   }
+
+
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   public synchronized void copy(String sourceKey, String destinationKey)
+   {
+      Entry entry = objects.get(sourceKey);
+      if(entry == null)
+      {
+         throw (new BackupStorageException(BackupStorageException.Reason.NOT_FOUND, "no such object: " + sourceKey));
+      }
+      objects.put(destinationKey, new Entry(entry.bytes(), entry.metadata(), clock.instant(), null));
+   }
+
+
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   public synchronized DeleteResult deletePermanent(String key)
+   {
+      Entry entry = objects.get(key);
+      if(entry == null)
+      {
+         return (DeleteResult.NOT_FOUND);
+      }
+      if(entry.lockedUntil() != null && entry.lockedUntil().isAfter(clock.instant()))
+      {
+         return (DeleteResult.LOCKED);
+      }
+      objects.remove(key);
+      return (DeleteResult.DELETED);
+   }
+
+
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   public String presignGet(String key, Duration ttl, String downloadName)
+   {
+      return ("memory://" + key + "?ttl=" + ttl.toSeconds());
+   }
+
+
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   public Set<StorageCapability> capabilities()
+   {
+      return (capabilities);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private StoredObjectInfo info(String key, Entry entry)
+   {
+      return (new StoredObjectInfo(key, entry.bytes().length, entry.lastModified(), entry.metadata(), entry.lockedUntil()));
+   }
+}

src/main/java/com/kingsrook/qbits/backups/storage/StoredObjectInfo.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.storage;
+
+
+import java.time.Instant;
+import java.util.Map;
+
+
+/*******************************************************************************
+ ** What the bucket says about one object.
+ **
+ ** @param key key relative to the bucket
+ ** @param size size in bytes
+ ** @param lastModified last modified instant (may be null)
+ ** @param metadata user metadata
+ ** @param lockedUntil object-lock retain-until, or null
+ *******************************************************************************/
+public record StoredObjectInfo(String key, long size, Instant lastModified, Map<String, String> metadata, Instant lockedUntil)
+{
+}

src/main/java/com/kingsrook/qbits/backups/storage/WriteOptions.java

@@ -0,0 +1,118 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.storage;
+
+
+import java.time.Instant;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+
+/*******************************************************************************
+ ** Options for writing one object.
+ *******************************************************************************/
+public class WriteOptions
+{
+   private String              contentType = "application/octet-stream";
+   private Map<String, String> metadata    = new LinkedHashMap<>();
+   private String              objectLockMode;
+   private Instant             retainUntil;
+
+
+
+   /*******************************************************************************
+    ** Start with defaults.
+    *******************************************************************************/
+   public static WriteOptions create()
+   {
+      return (new WriteOptions());
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for contentType
+    *******************************************************************************/
+   public String getContentType()
+   {
+      return (contentType);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for contentType
+    *******************************************************************************/
+   public WriteOptions withContentType(String contentType)
+   {
+      this.contentType = contentType;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for metadata (user metadata, which describes the object even without the catalog)
+    *******************************************************************************/
+   public Map<String, String> getMetadata()
+   {
+      return (metadata);
+   }
+
+
+
+   /*******************************************************************************
+    ** Add one metadata entry.
+    *******************************************************************************/
+   public WriteOptions withMetadata(String key, String value)
+   {
+      this.metadata.put(key, value);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for objectLockMode (GOVERNANCE or COMPLIANCE), null for none
+    *******************************************************************************/
+   public String getObjectLockMode()
+   {
+      return (objectLockMode);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for retainUntil
+    *******************************************************************************/
+   public Instant getRetainUntil()
+   {
+      return (retainUntil);
+   }
+
+
+
+   /*******************************************************************************
+    ** Request an object lock on the written object.
+    *******************************************************************************/
+   public WriteOptions withObjectLock(String mode, Instant retainUntil)
+   {
+      this.objectLockMode = mode;
+      this.retainUntil = retainUntil;
+      return (this);
+   }
+}

src/test/java/com/kingsrook/qbits/backups/BackupsTestUtils.java

@@ -0,0 +1,168 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups;
+
+
+import java.io.Serializable;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qbits.backups.scope.BackupTableMetaData;
+import com.kingsrook.qbits.backups.storage.BackupStorage;
+import com.kingsrook.qbits.backups.storage.InMemoryBackupStorage;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
+
+
+/*******************************************************************************
+ ** Test fixtures: an instance on the memory backend with the Backups qbit and a
+ ** small blog app (authors, posts with a media file key, an ephemeral table).
+ *******************************************************************************/
+public final class BackupsTestUtils
+{
+   public static final String BACKEND = "memory";
+
+   public static final InMemoryBackupStorage BACKUPS = new InMemoryBackupStorage();
+   public static final InMemoryBackupStorage MEDIA   = new InMemoryBackupStorage();
+
+
+
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private BackupsTestUtils()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Build the instance, install the qbit, and initialize the context.
+    *******************************************************************************/
+   public static QInstance init() throws Exception
+   {
+      BackupsRuntime.reset();
+      MemoryRecordStore.getInstance().reset();
+      BACKUPS.clear();
+      MEDIA.clear();
+
+      QInstance qInstance = new QInstance();
+      qInstance.withInstanceDefaultAuthentication(new com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData().withType(com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType.FULLY_ANONYMOUS).withName("anon"));
+      qInstance.addBackend(new QBackendMetaData().withName(BACKEND).withBackendType(MemoryBackendModule.class));
+
+      qInstance.addTable(new QTableMetaData().withName("author").withBackendName(BACKEND).withPrimaryKeyField("id")
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("name", QFieldType.STRING))
+         .withField(new QFieldMetaData("bio", QFieldType.TEXT)));
+
+      qInstance.addTable(new QTableMetaData().withName("blogPost").withBackendName(BACKEND).withPrimaryKeyField("id")
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("authorId", QFieldType.INTEGER).withPossibleValueSourceName("author"))
+         .withField(new QFieldMetaData("title", QFieldType.STRING))
+         .withField(new QFieldMetaData("body", QFieldType.TEXT))
+         .withField(new QFieldMetaData("price", QFieldType.DECIMAL))
+         .withField(new QFieldMetaData("published", QFieldType.BOOLEAN))
+         .withField(new QFieldMetaData("publishedOn", QFieldType.DATE))
+         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME))
+         .withField(new QFieldMetaData("mediaKey", QFieldType.STRING))
+         .withField(new QFieldMetaData("secretNote", QFieldType.STRING).withIsSensitive(true))
+         .withSupplementalMetaData(new BackupTableMetaData().withFileReferenceFields(List.of("mediaKey")).withFileBucketAlias("media")));
+      qInstance.addPossibleValueSource(com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource.newForTable("author"));
+
+      qInstance.addTable(new QTableMetaData().withName("userSession").withBackendName(BACKEND).withPrimaryKeyField("id")
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("token", QFieldType.STRING)));
+
+      Map<String, BackupStorage> buckets = Map.of("backups", BACKUPS, "media", MEDIA);
+      BackupsQBitConfig config = new BackupsQBitConfig().withBackendName(BACKEND).withInstanceName("testapp").withStorageResolver(buckets::get).withOpenWhenAnonymous(true);
+      new BackupsQBitProducer().withQBitConfig(config).withNamespace("backups").produce(qInstance);
+
+      new QInstanceValidator().validate(qInstance);
+      QContext.init(qInstance, new QSession());
+      return (qInstance);
+   }
+
+
+
+   /*******************************************************************************
+    ** Insert rows into a table.
+    *******************************************************************************/
+   public static void insert(String table, List<Map<String, Serializable>> rows) throws QException
+   {
+      InsertInput input = new InsertInput(table);
+      input.setRecords(rows.stream().map(r ->
+      {
+         QRecord record = new QRecord();
+         record.setValues(new LinkedHashMap<>(r));
+         return (record);
+      }).toList());
+      new InsertAction().execute(input);
+   }
+
+
+
+   /*******************************************************************************
+    ** The standard blog data: 3 authors, and the posts, some with media.
+    *******************************************************************************/
+   public static void loadBlog(int posts) throws Exception
+   {
+      insert("author", List.of(
+         Map.of("id", 1, "name", "Ada", "bio", "Line one\nLine two, with \"quotes\" and unicode é中"),
+         Map.of("id", 2, "name", "Grace", "bio", ""),
+         Map.of("id", 3, "name", "Linus")));
+      List<Map<String, Serializable>> rows = new java.util.ArrayList<>();
+      for(int i = 1; i <= posts; i++)
+      {
+         Map<String, Serializable> row = new LinkedHashMap<>();
+         row.put("id", i);
+         row.put("authorId", 1 + (i % 3));
+         row.put("title", "Post " + i);
+         row.put("body", "Body of " + i + ", with a comma");
+         row.put("price", new BigDecimal("12.3400").add(BigDecimal.valueOf(i)));
+         row.put("published", i % 2 == 0);
+         row.put("publishedOn", java.time.LocalDate.of(2026, 1, 1).plusDays(i));
+         row.put("createDate", Instant.parse("2026-10-12T02:00:00.123456Z").plusSeconds(i));
+         row.put("secretNote", "do-not-leak-" + i);
+         if(i % 2 == 1)
+         {
+            String key = "media/photo-" + (i % 3) + ".jpg";
+            row.put("mediaKey", key);
+            if(MEDIA.getBytes(key) == null)
+            {
+               MEDIA.putBytes(key, ("photo bytes " + (i % 3)).getBytes());
+            }
+         }
+         rows.add(row);
+      }
+      insert("blogPost", rows);
+      insert("userSession", List.of(Map.of("id", 1, "token", "tok")));
+   }
+}

src/test/java/com/kingsrook/qbits/backups/run/BackupEngineTest.java

@@ -0,0 +1,282 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.backups.run;
+
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+import java.util.zip.GZIPInputStream;
+import com.kingsrook.qbits.backups.BackupsTestUtils;
+import com.kingsrook.qbits.backups.manifest.BackupManifest;
+import com.kingsrook.qbits.backups.model.BackupsTables;
+import com.kingsrook.qbits.backups.scope.PlanSettings;
+import com.kingsrook.qbits.backups.scope.ScopeResolver;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** End to end tests of a backup run on the memory backend: layout, manifest last,
+ ** canonical rows, policies, file copy and deduplication, failure handling.
+ *******************************************************************************/
+class BackupEngineTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   void setUp() throws Exception
+   {
+      BackupsTestUtils.init();
+      BackupsTestUtils.loadBlog(7);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QRecord plan(Map<String, java.io.Serializable> overrides) throws Exception
+   {
+      Map<String, java.io.Serializable> values = new java.util.LinkedHashMap<>(Map.of("name", "Nightly", "planKey", "nightly", "isActive", true,
+         "scopeType", "INSTANCE", "storageBucketAlias", "backups", "slaHours", 26));
+      values.putAll(overrides);
+      Integer id = new BackupCatalog().insert(BackupsTables.PLAN, values);
+      return (new BackupCatalog().get(BackupsTables.PLAN, id).orElseThrow());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private BackupRunResult run(QRecord plan) throws Exception
+   {
+      return (new BackupEngine().execute(BackupRunFactory.forPlan(plan, "MANUAL")));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static String gunzip(byte[] bytes) throws IOException
+   {
+      return (new String(new GZIPInputStream(new ByteArrayInputStream(bytes)).readAllBytes(), StandardCharsets.UTF_8));
+   }
+
+
+
+   /*******************************************************************************
+    ** A nightly run writes parts, checksums and the manifest LAST, and the manifest
+    ** matches the rows and the stored bytes.
+    *******************************************************************************/
+   @Test
+   void testRunWritesManifestAndParts() throws Exception
+   {
+      BackupRunResult result = run(plan(Map.of()));
+      assertThat(result.getStatus()).isEqualTo("SUCCEEDED_WITH_WARNINGS");
+      assertThat(result.getWarnings()).anyMatch(w -> w.contains("BEST_EFFORT"));
+      assertThat(result.isUsable()).isTrue();
+
+      BackupManifest manifest = BackupManifest.read(BackupsTestUtils.BACKUPS.getBytes(result.getManifestKey()));
+      assertThat(manifest.runKey()).isEqualTo(result.getRunKey());
+      assertThat(manifest.restoreOrder()).containsSubsequence("author", "blogPost");
+      assertThat(manifest.table("userSession")).as("ephemeral tables are not backed up").isNull();
+      assertThat(manifest.table("backups_plan")).as("the qbit own tables are not backed up").isNull();
+
+      BackupManifest.TableEntry posts = manifest.table("blogPost");
+      assertThat(posts.rowCount()).isEqualTo(7);
+      assertThat(posts.excludedFields()).extracting(BackupManifest.ExcludedField::name).containsExactly("secretNote");
+      assertThat(posts.fingerprint()).startsWith("sha256:");
+
+      BackupManifest.Part part = posts.parts().get(0);
+      byte[] stored = BackupsTestUtils.BACKUPS.getBytes(result.getRunPrefix() + part.key());
+      assertThat(com.kingsrook.qbits.backups.codec.HashingOutputStream.sha256Hex(stored)).isEqualTo(part.sha256Stored());
+      String text = gunzip(stored);
+      assertThat(text.split("\n")).hasSize(7);
+      assertThat(text).doesNotContain("do-not-leak").doesNotContain("secretNote");
+      assertThat(text.split("\n")[0]).startsWith("{\"authorId\":2,\"body\":\"Body of 1, with a comma\",\"createDate\":\"20").contains(",\"id\":1,\"mediaKey\":\"media/photo-1.jpg\",\"price\":\"13.3400\"");
+      assertThat(text).contains("\"price\":\"13.3400\"").contains("\"publishedOn\":\"2026-01-02\"");
+      assertThat(com.kingsrook.qbits.backups.codec.HashingOutputStream.sha256Hex(text.getBytes(StandardCharsets.UTF_8))).isEqualTo(part.sha256Raw());
+
+      String checksums = new String(BackupsTestUtils.BACKUPS.getBytes(result.getRunPrefix() + "checksums.sha256"), StandardCharsets.UTF_8);
+      assertThat(checksums).contains(part.sha256Stored() + "  " + part.key());
+
+      QRecord run = new BackupCatalog().get(BackupsTables.RUN, result.getRunId()).orElseThrow();
+      assertThat(run.getValueString("status")).isEqualTo("SUCCEEDED_WITH_WARNINGS");
+      assertThat(run.getValueString("manifestKey")).isEqualTo(result.getManifestKey());
+      assertThat(run.getValueLong("rowsTotal")).isEqualTo(10L);
+      assertThat(new BackupCatalog().where(BackupsTables.ARTIFACT, "backupRunId", result.getRunId())).hasSizeGreaterThanOrEqualTo(4);
+
+      QRecord plan = new BackupCatalog().where(BackupsTables.PLAN, "planKey", "nightly").get(0);
+      assertThat(plan.getValueInteger("lastSuccessRunId")).isEqualTo(result.getRunId());
+      assertThat(plan.getValueString("slaStatus")).isEqualTo("OK");
+   }
+
+
+
+   /*******************************************************************************
+    ** Referenced files are copied into the content-addressed store; the second run
+    ** copies nothing new (deduplicated), and rows referencing the same bytes share one object.
+    *******************************************************************************/
+   @Test
+   void testFilesAreCopiedThenDeduplicated() throws Exception
+   {
+      QRecord         plan  = plan(Map.of());
+      BackupRunResult first = run(plan);
+      assertThat(first.getFilesReferenced()).isEqualTo(4);
+      assertThat(first.getFilesCopied()).isEqualTo(3);
+      assertThat(first.getFilesDeduplicated()).isEqualTo(1);
+      assertThat(first.getFilesMissing()).isZero();
+
+      List<QRecord> fileObjects = new BackupCatalog().query(BackupsTables.FILE_OBJECT, new com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter());
+      assertThat(fileObjects).hasSize(3);
+      for(QRecord fileObject : fileObjects)
+      {
+         byte[] stored = BackupsTestUtils.BACKUPS.getBytes(fileObject.getValueString("objectKey"));
+         assertThat(com.kingsrook.qbits.backups.codec.HashingOutputStream.sha256Hex(stored)).isEqualTo(fileObject.getValueString("sha256"));
+         assertThat(fileObject.getValueString("objectKey")).contains("/_files/sha256/" + fileObject.getValueString("sha256").substring(0, 2) + "/");
+      }
+
+      BackupRunResult second = run(plan);
+      assertThat(second.getFilesCopied()).isZero();
+      assertThat(second.getFilesDeduplicated()).isEqualTo(4);
+      assertThat(new BackupCatalog().query(BackupsTables.FILE_OBJECT, new com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter())).hasSize(3);
+
+      String index = gunzip(BackupsTestUtils.BACKUPS.getBytes(first.getRunPrefix() + "files/index-00001.jsonl.gz"));
+      assertThat(index.split("\n")).hasSize(4);
+      assertThat(index).contains("\"status\":\"COPIED\"").contains("\"status\":\"DEDUPLICATED\"");
+   }
+
+
+
+   /*******************************************************************************
+    ** REFERENCE_ONLY records the key and size without copying; a missing source is
+    ** listed and (under WARN) the run ends with warnings.
+    *******************************************************************************/
+   @Test
+   void testReferenceOnlyAndMissingFiles() throws Exception
+   {
+      BackupRunResult refOnly = run(plan(Map.of("fileObjectPolicy", "REFERENCE_ONLY")));
+      assertThat(refOnly.getFilesCopied()).isZero();
+      assertThat(new BackupCatalog().query(BackupsTables.FILE_OBJECT, new com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter())).isEmpty();
+      assertThat(gunzip(BackupsTestUtils.BACKUPS.getBytes(refOnly.getRunPrefix() + "files/index-00001.jsonl.gz"))).contains("REFERENCE_ONLY");
+
+      BackupsTestUtils.MEDIA.removeIgnoringLock("media/photo-1.jpg");
+      BackupRunResult missing = run(new BackupCatalog().where(BackupsTables.PLAN, "planKey", "nightly").get(0));
+      assertThat(missing.getFilesMissing()).isGreaterThan(0);
+      assertThat(missing.getWarnings()).anyMatch(w -> w.contains("missing"));
+      assertThat(missing.getStatus()).isEqualTo("SUCCEEDED_WITH_WARNINGS");
+   }
+
+
+
+   /*******************************************************************************
+    ** A storage failure mid-run leaves the run FAILED with no manifest (so it is
+    ** not usable and the checker treats the prefix as debris).
+    *******************************************************************************/
+   @Test
+   void testStorageFailureLeavesNoManifest() throws Exception
+   {
+      BackupsTestUtils.BACKUPS.withFailWritesFor(key -> key.contains("/tables/blogPost/"));
+      try
+      {
+         BackupRunResult result = run(plan(Map.of()));
+         assertThat(result.getStatus()).isEqualTo("FAILED");
+         assertThat(result.isUsable()).isFalse();
+         assertThat(result.getErrorMessage()).contains("injected write failure");
+         assertThat(BackupsTestUtils.BACKUPS.list(result.getRunPrefix() + "manifest.json")).isEmpty();
+         assertThat(new BackupCatalog().get(BackupsTables.RUN, result.getRunId()).orElseThrow().getValueString("status")).isEqualTo("FAILED");
+      }
+      finally
+      {
+         BackupsTestUtils.BACKUPS.withFailWritesFor(null);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Required snapshots fail before anything is written (no backend can open one yet).
+    *******************************************************************************/
+   @Test
+   void testSnapshotRequiredFailsBeforeWriting() throws Exception
+   {
+      BackupRunResult result = run(plan(Map.of("consistency", "SNAPSHOT_REQUIRED")));
+      assertThat(result.getStatus()).isEqualTo("FAILED");
+      assertThat(BackupsTestUtils.BACKUPS.list("")).isEmpty();
+   }
+
+
+
+   /*******************************************************************************
+    ** CSV (and gzip) output, a page size smaller than the table (keyset paging), part
+    ** rotation by rows, and the rowsHash staying the same whatever the format.
+    *******************************************************************************/
+   @Test
+   void testCsvAndPaging() throws Exception
+   {
+      BackupRunResult jsonl = run(plan(Map.of("pageSize", 2)));
+      BackupRunResult csv   = run(plan(Map.of("name", "Csv", "planKey", "csv", "format", "CSV", "pageSize", 3)));
+
+      BackupManifest jm = BackupManifest.read(BackupsTestUtils.BACKUPS.getBytes(jsonl.getManifestKey()));
+      BackupManifest cm = BackupManifest.read(BackupsTestUtils.BACKUPS.getBytes(csv.getManifestKey()));
+      assertThat(cm.table("blogPost").rowsHash()).isEqualTo(jm.table("blogPost").rowsHash());
+      assertThat(cm.table("blogPost").parts().get(0).key()).endsWith(".csv.gz");
+
+      String text = gunzip(BackupsTestUtils.BACKUPS.getBytes(csv.getRunPrefix() + cm.table("blogPost").parts().get(0).key()));
+      List<String> records = com.kingsrook.qbits.backups.codec.CsvRowCodec.splitRecords(text);
+      assertThat(records).hasSize(8);
+      assertThat(records.get(0)).startsWith("authorId,body,createDate");
+      String authors = gunzip(BackupsTestUtils.BACKUPS.getBytes(csv.getRunPrefix() + cm.table("author").parts().get(0).key()));
+      assertThat(com.kingsrook.qbits.backups.codec.CsvRowCodec.splitRecords(authors)).hasSize(4);
+      assertThat(authors).contains("\\N").contains("\"Line one\nLine two, with \"\"quotes\"\"");
+   }
+
+
+
+   /*******************************************************************************
+    ** Scope: explicit tables with a filter and field exclusions, and SECRET never included.
+    *******************************************************************************/
+   @Test
+   void testScopeRulesFilterAndExclusions() throws Exception
+   {
+      QRecord plan = plan(Map.of("scopeType", "TABLES"));
+      new BackupCatalog().insert(BackupsTables.PLAN_SCOPE, Map.of("backupPlanId", plan.getValueInteger("id"), "action", "INCLUDE", "selectorType", "TABLE",
+         "selectorValue", "blogPost", "filterJson", "{\"criteria\":[{\"fieldName\":\"published\",\"operator\":\"EQUALS\",\"values\":[true]}]}",
+         "includeAssociatedChildren", false, "fieldExclusions", "body"));
+      BackupRunResult result = run(plan);
+      BackupManifest  m      = BackupManifest.read(BackupsTestUtils.BACKUPS.getBytes(result.getManifestKey()));
+      assertThat(m.tables()).extracting(BackupManifest.TableEntry::name).containsExactly("blogPost");
+      assertThat(m.table("blogPost").rowCount()).isEqualTo(3);
+      assertThat(m.table("blogPost").excludedFields()).extracting(BackupManifest.ExcludedField::name).contains("body", "secretNote");
+      assertThat(gunzip(BackupsTestUtils.BACKUPS.getBytes(result.getRunPrefix() + m.table("blogPost").parts().get(0).key()))).doesNotContain("\"body\"");
+
+      ScopeResolver.ResolvedScope scope = new ScopeResolver(QContext.getQInstance()).resolve(PlanSettings.adHoc().with("scopeType", "TABLES"),
+         List.of(new ScopeResolver.ScopeRule(true, "TABLE", "backups_plan", null, false, null)));
+      assertThat(scope.tables()).isEmpty();
+      assertThat(scope.excluded()).extracting(ScopeResolver.ExcludedTable::reason).containsExactly("BACKUPS");
+   }
+}