From 5b7106b981afda520c975606aaaa9d8eccd2e6d4 Mon Sep 17 00:00:00 2001 From: James Maes Date: Tue, 13 Oct 2026 20:36:44 -0500 Subject: [PATCH 1/2] refactor(backups): core TableSchemaFingerprint and KeysetPager (QQQ 4.1.0-RC.6) Replace the RC.6 stand-ins in TableFingerprints, TableReader and StandardRestoreWriter with core TableSchemaFingerprint and KeysetPager. Core fingerprints differ from the local ones: descriptor names, unique keys as lists, association joinName, and empty-string backend types. New manifests therefore record fingerprintAlgorithm "qqq-table-schema-v1". A manifest without the field uses the frozen LegacyTableFingerprint, so old backups still verify and restore. Paging keeps ascending primary-key order, exclusive resume cursors, bounded pages and the caller's transaction. Restore writes now require the explicit backups.restore permission, checked during Review. Closes #19 (architecture review QRun-IO/qqq#1150, finding F3). Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- CHANGELOG.md | 5 +- README.md | 11 +- docs/RC6-MIGRATION.md | 100 ++++++++++ pom.xml | 4 +- .../backups/manifest/BackupManifest.java | 1 + .../manifest/LegacyTableFingerprint.java | 170 +++++++++++++++++ .../backups/manifest/TableFingerprints.java | 171 ++++++------------ .../qbits/backups/restore/RestoreEngine.java | 3 +- .../qbits/backups/restore/RestorePass.java | 6 +- .../qbits/backups/restore/RestoreWriter.java | 5 +- .../restore/StandardRestoreWriter.java | 113 ++---------- .../qbits/backups/run/BackupEngine.java | 4 +- .../qbits/backups/run/TableReader.java | 76 +++----- .../qbits/backups/verify/BackupVerifier.java | 4 +- .../qbits/backups/BackupsTestUtils.java | 2 +- .../manifest/TableFingerprintsTest.java | 65 +++++++ .../restore/LegacyManifestRestoreTest.java | 123 +++++++++++++ .../backups/restore/RestoreSafetyTest.java | 18 ++ .../qbits/backups/run/BackupEngineTest.java | 4 +- .../qbits/backups/run/TableReaderTest.java | 163 +++++++++++++++++ .../resources/manifests/legacy-schema.json | 130 +++++++++++++ 22 files changed, 896 insertions(+), 284 deletions(-) create mode 100644 docs/RC6-MIGRATION.md create mode 100644 src/main/java/com/kingsrook/qbits/backups/manifest/LegacyTableFingerprint.java create mode 100644 src/test/java/com/kingsrook/qbits/backups/manifest/TableFingerprintsTest.java create mode 100644 src/test/java/com/kingsrook/qbits/backups/restore/LegacyManifestRestoreTest.java create mode 100644 src/test/java/com/kingsrook/qbits/backups/run/TableReaderTest.java create mode 100644 src/test/resources/manifests/legacy-schema.json diff --git a/AGENTS.md b/AGENTS.md index 4e2d578..792b0fb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -35,7 +35,7 @@ This file provides guidance to Codex when working with code in this repository. ## Dependencies - Java 21. -- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). The default-active `qqq-snapshot` profile imports `qqq-bom-pom:4.1.0-RC.5` ahead of it, as in qbit-quick-search (ADR-0007). Remove that activation when the parent moves to 4.1. +- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). The default-active `qqq-snapshot` profile imports `qqq-bom-pom:4.1.0-RC.6` ahead of it, as in qbit-quick-search (ADR-0007). Remove that activation when the parent moves to 4.1. - Tests: JUnit 5, AssertJ. ## Release diff --git a/CHANGELOG.md b/CHANGELOG.md index 141ed86..697db61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,4 +11,7 @@ All notable changes to this project are documented here. The format follows Keep - Build against the released QQQ BOM 4.1.0-RC.5; replace reflective alerts binding with core QAlerts and real sink integration tests. - Register backup alert types for every instance while preserving custom backup sinks and no-op behavior without a configured core sink. - Use RC.5-valid alert names `backups.run.missing-in-storage` and `backups.plan.owner-invalid` (and matching deduplication keys). Hosts using the old underscore names must update routing rules. -- Mark fingerprint, keyset reader, and restore bridges for removal in RC.6. +- Target QQQ 4.1.0-RC.6 and replace the fingerprint, keyset pager and reflective restore bridges with typed core APIs. +- New manifests identify `qqq-table-schema-v1` and carry core schema descriptors; restore and content verification still accept pre-RC.6 manifests with the original fingerprint algorithm. +- Preserve primary-key ordering, exclusive resume cursors, bounded pages and caller-owned transactions; reject non-orderable primary keys and scope filters with skip/limit. +- Require the core `backups.restore` permission during Review when using a core restore writer, including anonymous-mode hosts; preserve supplied timestamps through `RestoreWriteFlag`. diff --git a/README.md b/README.md index c0ffdb9..bae86d2 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ ## Requirements - Java 21. -- QQQ 4.1 (built against `4.1.0-RC.5`). +- QQQ 4.1 (targets `4.1.0-RC.6`, not yet released). ## Install @@ -36,6 +36,15 @@ Follow the QQQ [adoption checklist](https://github.com/QRun-IO/qqq/blob/develop/ ./mvnw verify ``` +Before RC.6 is released, verify against the locally installed core build only: + +```bash +./mvnw -B verify -Dqqq.snapshot.version=4.1.0-ARCHRC6-SNAPSHOT +``` + +The POM stays on RC.6. See [the RC.6 migration notes](docs/RC6-MIGRATION.md) for +fingerprint compatibility, paging guarantees and restore permissions. + Always use the Maven wrapper (Maven 3.9.11); Maven 3.10 cannot read the parent POM. ## License diff --git a/docs/RC6-MIGRATION.md b/docs/RC6-MIGRATION.md new file mode 100644 index 0000000..933c5a2 --- /dev/null +++ b/docs/RC6-MIGRATION.md @@ -0,0 +1,100 @@ +# RC.6 core migration (issue #19, F3) + +The POM targets `4.1.0-RC.6`. Until that release is available, local verification +uses `./mvnw -B verify -Dqqq.snapshot.version=4.1.0-ARCHRC6-SNAPSHOT` against the +installed build from `qqq-wt-archrc6`. No core source changes are needed. + +## Fingerprint compatibility + +The algorithms are **not byte-compatible**, even for identical table metadata: + +| Descriptor property | Pre-RC.6 backup algorithm | Core algorithm | +| --- | --- | --- | +| Table and key | `name`, `primaryKey` | `table`, `primaryKeyField` | +| Fields | Map keyed by name | Sorted list with explicit `name` | +| Field flags | `required`, `pvs`, `heavy`, `sensitive` | `isRequired`, `possibleValueSource`, `isHeavy`, `isSensitive` | +| Unique keys | Sorted `+`-joined strings | Sorted lists of field names | +| Associations | Sorted `name->table` strings | Objects with name, associatedTable and joinName | +| Null backend type | JSON null | Empty string | +| Escaping | Historical hand-written escaping | Jackson JSON escaping | + +New table entries contain `fingerprintAlgorithm: "qqq-table-schema-v1"`, the core +`schema` descriptor, and `fingerprint: "sha256:" + core.getHash()`. The outer +manifest remains version 1; row parts, checksums, compression and storage paths +are unchanged. The instance/catalog hash still combines the ordered table +fingerprint strings, so its value changes too. New manifests require this RC.6 +reader; older readers do not understand the new descriptor shape. + +An absent algorithm identifies the old `qbit-backups-table-v1` algorithm. The +reader also accepts that explicit identifier. `LegacyTableFingerprint` freezes +the original descriptor and hash implementation solely for comparison against +existing backups; no new backup uses it. Restore and content verification select +the declared algorithm. Unknown identifiers never match a known fingerprint. +Restore normalizes either field shape for its existing compatibility analysis +when a table has changed. Fingerprint equality is not required for a compatible +schema migration, as before. + +`src/test/resources/manifests/legacy-schema.json` freezes descriptors and hashes +produced by the original stand-in for the blog fixture. Tests place these into a +real backup manifest without the algorithm field, verify its content, mutate the +live data, then restore and compare the complete content hash including dates. +They also cover optional additions and incompatible type changes in old schemas. + +## Paging and consistency + +`TableReader.Keyset` delegates to core `KeysetPager`. Every query uses +`(scope filter) AND pk > lastKey ORDER BY pk ASC LIMIT pageSize`, with no offset. +The resume key is exclusive. Page size is a row bound, **not a JDBC fetch-size +setting**. Core's normal bounded QueryAction path retains backend driver defaults; +this adapter does not opt into the large-result streaming query hint or change +JDBC fetch size. Heavy/hidden/password fields and untranslated stored values keep +the same query settings and existing backup permission boundary. + +Every page receives the caller's transaction. The pager does not open, commit, +close, or replace that transaction. A test opens a real core memory snapshot, +changes a later page's data, and proves every page still sees the original state. +The existing BackupEngine continues to report BEST_EFFORT and reject +SNAPSHOT_REQUIRED: adopting the consistent-read opening API at engine level is +separate work. This migration does not claim snapshot guarantees for ordinary +backup runs. + +Core can stream non-orderable keys without deterministic key order; backups +explicitly reject those keys instead. Core rejects scope filters carrying skip +or limit, and replaces user ordering with ascending primary-key ordering. Tests +cover OR filters, resume, full and partial pages, empty scans and invalid inputs. + +## Restore writes + +`StandardRestoreWriter` now constructs `RestoreWriteFlag` directly and calls +`QBackendModuleInterface.resyncKeySequences`, consuming typed `SequenceResyncResult` +values. Only RESYNCED results are reported as sequence changes; NOT_NEEDED and +NOT_APPLICABLE remain normal outcomes. There is no reflection or unavailable-core +fallback. The core flag preserves supplied dates and carries the requested +customizer, automation, audit and event settings. + +Core always requires an actual session permission `backups.restore` for its +restore flag, including hosts with general core enforcement disabled. The qbit's +anonymous-mode convenience setting does not grant that permission. Review now +reports this requirement before executing a restore; test sessions explicitly +hold it. Custom writers that do not use the core flag keep their existing qbit +permission checks. + +## HTTP operations and wire changes + +No HTTP operations are added, removed or modified: method, path and operationId +are unchanged (no operation list entries for this migration). The RC.6 UI API +extension documentation was reviewed; this task introduces no Javalin route or +middleware extension. Wire changes are confined to the manifest fields and +fingerprint values described above, plus an earlier restore permission error in +Review for anonymous sessions lacking `backups.restore`. + +## Local verification (2026-10-13) + +- `./mvnw -B clean verify -Dqqq.snapshot.version=4.1.0-ARCHRC6-SNAPSHOT`: + BUILD SUCCESS; 158 tests, zero failures/errors/skips; both JaCoCo gates passed. + Existing backup, restore and MinIO integration tests ran. +- Separate Checkstyle 14.3.0 audit of all 16 changed Java files (main and test): + zero violations, using the shared Kingsrook-derived rules with this repository's + Apache file header. This repository has no Checkstyle execution configured in + its POM; audit configuration and results are local under `.cache/style/`. +- `git diff --check`: passed. No commit, push or core source modification. diff --git a/pom.xml b/pom.xml index b000843..f1b90f6 100644 --- a/pom.xml +++ b/pom.xml @@ -192,14 +192,14 @@ - + qqq-snapshot true - 4.1.0-RC.5 + 4.1.0-RC.6 diff --git a/src/main/java/com/kingsrook/qbits/backups/manifest/BackupManifest.java b/src/main/java/com/kingsrook/qbits/backups/manifest/BackupManifest.java index 24eaa4d..5547c08 100644 --- a/src/main/java/com/kingsrook/qbits/backups/manifest/BackupManifest.java +++ b/src/main/java/com/kingsrook/qbits/backups/manifest/BackupManifest.java @@ -137,6 +137,7 @@ public record BackupManifest( String backend, String primaryKey, String fingerprint, + String fingerprintAlgorithm, Map schema, String filter, List excludedFields, diff --git a/src/main/java/com/kingsrook/qbits/backups/manifest/LegacyTableFingerprint.java b/src/main/java/com/kingsrook/qbits/backups/manifest/LegacyTableFingerprint.java new file mode 100644 index 0000000..d0775f1 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/backups/manifest/LegacyTableFingerprint.java @@ -0,0 +1,170 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.manifest; + + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import com.kingsrook.qbits.backups.codec.HashingOutputStream; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; + + +/******************************************************************************* + ** Frozen pre-RC.6 fingerprint algorithm, used only to read existing manifests. + ** Keep its descriptor and escaping byte-compatible with the original writer. + *******************************************************************************/ +final class LegacyTableFingerprint +{ + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private LegacyTableFingerprint() + { + } + + + + /******************************************************************************* + ** The descriptor of a table (sorted fields; plain maps and lists, so it is also + ** what goes in the manifest schema). + *******************************************************************************/ + public static Map descriptor(QTableMetaData table, String backendType) + { + Map d = new LinkedHashMap<>(); + d.put("name", table.getName()); + d.put("primaryKey", table.getPrimaryKeyField()); + d.put("backendType", backendType); + + Map fields = new TreeMap<>(); + for(QFieldMetaData field : new TreeMap<>(table.getFields()).values()) + { + Map f = new LinkedHashMap<>(); + f.put("type", field.getType().name()); + f.put("required", field.getIsRequired()); + f.put("maxLength", field.getMaxLength()); + f.put("pvs", field.getPossibleValueSourceName()); + f.put("heavy", field.getIsHeavy()); + f.put("sensitive", field.getIsSensitive()); + fields.put(field.getName(), f); + } + d.put("fields", fields); + + List uniqueKeys = new ArrayList<>(); + if(table.getUniqueKeys() != null) + { + for(UniqueKey key : table.getUniqueKeys()) + { + uniqueKeys.add(String.join("+", key.getFieldNames().stream().sorted().toList())); + } + } + uniqueKeys.sort(String::compareTo); + d.put("uniqueKeys", uniqueKeys); + + List associations = new ArrayList<>(); + if(table.getAssociations() != null) + { + for(Association association : table.getAssociations()) + { + associations.add(association.getName() + "->" + association.getAssociatedTableName()); + } + } + associations.sort(String::compareTo); + d.put("associations", associations); + return (d); + } + + + + /******************************************************************************* + ** The fingerprint of a table: sha256:hex of the canonical descriptor. + *******************************************************************************/ + public static String of(QTableMetaData table, String backendType) + { + return (fingerprintOf(descriptor(table, backendType))); + } + + + + /******************************************************************************* + ** The fingerprint of a descriptor (maps sorted by key at every level, no whitespace). + *******************************************************************************/ + public static String fingerprintOf(Map descriptor) + { + StringBuilder sb = new StringBuilder(); + canonical(descriptor, sb); + return ("sha256:" + HashingOutputStream.sha256Hex(sb.toString().getBytes(StandardCharsets.UTF_8))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + private static void canonical(Object o, StringBuilder sb) + { + if(o == null) + { + sb.append("null"); + } + else if(o instanceof Map map) + { + sb.append('{'); + boolean first = true; + for(Map.Entry e : new TreeMap<>((Map) map).entrySet()) + { + if(!first) + { + sb.append(','); + } + first = false; + sb.append('"').append(e.getKey()).append("\":"); + canonical(e.getValue(), sb); + } + sb.append('}'); + } + else if(o instanceof List list) + { + sb.append('['); + for(int i = 0; i < list.size(); i++) + { + if(i > 0) + { + sb.append(','); + } + canonical(list.get(i), sb); + } + sb.append(']'); + } + else if(o instanceof Number || o instanceof Boolean) + { + sb.append(o); + } + else + { + sb.append('"').append(String.valueOf(o).replace("\\", "\\\\").replace("\"", "\\\"")).append('"'); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/backups/manifest/TableFingerprints.java b/src/main/java/com/kingsrook/qbits/backups/manifest/TableFingerprints.java index a8f3268..1f752d1 100644 --- a/src/main/java/com/kingsrook/qbits/backups/manifest/TableFingerprints.java +++ b/src/main/java/com/kingsrook/qbits/backups/manifest/TableFingerprints.java @@ -18,29 +18,23 @@ package com.kingsrook.qbits.backups.manifest; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.TreeMap; import com.kingsrook.qbits.backups.codec.HashingOutputStream; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import com.kingsrook.qqq.backend.core.model.metadata.tables.TableSchemaFingerprint; /******************************************************************************* - ** STAND-IN(core=QRun-IO/qqq#1113, removeBy=4.1.0-RC.6) - ** A stable schema descriptor and SHA-256 fingerprint of a table (spec 5.5). - ** - ** This is an internal stand-in for the QQQ core TableSchemaFingerprint (core PR - ** feature/schema-fingerprint). When that ships in the next 4.1.x RC, this class - ** delegates to it and the descriptor below is kept only for reading old manifests. - ** Labels, adornments and help text are excluded, so cosmetic changes do not count. + ** Core schema fingerprints with read compatibility for pre-RC.6 manifests. *******************************************************************************/ public final class TableFingerprints { + public static final String CORE_ALGORITHM = "qqq-table-schema-v1"; + public static final String LEGACY_ALGORITHM = "qbit-backups-table-v1"; + + /******************************************************************************* ** Utility class. @@ -52,134 +46,83 @@ public final class TableFingerprints /******************************************************************************* - ** The descriptor of a table (sorted fields; plain maps and lists, so it is also - ** what goes in the manifest schema). + ** The core descriptor stored in new manifests. *******************************************************************************/ public static Map descriptor(QTableMetaData table, String backendType) { - Map d = new LinkedHashMap<>(); - d.put("name", table.getName()); - d.put("primaryKey", table.getPrimaryKeyField()); - d.put("backendType", backendType); - - Map fields = new TreeMap<>(); - for(QFieldMetaData field : new TreeMap<>(table.getFields()).values()) - { - Map f = new LinkedHashMap<>(); - f.put("type", field.getType().name()); - f.put("required", field.getIsRequired()); - f.put("maxLength", field.getMaxLength()); - f.put("pvs", field.getPossibleValueSourceName()); - f.put("heavy", field.getIsHeavy()); - f.put("sensitive", field.getIsSensitive()); - fields.put(field.getName(), f); - } - d.put("fields", fields); - - List uniqueKeys = new ArrayList<>(); - if(table.getUniqueKeys() != null) - { - for(UniqueKey key : table.getUniqueKeys()) - { - uniqueKeys.add(String.join("+", key.getFieldNames().stream().sorted().toList())); - } - } - uniqueKeys.sort(String::compareTo); - d.put("uniqueKeys", uniqueKeys); - - List associations = new ArrayList<>(); - if(table.getAssociations() != null) - { - for(Association association : table.getAssociations()) - { - associations.add(association.getName() + "->" + association.getAssociatedTableName()); - } - } - associations.sort(String::compareTo); - d.put("associations", associations); - return (d); + return (TableSchemaFingerprint.of(table, backendType).getDescriptor()); } /******************************************************************************* - ** The fingerprint of a table: sha256:hex of the canonical descriptor. + ** Retain the manifest's sha256 prefix around the core hash. *******************************************************************************/ public static String of(QTableMetaData table, String backendType) { - return (fingerprintOf(descriptor(table, backendType))); + return ("sha256:" + TableSchemaFingerprint.of(table, backendType).getHash()); } /******************************************************************************* - ** The fingerprint of a descriptor (maps sorted by key at every level, no whitespace). + ** An absent algorithm identifies manifests written before RC.6. Unknown + ** algorithms must not silently fall back to a different fingerprint. *******************************************************************************/ - public static String fingerprintOf(Map descriptor) + public static boolean matches(BackupManifest.TableEntry entry, QTableMetaData table, String backendType) { - StringBuilder sb = new StringBuilder(); - canonical(descriptor, sb); - return ("sha256:" + HashingOutputStream.sha256Hex(sb.toString().getBytes(StandardCharsets.UTF_8))); + if(entry.fingerprint() == null) + { + return (false); + } + String algorithm = entry.fingerprintAlgorithm(); + if(algorithm == null || LEGACY_ALGORITHM.equals(algorithm)) + { + return (entry.fingerprint().equals(LegacyTableFingerprint.of(table, backendType))); + } + return (CORE_ALGORITHM.equals(algorithm) && entry.fingerprint().equals(of(table, backendType))); } /******************************************************************************* - ** The fingerprint of a whole instance's tables: a hash of the table fingerprints. + ** Normalize both descriptor versions to field-name maps for restore's + ** compatibility analysis. Core fields use a list and explicit flag names. + *******************************************************************************/ + @SuppressWarnings("unchecked") + public static Map fields(Map descriptor) + { + Object fields = descriptor == null ? null : descriptor.get("fields"); + if(fields instanceof Map map) + { + return ((Map) map); + } + Map result = new LinkedHashMap<>(); + if(fields instanceof List list) + { + for(Object item : list) + { + Map field = (Map) item; + Map normalized = new LinkedHashMap<>(); + normalized.put("type", field.get("type")); + normalized.put("required", field.get("isRequired")); + normalized.put("maxLength", field.get("maxLength")); + normalized.put("pvs", field.get("possibleValueSource")); + normalized.put("heavy", field.get("isHeavy")); + normalized.put("sensitive", field.get("isSensitive")); + result.put((String) field.get("name"), normalized); + } + } + return (result); + } + + + + /******************************************************************************* + ** The instance fingerprint remains a hash of the ordered table fingerprints. *******************************************************************************/ public static String combine(List tableFingerprints) { return ("sha256:" + HashingOutputStream.sha256Hex(String.join("\n", tableFingerprints).getBytes(StandardCharsets.UTF_8))); } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @SuppressWarnings("unchecked") - private static void canonical(Object o, StringBuilder sb) - { - if(o == null) - { - sb.append("null"); - } - else if(o instanceof Map map) - { - sb.append('{'); - boolean first = true; - for(Map.Entry e : new TreeMap<>((Map) map).entrySet()) - { - if(!first) - { - sb.append(','); - } - first = false; - sb.append('"').append(e.getKey()).append("\":"); - canonical(e.getValue(), sb); - } - sb.append('}'); - } - else if(o instanceof List list) - { - sb.append('['); - for(int i = 0; i < list.size(); i++) - { - if(i > 0) - { - sb.append(','); - } - canonical(list.get(i), sb); - } - sb.append(']'); - } - else if(o instanceof Number || o instanceof Boolean) - { - sb.append(o); - } - else - { - sb.append('"').append(String.valueOf(o).replace("\\", "\\\\").replace("\"", "\\\"")).append('"'); - } - } } diff --git a/src/main/java/com/kingsrook/qbits/backups/restore/RestoreEngine.java b/src/main/java/com/kingsrook/qbits/backups/restore/RestoreEngine.java index fcf839f..726a8ee 100644 --- a/src/main/java/com/kingsrook/qbits/backups/restore/RestoreEngine.java +++ b/src/main/java/com/kingsrook/qbits/backups/restore/RestoreEngine.java @@ -105,7 +105,8 @@ public class RestoreEngine Map targets = new LinkedHashMap<>(); QInstance instance = QContext.getQInstance(); - if(!BackupsPermissionChecker.has(BackupsPermissions.RESTORE)) + if(!BackupsPermissionChecker.has(BackupsPermissions.RESTORE) + || (request.getWriter().usesCoreRestoreFlag() && (QContext.getQSession() == null || !QContext.getQSession().hasPermission(BackupsPermissions.RESTORE)))) { errors.add("Restoring needs the " + BackupsPermissions.RESTORE + " permission."); } diff --git a/src/main/java/com/kingsrook/qbits/backups/restore/RestorePass.java b/src/main/java/com/kingsrook/qbits/backups/restore/RestorePass.java index d76eb72..b5d704e 100644 --- a/src/main/java/com/kingsrook/qbits/backups/restore/RestorePass.java +++ b/src/main/java/com/kingsrook/qbits/backups/restore/RestorePass.java @@ -341,14 +341,14 @@ public class RestorePass String backendType = instance.getBackend(c.target.getBackendName()) == null ? null : instance.getBackend(c.target.getBackendName()).getBackendType(); RestoreOutcome.TableOutcome o = c.outcome; - if(c.entry.fingerprint() != null && c.entry.fingerprint().equals(TableFingerprints.of(c.target, backendType))) + if(TableFingerprints.matches(c.entry, c.target, backendType)) { o.setSchemaStatus("MATCH"); return; } - Map backupFields = c.entry.schema() == null ? Map.of() : (Map) c.entry.schema().getOrDefault("fields", Map.of()); - Map targetFields = (Map) TableFingerprints.descriptor(c.target, backendType).get("fields"); + Map backupFields = TableFingerprints.fields(c.entry.schema()); + Map targetFields = TableFingerprints.fields(TableFingerprints.descriptor(c.target, backendType)); boolean incompatible = false; boolean different = false; diff --git a/src/main/java/com/kingsrook/qbits/backups/restore/RestoreWriter.java b/src/main/java/com/kingsrook/qbits/backups/restore/RestoreWriter.java index d6ffb2f..7bb98e4 100644 --- a/src/main/java/com/kingsrook/qbits/backups/restore/RestoreWriter.java +++ b/src/main/java/com/kingsrook/qbits/backups/restore/RestoreWriter.java @@ -28,9 +28,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; /******************************************************************************* ** How a restore writes (spec 5.3 and 5.4). The default implementation uses the - ** plain Insert, Update and Delete actions, and selects the core RestoreWriteFlag - ** (QRun-IO/qqq#1111) and the core key-sequence resync (#1112) reflectively when the - ** running QQQ has them. A backend specific writer can be supplied on the request. + ** plain Insert, Update and Delete actions with core RestoreWriteFlag and typed + ** key-sequence resync. A backend specific writer can be supplied on the request. ** ** Whatever the implementation, record security, field security, unique keys and ** required-field validation still apply to every write. diff --git a/src/main/java/com/kingsrook/qbits/backups/restore/StandardRestoreWriter.java b/src/main/java/com/kingsrook/qbits/backups/restore/StandardRestoreWriter.java index fad1965..c65c203 100644 --- a/src/main/java/com/kingsrook/qbits/backups/restore/StandardRestoreWriter.java +++ b/src/main/java/com/kingsrook/qbits/backups/restore/StandardRestoreWriter.java @@ -18,7 +18,6 @@ package com.kingsrook.qbits.backups.restore; import java.io.Serializable; -import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Collection; import java.util.LinkedHashMap; @@ -30,8 +29,8 @@ import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.ActionFlag; +import com.kingsrook.qqq.backend.core.model.actions.tables.RestoreWriteFlag; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; @@ -41,50 +40,14 @@ import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; -import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; +import com.kingsrook.qqq.backend.core.modules.backend.SequenceResyncResult; /******************************************************************************* - ** STAND-IN(core=QRun-IO/qqq#1111, removeBy=4.1.0-RC.6) - ** STAND-IN(core=QRun-IO/qqq#1112, removeBy=4.1.0-RC.6) - ** The default RestoreWriter: the standard Insert, Update and Delete actions. - ** - ** Stand-ins (replace with the core API when QQQ 4.1.0-RC.5 ships): - **
    - **
  • RestoreWriteFlag (QRun-IO/qqq#1111) is created reflectively when the class is - ** present. Without it, the DML audit is omitted (a summary audit is written per - ** table instead), but created and modified dates may be set by the normal defaults.
    • - **
  • resyncKeySequences (QRun-IO/qqq#1112) is called reflectively on the backend - ** module when it exists; otherwise it is a no-op that the Review reports.
    • - **
+ ** The default RestoreWriter using core restore flags and key sequence resync. *******************************************************************************/ public class StandardRestoreWriter implements RestoreWriter { - private static final QLogger LOG = QLogger.getLogger(StandardRestoreWriter.class); - - private static final String FLAG_CLASS = "com.kingsrook.qqq.backend.core.model.actions.tables.RestoreWriteFlag"; - - private static final Class FLAG = lookup(FLAG_CLASS); - - - - /******************************************************************************* - ** - *******************************************************************************/ - private static Class lookup(String name) - { - try - { - return (Class.forName(name)); - } - catch(ClassNotFoundException | LinkageError e) - { - return (null); - } - } - - - /******************************************************************************* ** {@inheritDoc} *******************************************************************************/ @@ -150,27 +113,15 @@ public class StandardRestoreWriter implements RestoreWriter /******************************************************************************* - ** Create the core flag (when the running QQQ has it) and hand it to the input. + ** Apply the core restore behavior requested by the caller. *******************************************************************************/ - private void addFlag(RestoreRequest options, java.util.function.Consumer sink) throws QException + private void addFlag(RestoreRequest options, java.util.function.Consumer sink) { - if(FLAG == null) - { - return; - } - try - { - Object flag = FLAG.getDeclaredConstructor().newInstance(); - FLAG.getMethod("withSuppressAutomations", boolean.class).invoke(flag, options.getSuppressAutomations()); - FLAG.getMethod("withRunCustomizers", boolean.class).invoke(flag, options.getRunCustomizers()); - FLAG.getMethod("withPerRecordAudit", boolean.class).invoke(flag, options.getPerRecordAudit()); - FLAG.getMethod("withPublishEvents", boolean.class).invoke(flag, options.getPublishEvents()); - sink.accept((ActionFlag) flag); - } - catch(ReflectiveOperationException | ClassCastException e) - { - throw (new QException("Could not use the core restore flag: " + e.getMessage(), e)); - } + sink.accept(new RestoreWriteFlag() + .withSuppressAutomations(options.getSuppressAutomations()) + .withRunCustomizers(options.getRunCustomizers()) + .withPerRecordAudit(options.getPerRecordAudit()) + .withPublishEvents(options.getPublishEvents())); } @@ -181,7 +132,7 @@ public class StandardRestoreWriter implements RestoreWriter @Override public boolean usesCoreRestoreFlag() { - return (FLAG != null); + return (true); } @@ -192,24 +143,7 @@ public class StandardRestoreWriter implements RestoreWriter @Override public boolean canResyncSequences() { - return (resyncMethod() != null); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private static Method resyncMethod() - { - try - { - return (QBackendModuleInterface.class.getMethod("resyncKeySequences", QBackendMetaData.class, Collection.class)); - } - catch(NoSuchMethodException | LinkageError e) - { - return (null); - } + return (true); } @@ -220,12 +154,6 @@ public class StandardRestoreWriter implements RestoreWriter @Override public List resyncSequences(Collection tables) throws QException { - Method method = resyncMethod(); - if(method == null) - { - return (List.of()); - } - Map> byBackend = new LinkedHashMap<>(); for(QTableMetaData table : tables) { @@ -236,23 +164,14 @@ public class StandardRestoreWriter implements RestoreWriter for(Map.Entry> entry : byBackend.entrySet()) { QBackendMetaData backend = QContext.getQInstance().getBackend(entry.getKey()); - try + QBackendModuleInterface module = new QBackendModuleDispatcher().getQBackendModule(backend); + for(SequenceResyncResult result : module.resyncKeySequences(backend, entry.getValue())) { - QBackendModuleInterface module = new QBackendModuleDispatcher().getQBackendModule(backend); - for(Object result : (List) method.invoke(module, backend, entry.getValue())) + if(result.getStatus() == SequenceResyncResult.Status.RESYNCED) { - Object status = result.getClass().getMethod("getStatus").invoke(result); - if("RESYNCED".equals(String.valueOf(status))) - { - resynced.add(String.valueOf(result.getClass().getMethod("getTableName").invoke(result))); - } + resynced.add(result.getTableName()); } } - catch(ReflectiveOperationException e) - { - LOG.warn("Could not resync key sequences", e, logPair("backend", entry.getKey())); - throw (new QException("Could not resync key sequences on backend " + entry.getKey() + ": " + e.getMessage(), e)); - } } return (resynced); } diff --git a/src/main/java/com/kingsrook/qbits/backups/run/BackupEngine.java b/src/main/java/com/kingsrook/qbits/backups/run/BackupEngine.java index 2062b23..22e1c78 100644 --- a/src/main/java/com/kingsrook/qbits/backups/run/BackupEngine.java +++ b/src/main/java/com/kingsrook/qbits/backups/run/BackupEngine.java @@ -35,8 +35,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.kingsrook.qbits.backups.BackupsRuntime; import com.kingsrook.qbits.backups.alerts.BackupAlertSink; import com.kingsrook.qbits.backups.codec.HashingOutputStream; -import com.kingsrook.qbits.backups.events.BackupEvents; import com.kingsrook.qbits.backups.codec.TableArtifactWriter; +import com.kingsrook.qbits.backups.events.BackupEvents; import com.kingsrook.qbits.backups.files.FileCopier; import com.kingsrook.qbits.backups.files.FileReference; import com.kingsrook.qbits.backups.files.FileReferenceProvider; @@ -398,7 +398,7 @@ public class BackupEngine keyRange.put("min", allResults.isEmpty() ? null : allResults.get(0).firstKey()); keyRange.put("max", allResults.isEmpty() ? null : allResults.get(allResults.size() - 1).lastKey()); manifestTables.add(new BackupManifest.TableEntry(table.getName(), resolved.role().name(), table.getBackendName(), table.getPrimaryKeyField(), - fingerprint, TableFingerprints.descriptor(table, backendType), resolved.filterJson(), excluded, counter[0], rowsHash, keyRange, parts, consistency)); + fingerprint, TableFingerprints.CORE_ALGORITHM, TableFingerprints.descriptor(table, backendType), resolved.filterJson(), excluded, counter[0], rowsHash, keyRange, parts, consistency)); done++; catalogProgress(result, done, rows, bytesRaw, bytesStored); if(request.getCancelRequested().getAsBoolean()) diff --git a/src/main/java/com/kingsrook/qbits/backups/run/TableReader.java b/src/main/java/com/kingsrook/qbits/backups/run/TableReader.java index bd3583d..6460dd2 100644 --- a/src/main/java/com/kingsrook/qbits/backups/run/TableReader.java +++ b/src/main/java/com/kingsrook/qbits/backups/run/TableReader.java @@ -18,31 +18,20 @@ package com.kingsrook.qbits.backups.run; import java.io.Serializable; -import java.util.List; import java.util.function.Consumer; import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; -import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.actions.tables.KeysetPager; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.JsonUtils; /******************************************************************************* - ** STAND-IN(core=QRun-IO/qqq#1114, removeBy=4.1.0-RC.6) - ** Reads the rows of one table in primary key order, in keyset pages (WHERE pk > - ** lastKey ORDER BY pk LIMIT n), so the cost stays flat with depth and a read can - ** resume from a key. A read raw: hidden fields, passwords and heavy fields are - ** returned as stored, and possible values are not translated. - ** - ** This is an internal stand-in for the QQQ core KeysetPager and QueryInput - ** withRawRead (core PR feature/keyset-pager-raw-read); the run engine only - ** depends on this interface, so it is swapped when that ships in the next RC. + ** Reads rows in primary key order through core KeysetPager. Each bounded page + ** uses the supplied transaction and returns hidden, password and heavy fields + ** as stored, without possible-value translation or display generation. *******************************************************************************/ public interface TableReader { @@ -82,48 +71,25 @@ public interface TableReader @Override public void read(QTableMetaData table, String filterJson, Serializable startAfterKey, int pageSize, QBackendTransaction transaction, Consumer sink) throws QException { - String pk = table.getPrimaryKeyField(); - Serializable lastKey = startAfterKey; - QQueryFilter user = parseFilter(filterJson); - - while(true) + KeysetPager pager = KeysetPager.forTable(table.getName()) + .withFilter(parseFilter(filterJson)) + .startingAfter(startAfterKey) + .withPageSize(pageSize) + .withTransaction(transaction) + .withQueryInputCustomizer(input -> + { + input.setShouldOmitHiddenFields(false); + input.setShouldMaskPasswords(false); + input.setShouldFetchHeavyFields(true); + input.setShouldTranslatePossibleValues(false); + input.setShouldGenerateDisplayValues(false); + input.withPermissionBypass("backups run"); + }); + if(!pager.isPrimaryKeyOrderable()) { - QQueryFilter filter = new QQueryFilter(); - if(lastKey != null) - { - filter.addCriteria(new QFilterCriteria(pk, QCriteriaOperator.GREATER_THAN, List.of(lastKey))); - } - if(user != null) - { - filter.addSubFilter(user); - } - filter.withOrderBy(new QFilterOrderBy(pk, true)); - filter.setLimit(pageSize); - - QueryInput queryInput = new QueryInput(table.getName()); - queryInput.setFilter(filter); - queryInput.setShouldOmitHiddenFields(false); - queryInput.setShouldMaskPasswords(false); - queryInput.setShouldFetchHeavyFields(true); - queryInput.setShouldTranslatePossibleValues(false); - queryInput.setShouldGenerateDisplayValues(false); - if(transaction != null) - { - queryInput.setTransaction(transaction); - } - queryInput.withPermissionBypass("backups run"); - - List page = new QueryAction().execute(queryInput).getRecords(); - for(QRecord record : page) - { - sink.accept(record); - lastKey = record.getValue(pk); - } - if(page.size() < pageSize) - { - return; - } + throw (new QException("Backup requires an orderable primary key on table " + table.getName())); } + pager.forEachPage(page -> page.forEach(sink)); } } } diff --git a/src/main/java/com/kingsrook/qbits/backups/verify/BackupVerifier.java b/src/main/java/com/kingsrook/qbits/backups/verify/BackupVerifier.java index ece7cc5..bbf82c8 100644 --- a/src/main/java/com/kingsrook/qbits/backups/verify/BackupVerifier.java +++ b/src/main/java/com/kingsrook/qbits/backups/verify/BackupVerifier.java @@ -33,8 +33,8 @@ import java.util.Optional; import com.fasterxml.jackson.databind.ObjectMapper; import com.kingsrook.qbits.backups.BackupsQBitConfig; import com.kingsrook.qbits.backups.BackupsRuntime; -import com.kingsrook.qbits.backups.alerts.BackupAlerts; import com.kingsrook.qbits.backups.alerts.BackupAlertSink; +import com.kingsrook.qbits.backups.alerts.BackupAlerts; import com.kingsrook.qbits.backups.codec.ArtifactTransform; import com.kingsrook.qbits.backups.codec.HashingOutputStream; import com.kingsrook.qbits.backups.codec.JsonlRowCodec; @@ -274,7 +274,7 @@ public class BackupVerifier List excluded = entry.excludedFields() == null ? List.of() : entry.excludedFields().stream().map(BackupManifest.ExcludedField::name).toList(); codec = new JsonlRowCodec(table, excluded); String backendType = qInstance.getBackend(table.getBackendName()) == null ? null : qInstance.getBackend(table.getBackendName()).getBackendType(); - roundTrip = TableFingerprints.of(table, backendType).equals(entry.fingerprint()); + roundTrip = TableFingerprints.matches(entry, table, backendType); if(!roundTrip) { findings.warn("Table " + entry.name() + " has changed since this backup; rows were parsed but not checked for an exact round trip"); diff --git a/src/test/java/com/kingsrook/qbits/backups/BackupsTestUtils.java b/src/test/java/com/kingsrook/qbits/backups/BackupsTestUtils.java index 88ed3ff..2999abd 100644 --- a/src/test/java/com/kingsrook/qbits/backups/BackupsTestUtils.java +++ b/src/test/java/com/kingsrook/qbits/backups/BackupsTestUtils.java @@ -106,7 +106,7 @@ public final class BackupsTestUtils new BackupsQBitProducer().withQBitConfig(config).withNamespace("backups").produce(qInstance); new QInstanceValidator().validate(qInstance); - QContext.init(qInstance, new QSession()); + QContext.init(qInstance, new QSession().withPermissions("backups.restore")); return (qInstance); } diff --git a/src/test/java/com/kingsrook/qbits/backups/manifest/TableFingerprintsTest.java b/src/test/java/com/kingsrook/qbits/backups/manifest/TableFingerprintsTest.java new file mode 100644 index 0000000..78fede9 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/backups/manifest/TableFingerprintsTest.java @@ -0,0 +1,65 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.manifest; + + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.kingsrook.qbits.backups.BackupsTestUtils; +import com.kingsrook.qqq.backend.core.context.QContext; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Fingerprint algorithm dispatch is explicit; a marker cannot be ignored just + ** because its hash happens to match a different supported algorithm. + *******************************************************************************/ +class TableFingerprintsTest +{ + /******************************************************************************* + ** A frozen old hash matches only the legacy algorithm, and new hashes only + ** match the declared core algorithm. Cosmetic changes preserve both. + *******************************************************************************/ + @Test + void testAlgorithmDispatch() throws Exception + { + BackupsTestUtils.init(); + var table = QContext.getQInstance().getTable("blogPost"); + ObjectMapper mapper = new ObjectMapper(); + ObjectNode entry; + try(var in = getClass().getResourceAsStream("/manifests/legacy-schema.json")) + { + entry = (ObjectNode) mapper.readTree(in).get("blogPost"); + } + String oldHash = entry.get("fingerprint").asText(); + String coreHash = TableFingerprints.of(table, "memory"); + assertThat(coreHash).isNotEqualTo(oldHash); + table.setLabel("Cosmetic label"); + assertThat(TableFingerprints.matches(mapper.treeToValue(entry, BackupManifest.TableEntry.class), table, "memory")).isTrue(); + entry.put("fingerprintAlgorithm", TableFingerprints.LEGACY_ALGORITHM); + assertThat(TableFingerprints.matches(mapper.treeToValue(entry, BackupManifest.TableEntry.class), table, "memory")).isTrue(); + entry.put("fingerprintAlgorithm", TableFingerprints.CORE_ALGORITHM); + assertThat(TableFingerprints.matches(mapper.treeToValue(entry, BackupManifest.TableEntry.class), table, "memory")).isFalse(); + entry.put("fingerprint", coreHash); + assertThat(TableFingerprints.matches(mapper.treeToValue(entry, BackupManifest.TableEntry.class), table, "memory")).isTrue(); + entry.put("fingerprintAlgorithm", "unknown-v2"); + assertThat(TableFingerprints.matches(mapper.treeToValue(entry, BackupManifest.TableEntry.class), table, "memory")).isFalse(); + entry.remove("fingerprint"); + assertThat(TableFingerprints.matches(mapper.treeToValue(entry, BackupManifest.TableEntry.class), table, "memory")).isFalse(); + } +} diff --git a/src/test/java/com/kingsrook/qbits/backups/restore/LegacyManifestRestoreTest.java b/src/test/java/com/kingsrook/qbits/backups/restore/LegacyManifestRestoreTest.java new file mode 100644 index 0000000..8089add --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/backups/restore/LegacyManifestRestoreTest.java @@ -0,0 +1,123 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.restore; + + +import java.util.Map; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.kingsrook.qbits.backups.BackupsTestUtils; +import com.kingsrook.qbits.backups.codec.HashingOutputStream; +import com.kingsrook.qbits.backups.manifest.BackupManifest; +import com.kingsrook.qbits.backups.model.BackupsTables; +import com.kingsrook.qbits.backups.run.BackupCatalog; +import com.kingsrook.qbits.backups.run.BackupRunResult; +import com.kingsrook.qbits.backups.verify.BackupVerifier; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Exercise pre-RC.6 manifest fingerprints through verification and restore. + ** The schema fixture was frozen from the original stand-in, not recomputed + ** with the implementation being tested. + *******************************************************************************/ +class LegacyManifestRestoreTest +{ + /******************************************************************************* + ** Load the blog and grant the permission required by the core restore flag. + *******************************************************************************/ + @BeforeEach + void setUp() throws Exception + { + BackupsTestUtils.init(); + BackupsTestUtils.loadBlog(30); + } + + + + /******************************************************************************* + ** Build an old-format manifest around real backup parts and re-seal its hash. + *******************************************************************************/ + private BackupRunResult legacyBackup() throws Exception + { + BackupRunResult run = RestoreTestSupport.backup(Map.of()); + ObjectMapper mapper = new ObjectMapper(); + ObjectNode manifest = (ObjectNode) mapper.readTree(BackupsTestUtils.BACKUPS.getBytes(run.getManifestKey())); + JsonNode legacy; + try(var in = getClass().getResourceAsStream("/manifests/legacy-schema.json")) + { + legacy = mapper.readTree(in); + } + for(JsonNode table : manifest.get("tables")) + { + ObjectNode entry = (ObjectNode) table; + JsonNode old = legacy.get(entry.get("name").asText()); + entry.remove("fingerprintAlgorithm"); + entry.set("schema", old.get("schema")); + entry.set("fingerprint", old.get("fingerprint")); + } + byte[] bytes = mapper.writeValueAsBytes(manifest); + BackupsTestUtils.BACKUPS.putBytes(run.getManifestKey(), bytes); + new BackupCatalog().update(BackupsTables.RUN, run.getRunId(), Map.of("manifestSha256", HashingOutputStream.sha256Hex(bytes))); + assertThat(BackupManifest.read(bytes).table("blogPost").fingerprintAlgorithm()).isNull(); + return (run); + } + + + + /******************************************************************************* + ** Old hashes still receive exact codec verification and restore all data, + ** including original timestamps, after deletes, edits and additions. + *******************************************************************************/ + @Test + void testLegacyManifestVerifiesAndRestores() throws Exception + { + String before = RestoreTestSupport.contentHash("blogPost"); + BackupRunResult run = legacyBackup(); + var verified = new BackupVerifier().verify(run.getRunId(), BackupVerifier.CONTENT, "MANUAL", "tester"); + assertThat(verified.status()).isEqualTo("PASSED"); + assertThat(verified.messages()).noneMatch(message -> message.contains("has changed since")); + RestoreTestSupport.mutate(); + RestoreOutcome outcome = new RestoreEngine().execute(RestoreTestSupport.request(run, RestoreRequest.MODE_REPLACE, "blogPost").withConfirmationText("blogPost")); + assertThat(outcome.getErrors()).isEmpty(); + assertThat(outcome.getTables().get("blogPost").getSchemaStatus()).isEqualTo("MATCH"); + assertThat(RestoreTestSupport.contentHash("blogPost")).isEqualTo(before); + } + + + + /******************************************************************************* + ** Legacy field maps and core field lists also compare correctly when hashes + ** differ: optional additions are compatible and type changes are rejected. + *******************************************************************************/ + @Test + void testLegacySchemaChangesAreAnalyzed() throws Exception + { + BackupRunResult run = legacyBackup(); + QContext.getQInstance().getTable("author").withField(new QFieldMetaData("nickname", QFieldType.STRING)); + QContext.getQInstance().getTable("blogPost").getField("title").setType(QFieldType.INTEGER); + RestoreOutcome review = new RestoreEngine().review(RestoreTestSupport.request(run, RestoreRequest.MODE_UPSERT, "author", "blogPost")); + assertThat(review.getTables().get("author").getSchemaStatus()).isEqualTo("COMPATIBLE"); + assertThat(review.getTables().get("blogPost").getSchemaStatus()).isEqualTo("MISMATCH"); + } +} diff --git a/src/test/java/com/kingsrook/qbits/backups/restore/RestoreSafetyTest.java b/src/test/java/com/kingsrook/qbits/backups/restore/RestoreSafetyTest.java index acf465c..ff86388 100644 --- a/src/test/java/com/kingsrook/qbits/backups/restore/RestoreSafetyTest.java +++ b/src/test/java/com/kingsrook/qbits/backups/restore/RestoreSafetyTest.java @@ -187,4 +187,22 @@ class RestoreSafetyTest QContext.setQSession(new QSession().withPermissions("backups.view", "backups.restore")); assertThat(new RestoreEngine().plan(request).errors()).isEmpty(); } + + + /******************************************************************************* + ** The core restore permission is required even in anonymous demo mode, and + ** Review reports it before a restore can create a run or write any data. + *******************************************************************************/ + @Test + void testCoreRestorePermissionIsRequiredWhenAnonymous() throws Exception + { + BackupRunResult run = RestoreTestSupport.backup(Map.of()); + QContext.setQSession(new QSession()); + RestoreRequest request = RestoreTestSupport.request(run, RestoreRequest.MODE_UPSERT, "blogPost"); + assertThat(new RestoreEngine().plan(request).errors()).anyMatch(e -> e.contains("backups.restore")); + assertThatThrownBy(() -> new RestoreEngine().execute(request)).hasMessageContaining("backups.restore"); + assertThatThrownBy(() -> new StandardRestoreWriter().insert(QContext.getQInstance().getTable("blogPost"), + List.of(new com.kingsrook.qqq.backend.core.model.data.QRecord().withValue("id", 99)), request, null)).hasMessageContaining("backups.restore"); + } + } diff --git a/src/test/java/com/kingsrook/qbits/backups/run/BackupEngineTest.java b/src/test/java/com/kingsrook/qbits/backups/run/BackupEngineTest.java index 8b3ca0b..0b0f800 100644 --- a/src/test/java/com/kingsrook/qbits/backups/run/BackupEngineTest.java +++ b/src/test/java/com/kingsrook/qbits/backups/run/BackupEngineTest.java @@ -109,7 +109,9 @@ class BackupEngineTest BackupManifest.TableEntry posts = manifest.table("blogPost"); assertThat(posts.rowCount()).isEqualTo(7); assertThat(posts.excludedFields()).extracting(BackupManifest.ExcludedField::name).containsExactly("secretNote"); - assertThat(posts.fingerprint()).startsWith("sha256:"); + assertThat(posts.fingerprintAlgorithm()).isEqualTo(com.kingsrook.qbits.backups.manifest.TableFingerprints.CORE_ALGORITHM); + assertThat(posts.fingerprint()).isEqualTo("sha256:" + com.kingsrook.qqq.backend.core.model.metadata.tables.TableSchemaFingerprint.of(QContext.getQInstance().getTable("blogPost"), "memory").getHash()); + assertThat(posts.schema()).isEqualTo(com.kingsrook.qqq.backend.core.model.metadata.tables.TableSchemaFingerprint.of(QContext.getQInstance().getTable("blogPost"), "memory").getDescriptor()); BackupManifest.Part part = posts.parts().get(0); byte[] stored = BackupsTestUtils.BACKUPS.getBytes(result.getRunPrefix() + part.key()); diff --git a/src/test/java/com/kingsrook/qbits/backups/run/TableReaderTest.java b/src/test/java/com/kingsrook/qbits/backups/run/TableReaderTest.java new file mode 100644 index 0000000..d4f87eb --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/backups/run/TableReaderTest.java @@ -0,0 +1,163 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.backups.run; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qbits.backups.BackupsTestUtils; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.actions.consistentread.ConsistentReadInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** Integration coverage for the core pager adapter's ordering, bounded fetches, + ** raw fields and transaction lifetime. + *******************************************************************************/ +class TableReaderTest +{ + private QTableMetaData table; + + + + /******************************************************************************* + ** Seed more records than fit in a page. + *******************************************************************************/ + @BeforeEach + void setUp() throws Exception + { + BackupsTestUtils.init(); + BackupsTestUtils.loadBlog(7); + table = QContext.getQInstance().getTable("blogPost"); + table.getField("body").setIsHeavy(true); + table.getField("secretNote").setIsHidden(true); + table.withCustomizer(TableCustomizers.POST_QUERY_RECORD, new QCodeReference(CaptureQueries.class)); + CaptureQueries.inputs.clear(); + } + + + + /******************************************************************************* + ** An OR filter and a resume cursor are ANDed, with ascending keys even when + ** the caller requests descending order. Every query has the requested bound. + *******************************************************************************/ + @Test + void testFilterResumeOrderingAndRawFields() throws Exception + { + String filter = "{\"booleanOperator\":\"OR\",\"criteria\":[{\"fieldName\":\"id\",\"operator\":\"LESS_THAN_OR_EQUALS\",\"values\":[3]},{\"fieldName\":\"id\",\"operator\":\"GREATER_THAN\",\"values\":[5]}],\"orderBys\":[{\"fieldName\":\"id\",\"isAscending\":false}]}"; + List rows = new ArrayList<>(); + new TableReader.Keyset().read(table, filter, 2, 2, null, rows::add); + assertThat(rows).extracting(r -> r.getValueInteger("id")).containsExactly(3, 6, 7); + assertThat(rows.get(0).getValueString("body")).isEqualTo("Body of 3, with a comma"); + assertThat(rows.get(0).getValueString("secretNote")).isEqualTo("do-not-leak-3"); + assertThat(CaptureQueries.inputs).hasSize(2).allSatisfy(input -> + { + assertThat(input.getFilter().getLimit()).isEqualTo(2); + assertThat(input.getFilter().getSkip()).isNull(); + assertThat(input.getShouldOmitHiddenFields()).isFalse(); + assertThat(input.getShouldMaskPasswords()).isFalse(); + assertThat(input.getShouldFetchHeavyFields()).isTrue(); + assertThat(input.getShouldTranslatePossibleValues()).isFalse(); + assertThat(input.getShouldGenerateDisplayValues()).isFalse(); + }); + } + + + + /******************************************************************************* + ** Changes made after a snapshot was opened are invisible on every page. The + ** reader leaves the transaction open, with its commit callbacks unexecuted. + *******************************************************************************/ + @Test + void testConsistentSnapshotIsRetainedAcrossPages() throws Exception + { + List commits = new ArrayList<>(); + try(QBackendTransaction transaction = QBackendTransaction.openConsistentReadFor(new ConsistentReadInput().withBackendName(table.getBackendName()))) + { + transaction.addAfterCommitCallback(() -> commits.add(true)); + new UpdateAction().execute(new UpdateInput("blogPost").withRecords(List.of(new QRecord().withValue("id", 7).withValue("title", "Changed after snapshot")))); + CaptureQueries.inputs.clear(); + List rows = new ArrayList<>(); + new TableReader.Keyset().read(table, null, null, 2, transaction, rows::add); + assertThat(rows).extracting(r -> r.getValueInteger("id")).containsExactly(1, 2, 3, 4, 5, 6, 7); + assertThat(rows.get(6).getValueString("title")).isEqualTo("Post 7"); + assertThat(CaptureQueries.inputs).hasSize(4).allSatisfy(input -> assertThat(input.getTransaction()).isSameAs(transaction)); + assertThat(commits).isEmpty(); + transaction.commit(); + assertThat(commits).containsExactly(true); + } + } + + + + /******************************************************************************* + ** Exact page multiples, empty scans, invalid bounds and unsupported key types + ** must terminate or fail explicitly rather than drop rows or stream unordered. + *******************************************************************************/ + @Test + void testBoundariesAndInvalidPaging() throws Exception + { + List rows = new ArrayList<>(); + new TableReader.Keyset().read(table, null, 3, 2, null, rows::add); + assertThat(rows).extracting(r -> r.getValueInteger("id")).containsExactly(4, 5, 6, 7); + rows.clear(); + new TableReader.Keyset().read(table, null, 7, 2, null, rows::add); + assertThat(rows).isEmpty(); + assertThatThrownBy(() -> new TableReader.Keyset().read(table, null, null, 0, null, rows::add)).hasMessageContaining("Page size"); + assertThatThrownBy(() -> new TableReader.Keyset().read(table, "{\"limit\":1}", null, 2, null, rows::add)).hasMessageContaining("skip or limit"); + assertThatThrownBy(() -> new TableReader.Keyset().read(table, "{", null, 2, null, rows::add)).hasMessageContaining("scope filter"); + table.getField("id").setType(QFieldType.BOOLEAN); + assertThatThrownBy(() -> new TableReader.Keyset().read(table, null, null, 2, null, rows::add)).hasMessageContaining("orderable primary key"); + } + + + + /******************************************************************************* + ** Observe actual QueryAction inputs without replacing the backend or pager. + *******************************************************************************/ + public static class CaptureQueries implements TableCustomizerInterface + { + static final List inputs = new ArrayList<>(); + + + + /******************************************************************************* + ** Record each page and preserve the result unchanged. + *******************************************************************************/ + @Override + public List postQuery(QueryOrGetInputInterface input, List records) + { + inputs.add((QueryInput) input); + return (records); + } + } +} diff --git a/src/test/resources/manifests/legacy-schema.json b/src/test/resources/manifests/legacy-schema.json new file mode 100644 index 0000000..e8caca9 --- /dev/null +++ b/src/test/resources/manifests/legacy-schema.json @@ -0,0 +1,130 @@ +{ + "author" : { + "schema" : { + "name" : "author", + "primaryKey" : "id", + "backendType" : "memory", + "fields" : { + "bio" : { + "type" : "TEXT", + "required" : false, + "maxLength" : null, + "pvs" : null, + "heavy" : false, + "sensitive" : false + }, + "id" : { + "type" : "INTEGER", + "required" : false, + "maxLength" : null, + "pvs" : null, + "heavy" : false, + "sensitive" : false + }, + "name" : { + "type" : "STRING", + "required" : false, + "maxLength" : null, + "pvs" : null, + "heavy" : false, + "sensitive" : false + } + }, + "uniqueKeys" : [ ], + "associations" : [ ] + }, + "fingerprint" : "sha256:c0fce31286898a15a98f9f46515b15c6eee4ffeb1f1eb39d6100833dfe3f760e" + }, + "blogPost" : { + "schema" : { + "name" : "blogPost", + "primaryKey" : "id", + "backendType" : "memory", + "fields" : { + "authorId" : { + "type" : "INTEGER", + "required" : false, + "maxLength" : null, + "pvs" : "author", + "heavy" : false, + "sensitive" : false + }, + "body" : { + "type" : "TEXT", + "required" : false, + "maxLength" : null, + "pvs" : null, + "heavy" : false, + "sensitive" : false + }, + "createDate" : { + "type" : "DATE_TIME", + "required" : false, + "maxLength" : null, + "pvs" : null, + "heavy" : false, + "sensitive" : false + }, + "id" : { + "type" : "INTEGER", + "required" : false, + "maxLength" : null, + "pvs" : null, + "heavy" : false, + "sensitive" : false + }, + "mediaKey" : { + "type" : "STRING", + "required" : false, + "maxLength" : null, + "pvs" : null, + "heavy" : false, + "sensitive" : false + }, + "price" : { + "type" : "DECIMAL", + "required" : false, + "maxLength" : null, + "pvs" : null, + "heavy" : false, + "sensitive" : false + }, + "published" : { + "type" : "BOOLEAN", + "required" : false, + "maxLength" : null, + "pvs" : null, + "heavy" : false, + "sensitive" : false + }, + "publishedOn" : { + "type" : "DATE", + "required" : false, + "maxLength" : null, + "pvs" : null, + "heavy" : false, + "sensitive" : false + }, + "secretNote" : { + "type" : "STRING", + "required" : false, + "maxLength" : null, + "pvs" : null, + "heavy" : false, + "sensitive" : true + }, + "title" : { + "type" : "STRING", + "required" : false, + "maxLength" : null, + "pvs" : null, + "heavy" : false, + "sensitive" : false + } + }, + "uniqueKeys" : [ ], + "associations" : [ ] + }, + "fingerprint" : "sha256:3a4de7479908cd4b4ce8eaa7a7af43e7cbaabbdbc2b9741bebeb24a6bd41b9ea" + } +} \ No newline at end of file -- 2.54.0 From 054f6d90d7f33a002b83b848a952b786823a6fed Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 14 Oct 2026 07:15:24 -0500 Subject: [PATCH 2/2] build: qbit-build-parent 2.1.0 with architecture guardrails enforced Enforce the qbit-bom#10 guardrails (G1, G3, G4, G7) and run tests in strict raw-route mode (G2). All pass on the released QQQ 4.1.0-RC.6. Co-Authored-By: Claude Opus 5.5 --- pom.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f1b90f6..832f58d 100644 --- a/pom.xml +++ b/pom.xml @@ -18,6 +18,8 @@ + true + true 0.1.0-SNAPSHOT 21 21 @@ -27,7 +29,7 @@ com.kingsrook qbit-build-parent - 2.0.0 + 2.1.0 com.kingsrook.qbits -- 2.54.0