From ab283c79b98d5c87706880378c6bb5e48d78d03d Mon Sep 17 00:00:00 2001 From: James Maes Date: Tue, 10 Nov 2026 15:22:22 -0600 Subject: [PATCH] perf: keep the access fingerprint for the request and read rule rows once per change Core asks the field access resolver for its fingerprint before every field decision (FieldAccessHelper.getDecision), that is once per field of every record of a result and of every table in the metaData document. Each call recomputed it: a JSON key, a permission lookup for every table, another JSON document and a digest. Core also asks for the retention rules of a field on every audited field and every filter criterion, and each question was a query. - getAccessFingerprint keeps its value in the request context (QContext) and returns it while the session, its permission set, the channel, the policy and classification versions are unchanged and no active break-glass grant has expired. Nothing outlives the request. - RetentionRules.active answers from the active rules the runtime read once (SecuredDataRuntime.activeRetentionRules). A rule write clears them after commit; a thread that edits rules inside a transaction reads its own edits. - Policy, classification and retention changes are announced on QQQ's cluster bus (ClusterCaches, memo-invalidate), so every node reads again. Before, the versions were node-local and another node kept old rules until restart. - No node uses rule rows older than 30 seconds without reading them again (the safety net for a lost event). - A read that fails throws and replaces nothing: never an empty rule list. Measured in one JVM with GG Forge's composed instance (222 tables, secured data and collaboration wired, in-memory backend, platform administrator), released 0.1.0-RC.4 against this change, same machine, back to back: metaData request 866.6 ms to 125.3 ms (930 MB to 74 MB allocated, 96 to 34 queries); a 300-row list 1,566 ms to 24 ms; a 27-row list 174 to 230 ms down to 13 to 17 ms; queries per list 23 to 8. Co-Authored-By: Claude Fable 5.2 --- .../secureddata/SecuredDataResolver.java | 98 ++++++- .../qbits/secureddata/SecuredDataRuntime.java | 142 +++++++++- .../metadata/SecuredDataTableCustomizers.java | 7 +- .../secureddata/retention/RetentionRules.java | 63 ++++- .../SecuredDataRuleCachesTest.java | 261 ++++++++++++++++++ 5 files changed, 555 insertions(+), 16 deletions(-) create mode 100644 src/test/java/com/kingsrook/qbits/secureddata/SecuredDataRuleCachesTest.java diff --git a/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataResolver.java b/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataResolver.java index f0c137e..b4b867e 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataResolver.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataResolver.java @@ -75,9 +75,13 @@ public class SecuredDataResolver implements FieldAccessResolverInterface } private long cachedVersion; + private long policiesLoadedAt; private List policies = List.of(); private final Map users = new HashMap<>(); + /** Request-scoped (QContext) holder of the access fingerprint. */ + private static final String FINGERPRINT_KEY = SecuredDataResolver.class.getName() + ".accessFingerprint"; + /******************************************************************************* @@ -292,11 +296,13 @@ public class SecuredDataResolver implements FieldAccessResolverInterface private synchronized UserState state() { long version = runtime.policyVersion(); - if(cachedVersion != version) + if(cachedVersion != version || SecuredDataRuntime.tooOld(policiesLoadedAt)) { + long loadedAt = System.nanoTime(); policies = List.copyOf(SystemWrites.query(SecuredDataTables.POLICY, new QQueryFilter())); users.clear(); cachedVersion = version; + policiesLoadedAt = loadedAt; } QSession session = QContext.getQSession(); TreeSet held = new TreeSet<>(); @@ -308,7 +314,7 @@ public class SecuredDataResolver implements FieldAccessResolverInterface held.add(permission); } } - String key = JsonUtils.toJson(List.of(userId(), held)); + String key = userId() + '\u0000' + String.join("\u0000", held); UserState state = users.get(key); if(state == null) { @@ -356,11 +362,82 @@ public class SecuredDataResolver implements FieldAccessResolverInterface ** Stable across unrelated permissions/records; includes channel and exact expiry. *******************************************************************************/ @Override - public synchronized String getAccessFingerprint() + public String getAccessFingerprint() { - state(); - TreeSet relevant = new TreeSet<>(); + ///////////////////////////////////////////////////////////////////////////// + // core asks for the fingerprint before every field decision (once per // + // field of every record of a result), so the value is kept for the // + // request - and only while everything it is derived from is unchanged // + ///////////////////////////////////////////////////////////////////////////// QSession session = QContext.getQSession(); + String channel = Objects.requireNonNullElse(SecuredDataMetadata.channel(session), ""); + Instant now = runtime.config().getClock().instant(); + RequestFingerprint kept = QContext.getObject(FINGERPRINT_KEY, RequestFingerprint.class).orElse(null); + if(kept != null && kept.matches(this, session, channel, runtime.policyVersion(), runtime.classificationVersion(), now)) + { + return (kept.value()); + } + RequestFingerprint computed = computeAccessFingerprint(session, channel, now); + QContext.setObject(FINGERPRINT_KEY, computed); + return (computed.value()); + } + + + + /******************************************************************************* + ** The fingerprint and what it was derived from. It stops matching when the + ** session, its permissions, the channel, a policy or classification version + ** changes, or when the first active break-glass grant expires. + *******************************************************************************/ + private record RequestFingerprint(int resolver, QSession session, int permissionsIdentity, int permissionCount, String channel, long policyVersion, + long classificationVersion, Instant validUntil, String value) implements Serializable + { + /******************************************************************************* + ** Whether the kept value is still the one a fresh computation would give. + *******************************************************************************/ + boolean matches(SecuredDataResolver owner, QSession current, String currentChannel, long currentPolicyVersion, long currentClassificationVersion, Instant now) + { + return (resolver == System.identityHashCode(owner) && session == current && permissionsIdentity == identityOfPermissions(current) && permissionCount == countOfPermissions(current) + && channel.equals(currentChannel) && policyVersion == currentPolicyVersion && classificationVersion == currentClassificationVersion && now.isBefore(validUntil)); + } + } + + + + /******************************************************************************* + ** The permission set object of a session (it is replaced, not edited, when a + ** session is given new permissions). + *******************************************************************************/ + private static int identityOfPermissions(QSession session) + { + return (session == null || session.getPermissions() == null ? 0 : System.identityHashCode(session.getPermissions())); + } + + + + /******************************************************************************* + ** How many permissions a session holds (an edit in place changes it). + *******************************************************************************/ + private static int countOfPermissions(QSession session) + { + return (session == null || session.getPermissions() == null ? -1 : session.getPermissions().size()); + } + + + + /******************************************************************************* + ** Stable across unrelated permissions/records; includes channel and exact expiry. + *******************************************************************************/ + private synchronized RequestFingerprint computeAccessFingerprint(QSession session, String channel, Instant now) + { + ///////////////////////////////////////////////////////////////////////////// + // versions are read before the rows they describe, so a change during the // + // computation leaves a kept value that no longer matches // + ///////////////////////////////////////////////////////////////////////////// + long policyVersion = runtime.policyVersion(); + long classificationVersion = runtime.classificationVersion(); + TreeSet relevant = new TreeSet<>(); + List breakGlass = state().breakGlass(); for(QRecord row : policies) { String permission = row.getValueString("permission_name"); @@ -380,12 +457,15 @@ public class SecuredDataResolver implements FieldAccessResolverInterface } } } - List breaks = activeBreakGlass().stream().map(row -> row.getValueString("id")).sorted().toList(); - String payload = JsonUtils.toJson(List.of(userId(), relevant, runtime.policyVersion(), runtime.classificationVersion(), breaks, - Objects.requireNonNullElse(SecuredDataMetadata.channel(session), ""))); + List activeBreaks = breakGlass.stream().filter(row -> "ACTIVE".equals(row.getValueString("status"))) + .filter(row -> row.getValueInstant("expires_at") != null && row.getValueInstant("expires_at").isAfter(now)).toList(); + List breaks = activeBreaks.stream().map(row -> row.getValueString("id")).sorted().toList(); + Instant validUntil = activeBreaks.stream().map(row -> row.getValueInstant("expires_at")).min(Instant::compareTo).orElse(Instant.MAX); + String payload = JsonUtils.toJson(List.of(userId(), relevant, policyVersion, classificationVersion, breaks, channel)); try { - return (Base64.getUrlEncoder().withoutPadding().encodeToString(MessageDigest.getInstance("SHA-256").digest(payload.getBytes(StandardCharsets.UTF_8)))); + String value = Base64.getUrlEncoder().withoutPadding().encodeToString(MessageDigest.getInstance("SHA-256").digest(payload.getBytes(StandardCharsets.UTF_8))); + return (new RequestFingerprint(System.identityHashCode(this), session, identityOfPermissions(session), countOfPermissions(session), channel, policyVersion, classificationVersion, validUntil, value)); } catch(java.security.NoSuchAlgorithmException e) { diff --git a/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataRuntime.java b/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataRuntime.java index cc2ef2b..6f5e85d 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataRuntime.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/SecuredDataRuntime.java @@ -21,9 +21,21 @@ package com.kingsrook.qbits.secureddata; +import java.lang.ref.WeakReference; +import java.time.Duration; import java.util.List; import java.util.concurrent.atomic.AtomicLong; +import com.kingsrook.qbits.secureddata.metadata.SecuredDataTables; +import com.kingsrook.qbits.secureddata.support.SystemWrites; +import com.kingsrook.qqq.backend.core.cluster.ClusterBus; +import com.kingsrook.qqq.backend.core.cluster.ClusterCaches; +import com.kingsrook.qqq.backend.core.cluster.ClusterEvent; import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QSupplementalInstanceMetaData; @@ -34,9 +46,33 @@ import com.kingsrook.qqq.backend.core.model.metadata.QSupplementalInstanceMetaDa public class SecuredDataRuntime implements QSupplementalInstanceMetaData, AutoCloseable { public static final String NAME = "securedDataRuntime"; + + private static final QLogger LOG = QLogger.getLogger(SecuredDataRuntime.class); + + ///////////////////////////////////////////////////////////////////////////////// + // the three sets of rule rows a node keeps in memory. A change clears them on // + // this node at once and, through QQQ's cluster bus (ClusterCaches), on every // + // other node. MAX_RULE_AGE is the safety net for a lost event: no node uses // + // rule rows older than that without reading them again. // + ///////////////////////////////////////////////////////////////////////////////// + public static final String POLICY_CACHE = "securedData.policies"; + public static final String CLASSIFICATION_CACHE = "securedData.classifications"; + public static final String RETENTION_CACHE = "securedData.retentionRules"; + + public static final Duration MAX_RULE_AGE = Duration.ofSeconds(30); + + static + { + ClusterCaches.shared(POLICY_CACHE, POLICY_CACHE, () -> clearOnThisNode(POLICY_CACHE), null); + ClusterCaches.shared(CLASSIFICATION_CACHE, CLASSIFICATION_CACHE, () -> clearOnThisNode(CLASSIFICATION_CACHE), null); + ClusterCaches.shared(RETENTION_CACHE, RETENTION_CACHE, () -> clearOnThisNode(RETENTION_CACHE), null); + } + private static final AtomicLong STARTUP_VERSION = new AtomicLong(System.currentTimeMillis()); private final AtomicLong classificationVersion = new AtomicLong(STARTUP_VERSION.incrementAndGet()); private final AtomicLong policyVersion = new AtomicLong(STARTUP_VERSION.incrementAndGet()); + private final AtomicLong retentionVersion = new AtomicLong(STARTUP_VERSION.incrementAndGet()); + private final transient WeakReference owner; private final transient SecuredDataQBitConfig config; private final transient SecuredDataResolver resolver; private final transient SecuredDataKeySource keys = new SecuredDataKeySource(); @@ -44,6 +80,10 @@ public class SecuredDataRuntime implements QSupplementalInstanceMetaData, AutoCl private final transient SecuredDataAnomalies anomalies; private transient long cachedClassificationVersion; private transient List classificationRows = List.of(); + private transient long classificationRowsLoadedAt; + private transient long cachedRetentionVersion; + private transient long retentionRulesLoadedAt; + private transient List activeRetentionRules = List.of(); @@ -53,6 +93,7 @@ public class SecuredDataRuntime implements QSupplementalInstanceMetaData, AutoCl public SecuredDataRuntime(QInstance instance, SecuredDataQBitConfig config) { this.config = config; + this.owner = new WeakReference<>(instance); this.resolver = new SecuredDataResolver(this); this.anomalies = new SecuredDataAnomalies(config); this.logger = new SecuredDataAccessLogger(instance, this); @@ -137,6 +178,7 @@ public class SecuredDataRuntime implements QSupplementalInstanceMetaData, AutoCl public void invalidateClassifications() { classificationVersion.incrementAndGet(); + announce(CLASSIFICATION_CACHE); } @@ -147,6 +189,102 @@ public class SecuredDataRuntime implements QSupplementalInstanceMetaData, AutoCl public void invalidatePolicies() { policyVersion.incrementAndGet(); + announce(POLICY_CACHE); + } + + + + /******************************************************************************* + ** A retention rule changed: forget the active rules here and on every node. + *******************************************************************************/ + public void invalidateRetentionRules() + { + retentionVersion.incrementAndGet(); + announce(RETENTION_CACHE); + } + + + + /******************************************************************************* + ** Version of the retention rules this node holds. + *******************************************************************************/ + public long retentionVersion() + { + return (retentionVersion.get()); + } + + + + /******************************************************************************* + ** Whether rows loaded at this System.nanoTime are too old to use unread. + *******************************************************************************/ + static boolean tooOld(long loadedAtNanos) + { + return (System.nanoTime() - loadedAtNanos > MAX_RULE_AGE.toNanos()); + } + + + + /******************************************************************************* + ** Every active retention rule, read once per version and at most MAX_RULE_AGE + ** old. A read that fails throws and leaves nothing in place of the rules: + ** callers never see "no rules" because the rules could not be read. + *******************************************************************************/ + public synchronized List activeRetentionRules() + { + long version = retentionVersion(); + if(cachedRetentionVersion != version || tooOld(retentionRulesLoadedAt)) + { + long loadedAt = System.nanoTime(); + activeRetentionRules = List.copyOf(SystemWrites.query(SecuredDataTables.RETENTION_RULE, new QQueryFilter() + .withCriteria(new QFilterCriteria("is_active", QCriteriaOperator.EQUALS, true)))); + cachedRetentionVersion = version; + retentionRulesLoadedAt = loadedAt; + } + return (activeRetentionRules); + } + + + + /******************************************************************************* + ** Tell the other nodes (this node has already moved its version). A failure + ** to send is logged: the other nodes then read again within MAX_RULE_AGE. + *******************************************************************************/ + private void announce(String cache) + { + try + { + QInstance instance = owner.get(); + if(instance != null) + { + ClusterBus.publish(instance, new ClusterEvent(ClusterEvent.TYPE_MEMO_INVALIDATE, cache, null)); + } + } + catch(RuntimeException e) + { + LOG.warn("Could not announce a secured-data rule change; other nodes read again within " + MAX_RULE_AGE.toSeconds() + " seconds", e); + } + } + + + + /******************************************************************************* + ** A rule change announced on the cluster bus (also by this node): move the + ** version of the runtime of the instance the event arrived on. + *******************************************************************************/ + private static void clearOnThisNode(String cache) + { + SecuredDataRuntime runtime = current(); + if(runtime == null) + { + return; + } + switch(cache) + { + case POLICY_CACHE -> runtime.policyVersion.incrementAndGet(); + case CLASSIFICATION_CACHE -> runtime.classificationVersion.incrementAndGet(); + default -> runtime.retentionVersion.incrementAndGet(); + } } @@ -157,11 +295,13 @@ public class SecuredDataRuntime implements QSupplementalInstanceMetaData, AutoCl synchronized List classificationRows() { long version = classificationVersion(); - if(cachedClassificationVersion != version) + if(cachedClassificationVersion != version || tooOld(classificationRowsLoadedAt)) { + long loadedAt = System.nanoTime(); classificationRows = List.copyOf(com.kingsrook.qbits.secureddata.support.SystemWrites.query( com.kingsrook.qbits.secureddata.metadata.SecuredDataTables.CLASSIFICATION, new com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter())); cachedClassificationVersion = version; + classificationRowsLoadedAt = loadedAt; } return (classificationRows); } diff --git a/src/main/java/com/kingsrook/qbits/secureddata/metadata/SecuredDataTableCustomizers.java b/src/main/java/com/kingsrook/qbits/secureddata/metadata/SecuredDataTableCustomizers.java index eb1b439..c36d921 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/metadata/SecuredDataTableCustomizers.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/metadata/SecuredDataTableCustomizers.java @@ -238,7 +238,8 @@ public class SecuredDataTableCustomizers implements TableCustomizerInterface } SecuredDataRuntime runtime = SecuredDataRuntime.current(); Runnable invalidate = SecuredDataTables.CLASSIFICATION.equals(table) ? runtime::invalidateClassifications - : SecuredDataTables.POLICY.equals(table) || SecuredDataTables.BREAK_GLASS.equals(table) ? runtime::invalidatePolicies : null; + : SecuredDataTables.POLICY.equals(table) || SecuredDataTables.BREAK_GLASS.equals(table) ? runtime::invalidatePolicies + : SecuredDataTables.RETENTION_RULE.equals(table) ? runtime::invalidateRetentionRules : null; if(invalidate != null) { QBackendTransaction active = transaction == null ? QContext.getQBackendTransaction() : transaction; @@ -248,6 +249,10 @@ public class SecuredDataTableCustomizers implements TableCustomizerInterface } else { + if(SecuredDataTables.RETENTION_RULE.equals(table)) + { + com.kingsrook.qbits.secureddata.retention.RetentionRules.editedInTransaction(); + } active.addAfterCommitCallback(invalidate); } } diff --git a/src/main/java/com/kingsrook/qbits/secureddata/retention/RetentionRules.java b/src/main/java/com/kingsrook/qbits/secureddata/retention/RetentionRules.java index 224289e..f79624d 100644 --- a/src/main/java/com/kingsrook/qbits/secureddata/retention/RetentionRules.java +++ b/src/main/java/com/kingsrook/qbits/secureddata/retention/RetentionRules.java @@ -58,15 +58,68 @@ public final class RetentionRules return (names == null ? List.of() : Arrays.stream(names.split(",", -1)).map(String::trim).toList()); } + /////////////////////////////////////////////////////////////////////////////// + // set on a thread that changed a rule inside a transaction: until that // + // transaction ends, the thread reads the rules through it (its own edits), // + // not the rules every other thread shares // + /////////////////////////////////////////////////////////////////////////////// + private static final ThreadLocal EDITED_IN_TRANSACTION = ThreadLocal.withInitial(() -> false); + + + /******************************************************************************* - ** Fresh rules in the current transaction; edits take effect without a stale cache. + ** The active rules of a table that name this field. Core asks this for every + ** field of every audited write and every filter criterion of every query, so + ** the active rules are read once (SecuredDataRuntime.activeRetentionRules: per + ** change, and at most MAX_RULE_AGE old) instead of once per call. A committed + ** change is seen at once on the node that made it and on every node the + ** cluster event reaches; a transaction that edits rules reads its own edits. + ** A failed read throws: it is never "no rules". *******************************************************************************/ public static List active(String table, String field) { - return (SystemWrites.query(SecuredDataTables.RETENTION_RULE, new QQueryFilter() - .withCriteria(new QFilterCriteria("is_active", QCriteriaOperator.EQUALS, true)) - .withCriteria(new QFilterCriteria("table_name", QCriteriaOperator.EQUALS, table))) - .stream().filter(rule -> fields(rule).contains(field)).toList()); + SecuredDataRuntime runtime = SecuredDataRuntime.current(); + List rules; + if(runtime == null || readsOwnEdits()) + { + rules = SystemWrites.query(SecuredDataTables.RETENTION_RULE, new QQueryFilter() + .withCriteria(new QFilterCriteria("is_active", QCriteriaOperator.EQUALS, true)) + .withCriteria(new QFilterCriteria("table_name", QCriteriaOperator.EQUALS, table))); + } + else + { + rules = runtime.activeRetentionRules(); + } + return (rules.stream().filter(rule -> table != null && table.equals(rule.getValueString("table_name"))).filter(rule -> fields(rule).contains(field)).toList()); + } + + + + /******************************************************************************* + ** A rule was written on this thread inside a transaction that is still open. + *******************************************************************************/ + public static void editedInTransaction() + { + EDITED_IN_TRANSACTION.set(true); + } + + + + /******************************************************************************* + ** Whether this thread must read the rules through its open transaction. + *******************************************************************************/ + private static boolean readsOwnEdits() + { + if(!EDITED_IN_TRANSACTION.get()) + { + return (false); + } + if(QContext.getQBackendTransaction() == null) + { + EDITED_IN_TRANSACTION.remove(); + return (false); + } + return (true); } /******************************************************************************* diff --git a/src/test/java/com/kingsrook/qbits/secureddata/SecuredDataRuleCachesTest.java b/src/test/java/com/kingsrook/qbits/secureddata/SecuredDataRuleCachesTest.java new file mode 100644 index 0000000..ced3705 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/secureddata/SecuredDataRuleCachesTest.java @@ -0,0 +1,261 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.secureddata; + + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import com.kingsrook.qbits.secureddata.metadata.SecuredDataTables; +import com.kingsrook.qbits.secureddata.permissions.SecuredDataPermissions; +import com.kingsrook.qbits.secureddata.retention.RetentionRules; +import com.kingsrook.qbits.secureddata.support.SecuredDataMetadata; +import com.kingsrook.qqq.backend.core.cluster.ClusterBus; +import com.kingsrook.qqq.backend.core.cluster.ClusterCaches; +import com.kingsrook.qqq.backend.core.cluster.ClusterEvent; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static com.kingsrook.qbits.secureddata.SecuredDataTestSupport.close; +import static com.kingsrook.qbits.secureddata.SecuredDataTestSupport.delete; +import static com.kingsrook.qbits.secureddata.SecuredDataTestSupport.insert; +import static com.kingsrook.qbits.secureddata.SecuredDataTestSupport.setup; +import static com.kingsrook.qbits.secureddata.SecuredDataTestSupport.update; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** The rule rows a node keeps (retention rules, policies, classifications) and + ** the access fingerprint of a request: read once, cleared by every change on + ** every node, never older than the maximum age, and never an empty answer when + ** the rules cannot be read. + *******************************************************************************/ +class SecuredDataRuleCachesTest +{ + private static final List ANNOUNCED = new ArrayList<>(); + + static + { + ClusterBus.on(ClusterEvent.TYPE_MEMO_INVALIDATE, (instance, event) -> ANNOUNCED.add(event.getName())); + } + + QInstance instance; + QTableMetaData person; + SecuredDataRuntime runtime; + + + + /******************************************************************************* + ** A person table with a date, so a retention rule on it is valid. + *******************************************************************************/ + @BeforeEach + void init() throws Exception + { + instance = setup(); + person = instance.getTable("person"); + person.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("createDate")); + runtime = SecuredDataRuntime.current(); + MemoryRecordStore.setCollectStatistics(true); + MemoryRecordStore.resetStatistics(); + ANNOUNCED.clear(); + } + + + + /******************************************************************************* + ** Cleanup. + *******************************************************************************/ + @AfterEach + void cleanup() + { + MemoryRecordStore.setCollectStatistics(false); + close(); + } + + + + /******************************************************************************* + ** An active rule on one field of the person table. + *******************************************************************************/ + private static QRecord rule(String field) throws Exception + { + return (insert(SecuredDataTables.RETENTION_RULE, new QRecord().withValue("name", "30 days: " + field).withValue("table_name", "person") + .withValue("field_names", field).withValue("age_field_name", "createDate").withValue("age_days", 30).withValue("action", "NULL_WITH_MARKER"))); + } + + + + /******************************************************************************* + ** Queries the in-memory store has run since the last reset. + *******************************************************************************/ + private static int queries() + { + return (MemoryRecordStore.getStatistics().getOrDefault(MemoryRecordStore.STAT_QUERIES_RAN, 0)); + } + + + + /******************************************************************************* + ** Core asks for the rules of a field on every audited field and filter + ** criterion: many questions are one read, and every committed change (insert, + ** update, delete) is seen by the next question. + *******************************************************************************/ + @Test + void activeRetentionRulesAreReadOncePerChange() throws Exception + { + QRecord notes = rule("notes"); + assertThat(notes.getErrors()).isNullOrEmpty(); + MemoryRecordStore.resetStatistics(); + for(int i = 0; i < 50; i++) + { + assertThat(RetentionRules.active("person", "notes")).hasSize(1); + assertThat(RetentionRules.active("person", "email")).isEmpty(); + assertThat(RetentionRules.active("anotherTable", "notes")).isEmpty(); + assertThat(RetentionRules.changedOnly("person", "notes")).isFalse(); + } + assertThat(queries()).isLessThanOrEqualTo(1); + + QRecord ssn = rule("ssn"); + assertThat(RetentionRules.active("person", "ssn")).hasSize(1); + + update(SecuredDataTables.RETENTION_RULE, new QRecord().withValue("id", notes.getValue("id")).withValue("is_active", false)); + assertThat(RetentionRules.active("person", "notes")).isEmpty(); + + delete(SecuredDataTables.RETENTION_RULE, ssn); + assertThat(RetentionRules.active("person", "ssn")).isEmpty(); + assertThat(ANNOUNCED).contains(SecuredDataRuntime.RETENTION_CACHE); + } + + + + /******************************************************************************* + ** A rule change made on another node arrives as a cluster cache event: each + ** of the three kinds of rules is read again here. A change made here is + ** announced to the other nodes. + *******************************************************************************/ + @Test + void aChangeOnAnyNodeClearsTheRulesOnEveryNode() throws Exception + { + rule("notes"); + assertThat(RetentionRules.active("person", "notes")).hasSize(1); + + long retention = runtime.retentionVersion(); + long policies = runtime.policyVersion(); + long classifications = runtime.classificationVersion(); + ClusterCaches.invalidate(SecuredDataRuntime.RETENTION_CACHE); + ClusterCaches.invalidate(SecuredDataRuntime.POLICY_CACHE); + ClusterCaches.invalidate(SecuredDataRuntime.CLASSIFICATION_CACHE); + assertThat(runtime.retentionVersion()).isGreaterThan(retention); + assertThat(runtime.policyVersion()).isGreaterThan(policies); + assertThat(runtime.classificationVersion()).isGreaterThan(classifications); + + MemoryRecordStore.resetStatistics(); + assertThat(RetentionRules.active("person", "notes")).hasSize(1); + assertThat(queries()).isGreaterThanOrEqualTo(1); + MemoryRecordStore.resetStatistics(); + assertThat(RetentionRules.active("person", "notes")).hasSize(1); + assertThat(queries()).isZero(); + + ANNOUNCED.clear(); + runtime.invalidatePolicies(); + runtime.invalidateClassifications(); + runtime.invalidateRetentionRules(); + assertThat(ANNOUNCED).contains(SecuredDataRuntime.POLICY_CACHE, SecuredDataRuntime.CLASSIFICATION_CACHE, SecuredDataRuntime.RETENTION_CACHE); + } + + + + /******************************************************************************* + ** Fail closed: when the rules cannot be read, asking for them throws. The + ** rules read before are not used, and the answer is never an empty list. + *******************************************************************************/ + @Test + void rulesThatCannotBeReadAreNeverAnEmptyAnswer() throws Exception + { + rule("notes"); + assertThat(RetentionRules.active("person", "notes")).hasSize(1); + + QTableMetaData ruleTable = instance.getTables().remove(SecuredDataTables.RETENTION_RULE); + try + { + runtime.invalidateRetentionRules(); + assertThatThrownBy(() -> RetentionRules.active("person", "notes")).isInstanceOf(RuntimeException.class); + assertThatThrownBy(() -> RetentionRules.active("person", "notes")).isInstanceOf(RuntimeException.class); + } + finally + { + instance.getTables().put(SecuredDataTables.RETENTION_RULE, ruleTable); + } + assertThat(RetentionRules.active("person", "notes")).hasSize(1); + } + + + + /******************************************************************************* + ** The safety net for a lost cluster event: rows are read again once they are + ** older than the maximum age. + *******************************************************************************/ + @Test + void ruleRowsAreNotUsedPastTheirMaximumAge() + { + assertThat(SecuredDataRuntime.tooOld(System.nanoTime())).isFalse(); + assertThat(SecuredDataRuntime.tooOld(System.nanoTime() - SecuredDataRuntime.MAX_RULE_AGE.plusSeconds(1).toNanos())).isTrue(); + assertThat(SecuredDataRuntime.MAX_RULE_AGE.toSeconds()).isLessThanOrEqualTo(60); + } + + + + /******************************************************************************* + ** Core asks for the access fingerprint before every field decision. Within a + ** request the same value object is returned; a permission that matters, given + ** to the same session object, changes it at once. + *******************************************************************************/ + @Test + void theAccessFingerprintIsKeptForTheRequestAndTracksThePermissions() + { + SecuredDataResolver resolver = runtime.resolver(); + QSession session = QContext.getQSession(); + String first = resolver.getAccessFingerprint(); + for(int i = 0; i < 20; i++) + { + assertThat(resolver.getAccessFingerprint()).isSameAs(first); + } + + session.getPermissions().add("unrelated.permission"); + assertThat(resolver.getAccessFingerprint()).isEqualTo(first); + + session.getPermissions().add(SecuredDataMetadata.permissionBase(person) + "." + SecuredDataPermissions.FULL); + String granted = resolver.getAccessFingerprint(); + assertThat(granted).isNotEqualTo(first); + + session.setPermissions(new HashSet<>(List.of("person.read"))); + assertThat(resolver.getAccessFingerprint()).isNotEqualTo(granted); + } +} -- 2.54.0