/* * QQQ - Low-code Application Framework for Engineers. * Copyright (C) 2021-2025. Kingsrook, LLC * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States * contact@kingsrook.com * https://github.com/Kingsrook/ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.kingsrook.qbits.secureddata; import java.util.ArrayList; import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import com.kingsrook.qbits.secureddata.metadata.SecuredDataTables; import com.kingsrook.qbits.secureddata.retention.RetentionRules; import com.kingsrook.qbits.secureddata.support.SystemWrites; import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.actions.ai.DataClassification; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.permissions.fieldaccess.FieldAccessDecision; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.actions.tables.query.VirtualFieldValueProviderInterface; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.fields.QVirtualFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static com.kingsrook.qbits.secureddata.SecuredDataTestSupport.classification; import static com.kingsrook.qbits.secureddata.SecuredDataTestSupport.close; import static com.kingsrook.qbits.secureddata.SecuredDataTestSupport.policyRow; import static com.kingsrook.qbits.secureddata.SecuredDataTestSupport.setup; import static com.kingsrook.qbits.secureddata.SecuredDataTestSupport.user; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; /******************************************************************************* ** Reading rule rows (classifications, retention rules, policies) is a query on ** a secured-data table, and other code runs inside that query: a virtual field ** value provider or a post-query customizer that another QBit or the host put ** on the table. When that code runs a query with filter criteria, core asks ** this QBit for a classification - which needs the rows being read. ** ** These tests put such code on the rule tables and prove: the read asks for ** stored fields only, so no value provider runs inside it; a question asked ** inside a read is answered from the rows held before and starts no other ** read; only the reading thread, as the system, can be given that answer; and ** a read that fails is tried again by the next question. *******************************************************************************/ class SecuredDataRuleReloadTest { QInstance instance; QTableMetaData person; SecuredDataRuntime runtime; /******************************************************************************* ** Something to do inside a query on a rule table. *******************************************************************************/ @FunctionalInterface interface NestedWork { /******************************************************************************* ** Do it. *******************************************************************************/ void run() throws Exception; } /******************************************************************************* ** Stands in for another QBit's virtual field on a secured-data table: filling ** it runs a query that has filter criteria. *******************************************************************************/ public static class NestedQueryProvider implements VirtualFieldValueProviderInterface { static final AtomicInteger RUNS = new AtomicInteger(); /******************************************************************************* ** Fill the field from a query with criteria, as a real provider does. *******************************************************************************/ @Override public void fill(QTableMetaData table, String fieldName, List records, QueryInput input) throws QException { RUNS.incrementAndGet(); queryWithCriteria(); records.forEach(record -> record.setValue(fieldName, "filled")); } } /******************************************************************************* ** A post-query customizer on a secured-data table. Core runs it for every ** query on the table, whatever fields the query names. *******************************************************************************/ public static class NestedWorkCustomizer implements TableCustomizerInterface { static final AtomicInteger RUNS = new AtomicInteger(); static volatile NestedWork work; /******************************************************************************* ** Run the work the test asked for, inside the query. *******************************************************************************/ @Override public List postQuery(QueryOrGetInputInterface queryInput, List records) throws QException { NestedWork toRun = work; if(toRun != null) { RUNS.incrementAndGet(); try { toRun.run(); } catch(QException | RuntimeException e) { throw (e); } catch(Exception e) { throw (new QException("Nested work failed", e)); } } return (records); } } /******************************************************************************* ** A person table with a date, so a retention rule on it is valid. *******************************************************************************/ @BeforeEach void init() throws Exception { instance = setup(); person = instance.getTable("person"); person.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("createDate")); runtime = SecuredDataRuntime.current(); NestedQueryProvider.RUNS.set(0); NestedWorkCustomizer.RUNS.set(0); NestedWorkCustomizer.work = null; } /******************************************************************************* ** Cleanup. *******************************************************************************/ @AfterEach void cleanup() { NestedWorkCustomizer.work = null; close(); } /******************************************************************************* ** A query with filter criteria on a host table: core validates the operand ** of a classified field, so it asks for the field's classification - and ** this QBit then asks whether a retention rule names the field. *******************************************************************************/ private static void queryWithCriteria() throws QException { new QueryAction().execute(new QueryInput("person").withFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, 1)))); } /******************************************************************************* ** Another QBit's virtual field on a secured-data table. *******************************************************************************/ private void addProvider(String table) { instance.getTable(table).addVirtualField(new QVirtualFieldMetaData("labels", QFieldType.STRING).withIsQuerySelectable(true) .withValueProvider(new QCodeReference(NestedQueryProvider.class))); } /******************************************************************************* ** A post-query customizer on a secured-data table. *******************************************************************************/ private void addCustomizer(String table) { instance.getTable(table).withCustomizer(TableCustomizers.POST_QUERY_RECORD, new QCodeReference(NestedWorkCustomizer.class)); } /******************************************************************************* ** A row as another node writes it: straight into the store. None of this ** node's customizers run and nothing here reads the rule rows; this node ** learns of the change when its version moves (the cluster event). *******************************************************************************/ private static QRecord writtenElsewhere(String table, QRecord row) throws QException { return (MemoryRecordStore.getInstance().insert(new InsertInput(table).withRecord(row), true).getFirst()); } /******************************************************************************* ** A classification of a person field, written by another node. *******************************************************************************/ private QRecord classifiedElsewhere(String field, String classification) throws QException { QRecord row = writtenElsewhere(SecuredDataTables.CLASSIFICATION, new QRecord().withValue("table_name", "person") .withValue("field_name", field).withValue("classification", classification)); runtime.invalidateClassifications(); return (row); } /******************************************************************************* ** A classification row changed by another node. *******************************************************************************/ private void reclassifiedElsewhere(QRecord row, String classification) throws QException { MemoryRecordStore.getInstance().update(new UpdateInput(SecuredDataTables.CLASSIFICATION).withRecord(new QRecord() .withValue("id", row.getValue("id")).withValue("classification", classification)), false); runtime.invalidateClassifications(); } /******************************************************************************* ** An active retention rule on one field of the person table, written by ** another node. *******************************************************************************/ private void ruleWrittenElsewhere(String field) throws QException { writtenElsewhere(SecuredDataTables.RETENTION_RULE, new QRecord().withValue("name", "30 days: " + field).withValue("table_name", "person") .withValue("field_names", field).withValue("age_field_name", "createDate").withValue("age_days", 30).withValue("action", "NULL_WITH_MARKER") .withValue("audit_treatment", "SHREDDABLE").withValue("is_active", true)); runtime.invalidateRetentionRules(); } /******************************************************************************* ** The classification of a person field. *******************************************************************************/ private DataClassification classificationOf(String field) { return (SecuredDataClassifications.effective(person, field)); } /******************************************************************************* ** The reported failure: a value provider on the classification table whose ** work is a query with criteria, and one classification row. Before the fix ** the next classification question recursed until the stack overflowed, and ** so did every question after it. Now the rule read names its stored fields, ** so the provider does not run inside it at all. *******************************************************************************/ @Test void aValueProviderOnTheClassificationTableDoesNotRunInsideTheRuleRead() throws Exception { addProvider(SecuredDataTables.CLASSIFICATION); classification("notes", "PHI"); assertThat(classificationOf("notes")).isEqualTo(DataClassification.PHI); queryWithCriteria(); assertThat(NestedQueryProvider.RUNS).hasValue(0); assertThat(SystemWrites.query(SecuredDataTables.CLASSIFICATION, new QQueryFilter())).singleElement() .satisfies(row -> assertThat(row.getValues()).doesNotContainKey("labels").containsEntry("field_name", "notes")); } /******************************************************************************* ** The virtual field is still there for a query that asks for every field (a ** screen listing the classifications). Its provider's query with criteria ** asks for a classification while the rows are out of date: the rows are ** read once, inside the provider, and nothing recurses. *******************************************************************************/ @Test void anOrdinaryQueryOnTheClassificationTableStillFillsTheVirtualField() throws Exception { addProvider(SecuredDataTables.CLASSIFICATION); classifiedElsewhere("notes", "PHI"); List rows = SystemWrites.call(() -> new QueryAction().execute(new QueryInput(SecuredDataTables.CLASSIFICATION)).getRecords()); assertThat(rows).singleElement().satisfies(row -> assertThat(row.getValueString("labels")).isEqualTo("filled")); assertThat(NestedQueryProvider.RUNS).hasValue(1); assertThat(classificationOf("notes")).isEqualTo(DataClassification.PHI); } /******************************************************************************* ** A post-query customizer runs inside the rule read whatever fields the read ** names. Its query with criteria asks for a classification in the middle of ** the read of the classifications: before the fix that started the read again ** and recursed until the stack overflowed. Now the rows are read once. *******************************************************************************/ @Test void aQuestionAskedInsideTheClassificationReadDoesNotStartAnotherRead() throws Exception { addCustomizer(SecuredDataTables.CLASSIFICATION); classifiedElsewhere("notes", "PHI"); NestedWorkCustomizer.work = SecuredDataRuleReloadTest::queryWithCriteria; assertThat(classificationOf("notes")).isEqualTo(DataClassification.PHI); assertThat(NestedWorkCustomizer.RUNS).hasValue(1); assertThat(classificationOf("email")).isEqualTo(DataClassification.PII); assertThat(NestedWorkCustomizer.RUNS).hasValue(1); } /******************************************************************************* ** What a question asked inside the read is told, and who can ask it. ** ** The nested question gets the rows this node held when the read began (no ** rows, the first time; then the rows of the previous version), in both ** directions of a change. It can only be asked by the thread doing the read, ** which is inside the QBit's system read: the session is the system session ** and the resolver answers FULL there without looking at any classification, ** so no person's field access is decided from that answer. The question that ** started the read gets the new rows. *******************************************************************************/ @Test void aQuestionAskedInsideTheReadIsAnsweredFromTheRowsHeldBefore() throws Exception { addCustomizer(SecuredDataTables.CLASSIFICATION); assertThat(classificationOf("notes")).isNull(); List nestedAnswers = new ArrayList<>(); List nestedAsSystem = new ArrayList<>(); List nestedDecisions = new ArrayList<>(); NestedWork ask = () -> { nestedAnswers.add(classificationOf("notes")); nestedAsSystem.add(QContext.getQSession() instanceof QSystemUserSession && SystemWrites.active()); nestedDecisions.add(runtime.resolver().resolve(person, person.getField("notes"))); }; ///////////////////////////////////////////////////////////////////////// // a row raises the field: inside the read it is still unclassified, // // and the question that started the read is answered PHI // ///////////////////////////////////////////////////////////////////////// QRecord row = classifiedElsewhere("notes", "PHI"); NestedWorkCustomizer.work = ask; assertThat(classificationOf("notes")).isEqualTo(DataClassification.PHI); NestedWorkCustomizer.work = null; assertThat(nestedAnswers).containsExactly((DataClassification) null); ///////////////////////////////////////////////////////////////////////// // the row is changed to lower the field: inside the read it is still // // PHI, and the question that started the read is answered PUBLIC // ///////////////////////////////////////////////////////////////////////// reclassifiedElsewhere(row, "PUBLIC"); NestedWorkCustomizer.work = ask; assertThat(classificationOf("notes")).isEqualTo(DataClassification.PUBLIC); NestedWorkCustomizer.work = null; assertThat(nestedAnswers).containsExactly(null, DataClassification.PHI); assertThat(nestedAsSystem).containsExactly(true, true); assertThat(nestedDecisions).containsExactly(FieldAccessDecision.FULL, FieldAccessDecision.FULL); assertThat(NestedWorkCustomizer.RUNS).hasValue(2); } /******************************************************************************* ** A request on another thread is never given the rows held before: while the ** read is running it waits, and then it is answered from the rows the read ** returned. *******************************************************************************/ @Test void aRequestOnAnotherThreadWaitsForTheReadAndIsGivenItsRows() throws Exception { addCustomizer(SecuredDataTables.CLASSIFICATION); QRecord row = classifiedElsewhere("notes", "PUBLIC"); assertThat(classificationOf("notes")).isEqualTo(DataClassification.PUBLIC); reclassifiedElsewhere(row, "PHI"); CountDownLatch readIsRunning = new CountDownLatch(1); CountDownLatch letReadFinish = new CountDownLatch(1); NestedWorkCustomizer.work = () -> { readIsRunning.countDown(); assertThat(letReadFinish.await(30, TimeUnit.SECONDS)).isTrue(); }; ExecutorService pool = Executors.newFixedThreadPool(2); try { Future reader = pool.submit(asUser("u2", () -> classificationOf("notes"))); assertThat(readIsRunning.await(30, TimeUnit.SECONDS)).isTrue(); Future other = pool.submit(asUser("u3", () -> classificationOf("notes"))); assertThatThrownBy(() -> other.get(500, TimeUnit.MILLISECONDS)).isInstanceOf(TimeoutException.class); letReadFinish.countDown(); assertThat(reader.get(30, TimeUnit.SECONDS)).isEqualTo(DataClassification.PHI); assertThat(other.get(30, TimeUnit.SECONDS)).isEqualTo(DataClassification.PHI); assertThat(NestedWorkCustomizer.RUNS).hasValue(1); } finally { letReadFinish.countDown(); pool.shutdownNow(); } } /******************************************************************************* ** A person's request on its own thread. *******************************************************************************/ private Callable asUser(String id, Callable work) { return (() -> { QContext.init(instance, user(id)); try { return (work.call()); } finally { QContext.clear(); } }); } /******************************************************************************* ** A read that fails (here, in code running inside it, after that code asked a ** nested question) throws to the request that started it and leaves nothing ** behind: the next request reads again and gets the current rows - not the ** rows held before, and not another failure. An Error, such as the stack ** overflow this defect used to end in, is no different. *******************************************************************************/ @Test void aClassificationReadThatFailsIsTriedAgainByTheNextQuestion() throws Exception { addCustomizer(SecuredDataTables.CLASSIFICATION); QRecord row = classifiedElsewhere("notes", "PUBLIC"); assertThat(classificationOf("notes")).isEqualTo(DataClassification.PUBLIC); reclassifiedElsewhere(row, "PHI"); NestedWorkCustomizer.work = () -> { classificationOf("notes"); throw (new QException("The store is unavailable")); }; assertThatThrownBy(() -> classificationOf("notes")).isInstanceOf(IllegalStateException.class); NestedWorkCustomizer.work = () -> { classificationOf("notes"); throw (new StackOverflowError()); }; assertThatThrownBy(() -> classificationOf("notes")).isInstanceOf(StackOverflowError.class); NestedWorkCustomizer.work = null; assertThat(classificationOf("notes")).isEqualTo(DataClassification.PHI); } /******************************************************************************* ** The active retention rules have the same shape: core's classification ** question for a host field also asks whether a retention rule names the ** field. A value provider on the retention rule table, with one active rule, ** recursed the same way before the fix; it does not run inside the rule read. *******************************************************************************/ @Test void aValueProviderOnTheRetentionRuleTableDoesNotRunInsideTheRuleRead() throws Exception { addProvider(SecuredDataTables.RETENTION_RULE); ruleWrittenElsewhere("notes"); assertThat(RetentionRules.active("person", "notes")).hasSize(1); queryWithCriteria(); assertThat(NestedQueryProvider.RUNS).hasValue(0); } /******************************************************************************* ** A question asked inside the read of the retention rules starts no other ** read and is answered from the rules held before; the question that started ** the read gets the new rules. A read that fails is tried again. *******************************************************************************/ @Test void aQuestionAskedInsideTheRetentionRuleReadIsAnsweredFromTheRulesHeldBefore() throws Exception { addCustomizer(SecuredDataTables.RETENTION_RULE); assertThat(RetentionRules.active("person", "notes")).isEmpty(); ruleWrittenElsewhere("notes"); List nestedAnswers = new ArrayList<>(); NestedWorkCustomizer.work = () -> { queryWithCriteria(); nestedAnswers.add(RetentionRules.active("person", "notes").size()); if(nestedAnswers.size() == 1) { throw (new QException("The store is unavailable")); } }; assertThatThrownBy(() -> RetentionRules.active("person", "notes")).isInstanceOf(IllegalStateException.class); assertThat(RetentionRules.active("person", "notes")).hasSize(1); assertThat(nestedAnswers).containsExactly(0, 0); assertThat(NestedWorkCustomizer.RUNS).hasValue(2); assertThat(RetentionRules.active("person", "notes")).hasSize(1); assertThat(NestedWorkCustomizer.RUNS).hasValue(2); } /******************************************************************************* ** A transaction that edited a rule reads the rules through itself, not from ** the rules the node keeps. A question asked inside that read is answered ** from the rules every thread shares: before the fix it read the edits again, ** and recursed until the stack overflowed. *******************************************************************************/ @Test void aQuestionAskedInsideAReadOfATransactionsOwnEditsDoesNotReadThemAgain() throws Exception { addCustomizer(SecuredDataTables.RETENTION_RULE); ruleWrittenElsewhere("notes"); assertThat(RetentionRules.active("person", "notes")).hasSize(1); QContext.setTransaction(new QBackendTransaction()); RetentionRules.editedInTransaction(); try { NestedWorkCustomizer.work = SecuredDataRuleReloadTest::queryWithCriteria; assertThat(RetentionRules.active("person", "notes")).hasSize(1); assertThat(NestedWorkCustomizer.RUNS).hasValue(1); } finally { //////////////////////////////////////////////////////////////////////// // the next question outside a transaction ends this thread's edits // //////////////////////////////////////////////////////////////////////// NestedWorkCustomizer.work = null; QContext.setTransaction(null); RetentionRules.active("person", "notes"); } } /******************************************************************************* ** The policies are read the same way (rows first, version afterwards), so ** they are guarded the same way: code inside the policy read that asks the ** resolver a policy question starts no other read, and a read that fails is ** tried again. *******************************************************************************/ @Test void aQuestionAskedInsideThePolicyReadDoesNotStartAnotherRead() throws Exception { addCustomizer(SecuredDataTables.POLICY); addProvider(SecuredDataTables.POLICY); assertThat(runtime.resolver().revealPolicies(person, "email")).isEmpty(); writtenElsewhere(SecuredDataTables.POLICY, policyRow("cs_admin", "PII", "REVEALABLE")); runtime.invalidatePolicies(); List nestedAnswers = new ArrayList<>(); NestedWorkCustomizer.work = () -> { nestedAnswers.add(runtime.resolver().revealPolicies(person, "email").size()); if(nestedAnswers.size() == 1) { throw (new QException("The store is unavailable")); } }; assertThatThrownBy(() -> runtime.resolver().revealPolicies(person, "email")).isInstanceOf(IllegalStateException.class); assertThat(runtime.resolver().revealPolicies(person, "email")).hasSize(1); assertThat(nestedAnswers).containsExactly(0, 0); assertThat(NestedQueryProvider.RUNS).hasValue(0); NestedWorkCustomizer.RUNS.set(0); assertThat(runtime.resolver().revealPolicies(person, "email")).hasSize(1); assertThat(NestedWorkCustomizer.RUNS).hasValue(0); } /******************************************************************************* ** The rows themselves, without a table: nothing is held before the first ** read, a nested question never reads, a failed read is forgotten, and a new ** version is read once. *******************************************************************************/ @Test void ruleRowsAreReadOncePerVersionAndNeverFromInsideTheirOwnRead() { RuleRows rows = new RuleRows(); AtomicInteger reads = new AtomicInteger(); List first = List.of(new QRecord().withValue("id", 1)); List second = List.of(new QRecord().withValue("id", 2)); assertThatThrownBy(() -> rows.get(1, () -> { reads.incrementAndGet(); assertThat(rows.get(1, () -> fail("a nested question must not read"))).isEmpty(); throw (new IllegalStateException("The store is unavailable")); })).isInstanceOf(IllegalStateException.class); assertThat(rows.get(1, () -> { reads.incrementAndGet(); assertThat(rows.get(1, () -> fail("a nested question must not read"))).isEmpty(); return (first); })).isEqualTo(first); assertThat(rows.get(1, () -> fail("rows of this version are held"))).isEqualTo(first); assertThat(rows.get(2, () -> { reads.incrementAndGet(); assertThat(rows.get(2, () -> fail("a nested question must not read"))).isEqualTo(first); return (second); })).isEqualTo(second); assertThat(reads).hasValue(3); } /******************************************************************************* ** A read that must not happen. *******************************************************************************/ private static List fail(String why) { throw (new AssertionError(why)); } }