ci: use the shared branch-aware CodeQL worker

Co-Authored-By: Codex <noreply@openai.com>

.circleci/config.yml

@@ -1,65 +1,14 @@
 version: 2.1
-
+setup: true
 orbs:
-  qqq-orb: kingsrook/qqq-orb@0.6.9
-
-commands:
-  configure-package-install:
-    steps:
-      - run:
-          name: Keep package installation unattended
-          command: |
-            # These build machines are discarded after the job.
-            sudo install -d /etc/needrestart/conf.d
-            printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
-
+  qqq-orb: kingsrook/qqq-orb@0.7.12
 workflows:
-
-  test_only:
+  setup:
     jobs:
-      - qqq-orb/mvn_test_only:
-          pre-steps:
-            - configure-package-install
-          context: [ qqq-maven-registry-credentials ]
-          filters:
-            branches:
-              ignore: /(develop|release/.*|integration.*)/
-            tags:
-              only: []
-
-  publish_snapshot:
-    jobs:
-      - qqq-orb/mvn_publish:
-          pre-steps:
-            - configure-package-install
-          context: [ qqq-maven-registry-credentials ]
-          branch_type: snapshot
-          filters:
-            branches:
-              only:
-                - develop
-
-  publish_release_candidate:
-    jobs:
-      - qqq-orb/mvn_publish:
-          pre-steps:
-            - configure-package-install
-          context: [ qqq-maven-registry-credentials ]
-          branch_type: release_candidate
-          filters:
-            branches:
-              only:
-                - /release\/.*/
-
-  publish_release:
-    jobs:
-      - qqq-orb/mvn_publish:
-          pre-steps:
-            - configure-package-install
-          context: [ qqq-maven-registry-credentials ]
-          branch_type: release
-          filters:
-            branches:
-              ignore: /.*/
-            tags:
-              only: [ /^v\d+\.\d+\.\d+$/ ]
+    - qqq-orb/generate_pipeline:
+        orb_version: 0.7.12
+        filters:
+          branches:
+            ignore: automation/qbit-rc-pending
+          tags:
+            only: /^v\d+\.\d+\.\d+$/

.github/workflows/codeql.yml

@@ -0,0 +1,23 @@
+name: CodeQL
+'on':
+  push:
+    branches:
+    - main
+    - develop
+    - release/**
+    - hotfix/**
+  pull_request: {}
+  schedule:
+  - cron: 41 16 * * 0
+  workflow_dispatch: {}
+permissions:
+  contents: read
+  actions: read
+  security-events: write
+  packages: read
+concurrency:
+  group: codeql-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+jobs:
+  analyze:
+    uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22

.github/workflows/release-rc.yml

@@ -1,25 +1,74 @@
-# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
-# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
-# the release line and the release/* push runs CircleCI publish_release_candidate.
 name: Release candidate
-
-on:
+'on':
   workflow_dispatch:
     inputs:
       dry_run:
-        description: Print the plan without merging or creating branches
+        description: Plan a cut without changing release state
         type: boolean
         default: false
+      finalize:
+        description: Retry finalization of an existing cut; do not cut another RC
+        type: boolean
+        default: false
+      recover:
+        description: Recover the same published RC after a deployment timeout; never
+          reupload
+        type: boolean
+        default: false
+      circle_build:
+        description: Original Circle publisher build number (recovery only)
+        type: string
+        default: ''
+      merge_sha:
+        description: Existing cut merge_sha (finalize retry only)
+        type: string
+        default: ''
+      bump_sha:
+        description: Existing cut bump_sha (finalize retry only)
+        type: string
+        default: ''
+      branch:
+        description: Existing cut branch (finalize retry only)
+        type: string
+        default: ''
+      version:
+        description: Existing cut version (finalize retry only)
+        type: string
+        default: ''
   pull_request:
-    types: [closed]
-    branches: [develop]
-
+    types:
+    - closed
+    branches:
+    - develop
+  repository_dispatch:
+    types:
+    - qbit-rc-published
 permissions:
   contents: write
-
+  actions: read
+  checks: read
 jobs:
   release-rc:
-    if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
-    uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
+    if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
+      || (github.event_name == 'pull_request' && github.event.pull_request.merged
+      == true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
+      'release:rc'))
+    uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
     with:
-      dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
+      dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
+        }}
+      finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
+        == 'workflow_dispatch' && inputs.finalize) }}
+      recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
+        }}
+      circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
+        || '' }}
+      merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
+        || inputs.merge_sha || '' }}
+      bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
+        || inputs.bump_sha || '' }}
+      branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
+        || inputs.branch || '' }}
+      version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
+        || inputs.version || '' }}
+    secrets: inherit

.qqq-ci.yml

@@ -0,0 +1,16 @@
+profile: qbit
+orb_version: 0.7.12
+default_branch: develop
+java_version: '21'
+browsers: true
+api_checks: false
+docs_only: false
+release_on_main: false
+release_tag_policy: stable
+test_branch_policy: non-release-no-integration
+publish:
+  snapshot: true
+  release_candidate: true
+  release: true
+  asciidoc: false
+release_completion: true

CHANGELOG.md

@@ -4,6 +4,12 @@ All notable changes are documented here, following [Keep a Changelog](https://ke
 
 ## [Unreleased]
 
+### Fixed: access-log lifecycle
+- The access-log worker and JVM shutdown hook hold the logger weakly, so an instance the host drops (tests, reloads) can be collected; its worker stops at the next tick.
+- `withRegisterShutdownHook(false)` for hosts that call `SecuredDataRuntime.close()` themselves; `SecuredDataAccessLogger.isWorkerRunning()`.
+- Virtual (computed) fields resolve through the table classification instead of throwing (#18).
+- Idle access-log ticks open no context, so the worker never validates an unfinished host instance, and a failed tick no longer stops the worker (#18).
+
 ### Changed: UI API 1.6.0
 - UI API extension secured-data 1.1.0 (requires QQQ 4.1.0-RC.15, UI API 1.6.0): operationIds, standard error responses with BasicErrorResponseV1, extension tags, and retained SecuredDataErrorV1 schema deprecated since 1.1.0.
 

README.md

@@ -101,7 +101,7 @@ Policy/break-glass caches are bounded per instance and keyed by relevant user pe
 
 The extra `search_value_fingerprints` TEXT column persists JSON arrays of HMAC-SHA256 fingerprints for distinct-value detection after restarts and across nodes. Use a shared secret key supplier on multi-node deployments; the default is a random per-JVM key, so historical fingerprints from another JVM/key cannot be compared reliably.
 
-One daemon worker drains a bounded queue as a system session. Enqueueing does not wait for persistence. Full queues and failed writes warn without values and increment `getDroppedCount()`. The app can expose this counter later. Explicitly call the instance's `SecuredDataRuntime.close()` at host shutdown; a JVM shutdown hook also drains accepted rows. An abrupt process crash can lose queued rows.
+One daemon worker drains a bounded queue as a system session. Enqueueing does not wait for persistence. Full queues and failed writes warn without values and increment `getDroppedCount()`. The app can expose this counter later. Explicitly call the instance's `SecuredDataRuntime.close()` at host shutdown; a JVM shutdown hook also drains accepted rows unless `withRegisterShutdownHook(false)` turns it off (hosts that close the runtime themselves). Neither the worker nor the hook holds the instance strongly: an instance the host drops (tests, reloads) can be collected, and its worker stops at the next tick. An abrupt process crash can lose queued rows.
 
 Flagged rows raise the optional alerts producer signal `secured_data.enumeration`, deduplicated by `secured_data.enumeration:<user>:<table>.<field>:<UTC date>`. RC.5 already provides `QAlerts`; without an installed sink it logs the signal. Its catalog validator rejects underscores in type names, so this mandated type is sent through the producer without catalog registration until that core restriction is reconciled. No values or fingerprints are sent to the alert sink.
 
@@ -144,6 +144,7 @@ The classification field picker uses CUSTOM possible values, reading `table_name
 | `withFingerprintKeySupplier(Supplier<byte[]>)` | Random 32-byte JVM key; supplied key must be at least 16 bytes |
 | `withAccessLogQueueCapacity(int)` | 4096 rows |
 | `withAccessLogFlushIntervalMillis(long)` | 250 ms |
+| `withRegisterShutdownHook(boolean)` | true; JVM hook that drains the access log at shutdown |
 | `withExactMatchPerHour(int)` | 20; zero disables |
 | `withMissAlertConsecutive(int)` | 5; zero disables |
 | `withDistinctValueAlert(int)` | 15; zero disables |

src/main/java/com/kingsrook/qbits/secureddata/SecuredDataAccessLogger.java

@@ -21,6 +21,7 @@
 package com.kingsrook.qbits.secureddata;
 
 
+import java.lang.ref.WeakReference;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayDeque;
@@ -90,7 +91,12 @@ public class SecuredDataAccessLogger implements AutoCloseable
 
 
    /*******************************************************************************
-    ** One daemon drains this instance's queue; the JVM hook flushes on shutdown.
+    ** One daemon drains this instance's queue; the optional JVM hook flushes on
+    ** shutdown. Neither holds the logger strongly: once the host drops its
+    ** instance (a test, a reloaded instance), the logger, runtime and instance
+    ** can be collected, and the worker stops at its next tick. Hosts that close
+    ** the runtime themselves can turn the hook off
+    ** (SecuredDataQBitConfig.withRegisterShutdownHook(false)).
     *******************************************************************************/
    public SecuredDataAccessLogger(QInstance instance, SecuredDataRuntime runtime)
    {
@@ -103,10 +109,69 @@ public class SecuredDataAccessLogger implements AutoCloseable
          thread.setDaemon(true);
          return (thread);
       });
+      WeakReference<SecuredDataAccessLogger> self = new WeakReference<>(this);
       long interval = runtime.config().getAccessLogFlushIntervalMillis();
-      worker.scheduleWithFixedDelay(this::flush, interval, interval, TimeUnit.MILLISECONDS);
-      shutdownHook = new Thread(this::close, "secured-data-access-log-shutdown");
-      Runtime.getRuntime().addShutdownHook(shutdownHook);
+      worker.scheduleWithFixedDelay(new WeakTick(self, worker), interval, interval, TimeUnit.MILLISECONDS);
+      shutdownHook = runtime.config().getRegisterShutdownHook() ? new Thread(new WeakClose(self), "secured-data-access-log-shutdown") : null;
+      if(shutdownHook != null)
+      {
+         Runtime.getRuntime().addShutdownHook(shutdownHook);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The worker's task, holding the logger weakly: when the logger has been
+    ** collected the worker shuts itself down.
+    *******************************************************************************/
+   private record WeakTick(WeakReference<SecuredDataAccessLogger> logger, ScheduledExecutorService worker) implements Runnable
+   {
+      /*******************************************************************************
+       ** Drain, or stop once the logger is gone.
+       *******************************************************************************/
+      @Override
+      public void run()
+      {
+         SecuredDataAccessLogger target = logger.get();
+         if(target == null)
+         {
+            worker.shutdown();
+            return;
+         }
+         target.scheduledFlush();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The JVM shutdown hook, holding the logger weakly.
+    *******************************************************************************/
+   private record WeakClose(WeakReference<SecuredDataAccessLogger> logger) implements Runnable
+   {
+      /*******************************************************************************
+       ** Drain accepted rows at JVM shutdown, if the logger still exists.
+       *******************************************************************************/
+      @Override
+      public void run()
+      {
+         SecuredDataAccessLogger target = logger.get();
+         if(target != null)
+         {
+            target.close();
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the background worker is still running (false after close).
+    *******************************************************************************/
+   public boolean isWorkerRunning()
+   {
+      return (!worker.isShutdown());
    }
 
 
@@ -332,13 +397,38 @@ public class SecuredDataAccessLogger implements AutoCloseable
 
 
 
+   /*******************************************************************************
+    ** The worker's tick. An exception would cancel every later run of a
+    ** scheduled task, so a failed drain is logged (value-free) and retried on
+    ** the next tick; the rows stay queued.
+    *******************************************************************************/
+   void scheduledFlush()
+   {
+      try
+      {
+         flush();
+      }
+      catch(RuntimeException e)
+      {
+         LOG.warn("Unable to drain the secured-data access log; retrying on the next tick");
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Drain safely both from the worker and explicit test/host shutdown calls.
+    ** An empty queue opens no context: QContext validates an unvalidated
+    ** instance, and the worker starts while the host is still building it.
     *******************************************************************************/
    public void flush()
    {
       synchronized(flushLock)
       {
+         if(queue.isEmpty())
+         {
+            return;
+         }
          QContext.withTemporaryContext(new CapturedContext(instance, new QSystemUserSession(instance)), () ->
          {
             PendingLog pending;
@@ -463,7 +553,7 @@ public class SecuredDataAccessLogger implements AutoCloseable
       }
       worker.shutdown();
       flush();
-      if(Thread.currentThread() != shutdownHook)
+      if(shutdownHook != null && Thread.currentThread() != shutdownHook)
       {
          try
          {

src/main/java/com/kingsrook/qbits/secureddata/SecuredDataClassifications.java

@@ -54,7 +54,11 @@ public final class SecuredDataClassifications
       {
          return (DataClassification.valueOf(row.getValueString("classification")));
       }
-      QFieldMetaData field = table.getField(fieldName);
+      ////////////////////////////////////////////////////////////////////
+      // getFields, not getField: a virtual field is not in the map and  //
+      // getField throws, while it still takes the table classification //
+      ////////////////////////////////////////////////////////////////////
+      QFieldMetaData field = table.getFields() == null || fieldName == null ? null : table.getFields().get(fieldName);
       DataClassification classification = field == null ? null : SecuredDataMetadata.classification(table, field);
       return (classification == null ? SecuredDataMetadata.classification(table, null) : classification);
    }

src/main/java/com/kingsrook/qbits/secureddata/SecuredDataQBitConfig.java

@@ -115,6 +115,7 @@ public class SecuredDataQBitConfig implements QBitConfig
    private Clock clock = Clock.systemUTC();
    private int accessLogQueueCapacity = 4096;
    private long accessLogFlushIntervalMillis = 250;
+   private boolean registerShutdownHook = true;
    private int exactMatchPerHour = 20;
    private int missAlertConsecutive = 5;
    private int distinctValueAlert = 15;
@@ -216,6 +217,29 @@ public class SecuredDataQBitConfig implements QBitConfig
 
 
 
+   /*******************************************************************************
+    ** Whether the access logger registers a JVM shutdown hook that drains
+    ** accepted rows (default true). Hosts that call SecuredDataRuntime.close()
+    ** from their own shutdown sequence turn it off.
+    *******************************************************************************/
+   public boolean getRegisterShutdownHook()
+   {
+      return (registerShutdownHook);
+   }
+
+
+
+   /*******************************************************************************
+    ** Configure registerShutdownHook.
+    *******************************************************************************/
+   public SecuredDataQBitConfig withRegisterShutdownHook(boolean registerShutdownHook)
+   {
+      this.registerShutdownHook = registerShutdownHook;
+      return (this);
+   }
+
+
+
    /*******************************************************************************
     ** Configure accessLogFlushIntervalMillis.
     *******************************************************************************/

src/test/java/com/kingsrook/qbits/secureddata/SecuredDataAccessLoggerTest.java

@@ -22,6 +22,7 @@ package com.kingsrook.qbits.secureddata;
 
 
 import java.io.Serializable;
+import java.lang.ref.WeakReference;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
@@ -46,6 +47,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Test;
@@ -359,4 +361,92 @@ class SecuredDataAccessLoggerTest
       {
       }
    }
+
+
+   /*******************************************************************************
+    ** The worker starts while the host is still building its instance. An idle
+    ** tick opens no context, so it never validates (or enriches) an unfinished
+    ** instance on the worker thread, and a failing tick is logged instead of
+    ** cancelling every later tick.
+    *******************************************************************************/
+   @Test
+   void idleTicksNeverValidateAnUnfinishedInstance() throws Exception
+   {
+      setup();
+      SecuredDataRuntime runtime = SecuredDataRuntime.current();
+      QInstance unfinished = SecuredDataTestSupport.base();
+      unfinished.addTable(new QTableMetaData().withName("unfinished").withBackendName("not_registered_yet"));
+      SecuredDataAccessLogger idle = new SecuredDataAccessLogger(unfinished, runtime);
+      try
+      {
+         idle.flush();
+         idle.scheduledFlush();
+         assertThat(unfinished.getHasBeenValidated()).isFalse();
+      }
+      finally
+      {
+         idle.close();
+      }
+   }
+
+
+   /*******************************************************************************
+    ** Neither the worker thread nor the JVM shutdown hook keeps a discarded
+    ** instance alive: a host or test that drops its instance without closing
+    ** the runtime lets it be collected.
+    *******************************************************************************/
+   @Test
+   void aDiscardedInstanceCanBeCollected() throws Exception
+   {
+      close();
+      WeakReference<QInstance> discarded = discardedInstance();
+      long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
+      List<byte[]> pressure = new ArrayList<>();
+      while(discarded.get() != null && System.nanoTime() < deadline)
+      {
+         pressure.add(new byte[1 << 20]);
+         if(pressure.size() > 64)
+         {
+            pressure.clear();
+         }
+         System.gc();
+         Thread.sleep(20);
+      }
+      assertThat(discarded.get()).as("instance still reachable from the access logger").isNull();
+   }
+
+
+
+   /*******************************************************************************
+    ** Build, use and drop an instance whose worker ticks every 10 ms.
+    *******************************************************************************/
+   private static WeakReference<QInstance> discardedInstance() throws Exception
+   {
+      QInstance instance = SecuredDataTestSupport.base();
+      instance.addBackend(new com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData().withName("memory")
+         .withBackendType(com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule.class));
+      instance.addTable(person("memory"));
+      new SecuredDataQBitProducer().withQBitConfig(new SecuredDataQBitConfig().withAccessLogFlushIntervalMillis(10)).produce(instance);
+      Thread.sleep(50);
+      QContext.clear();
+      return (new WeakReference<>(instance));
+   }
+
+
+
+   /*******************************************************************************
+    ** Hosts that close the runtime themselves can skip the JVM hook; close
+    ** still drains and stops the worker.
+    *******************************************************************************/
+   @Test
+   void shutdownHookIsOptional() throws Exception
+   {
+      setup(config -> config.withRegisterShutdownHook(false));
+      var logger = SecuredDataRuntime.current().logger();
+      assertThat(logger.isWorkerRunning()).isTrue();
+      logger.onAccess(event(FieldAccessEventType.VIEW, null, null));
+      logger.close();
+      assertThat(logger.isWorkerRunning()).isFalse();
+      assertThat(logs()).hasSize(1);
+   }
 }

src/test/java/com/kingsrook/qbits/secureddata/SecuredDataResolverTest.java

@@ -33,6 +33,7 @@ import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QVirtualFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
@@ -326,4 +327,22 @@ class SecuredDataResolverTest
       CLOCK.now = NOW.plusSeconds(10);
       assertThat(resolver.isRevealActive("person", "email", 7)).isFalse();
    }
+
+
+   /*******************************************************************************
+    ** A virtual (computed) field is not in getFields, and getField throws for
+    ** it: it resolves through the table classification instead of failing the
+    ** whole read.
+    *******************************************************************************/
+   @Test
+   void virtualFieldsTakeTheTableClassification() throws Exception
+   {
+      QVirtualFieldMetaData display = new QVirtualFieldMetaData("displayName", QFieldType.STRING);
+      person.withVirtualFields(List.of(display));
+      assertThat(SecuredDataClassifications.effective(person, "displayName")).isNull();
+      assertThat(resolver.resolve(person, display).tier()).isEqualTo(FieldAccessTier.FULL);
+      person.withSupplementalMetaData(new AiTableMetaData().withClassification(DataClassification.PII));
+      assertThat(SecuredDataClassifications.effective(person, "displayName")).isEqualTo(DataClassification.PII);
+      assertThat(resolver.resolve(person, display).tier()).isEqualTo(FieldAccessTier.MASKED);
+   }
 }