ci: use the shared branch-aware CodeQL worker

Co-Authored-By: Codex <noreply@openai.com>

.circleci/config.yml

@@ -1,65 +1,14 @@
 version: 2.1
-
+setup: true
 orbs:
-  qqq-orb: kingsrook/qqq-orb@0.6.9
-
-commands:
-  configure-package-install:
-    steps:
-      - run:
-          name: Keep package installation unattended
-          command: |
-            # These build machines are discarded after the job.
-            sudo install -d /etc/needrestart/conf.d
-            printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
-
+  qqq-orb: kingsrook/qqq-orb@0.7.12
 workflows:
-
-  test_only:
+  setup:
     jobs:
-      - qqq-orb/mvn_test_only:
-          pre-steps:
-            - configure-package-install
-          context: [ qqq-maven-registry-credentials ]
-          filters:
-            branches:
-              ignore: /(develop|release/.*|integration.*)/
-            tags:
-              only: []
-
-  publish_snapshot:
-    jobs:
-      - qqq-orb/mvn_publish:
-          pre-steps:
-            - configure-package-install
-          context: [ qqq-maven-registry-credentials ]
-          branch_type: snapshot
-          filters:
-            branches:
-              only:
-                - develop
-
-  publish_release_candidate:
-    jobs:
-      - qqq-orb/mvn_publish:
-          pre-steps:
-            - configure-package-install
-          context: [ qqq-maven-registry-credentials ]
-          branch_type: release_candidate
-          filters:
-            branches:
-              only:
-                - /release\/.*/
-
-  publish_release:
-    jobs:
-      - qqq-orb/mvn_publish:
-          pre-steps:
-            - configure-package-install
-          context: [ qqq-maven-registry-credentials ]
-          branch_type: release
-          filters:
-            branches:
-              ignore: /.*/
-            tags:
-              only: [ /^v\d+\.\d+\.\d+$/ ]
+    - qqq-orb/generate_pipeline:
+        orb_version: 0.7.12
+        filters:
+          branches:
+            ignore: automation/qbit-rc-pending
+          tags:
+            only: /^v\d+\.\d+\.\d+$/

.github/workflows/codeql.yml

@@ -0,0 +1,23 @@
+name: CodeQL
+'on':
+  push:
+    branches:
+    - main
+    - develop
+    - release/**
+    - hotfix/**
+  pull_request: {}
+  schedule:
+  - cron: 41 16 * * 0
+  workflow_dispatch: {}
+permissions:
+  contents: read
+  actions: read
+  security-events: write
+  packages: read
+concurrency:
+  group: codeql-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+jobs:
+  analyze:
+    uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22

.github/workflows/release-rc.yml

@@ -1,25 +1,74 @@
-# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
-# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
-# the release line and the release/* push runs CircleCI publish_release_candidate.
 name: Release candidate
-
-on:
+'on':
   workflow_dispatch:
     inputs:
       dry_run:
-        description: Print the plan without merging or creating branches
+        description: Plan a cut without changing release state
         type: boolean
         default: false
+      finalize:
+        description: Retry finalization of an existing cut; do not cut another RC
+        type: boolean
+        default: false
+      recover:
+        description: Recover the same published RC after a deployment timeout; never
+          reupload
+        type: boolean
+        default: false
+      circle_build:
+        description: Original Circle publisher build number (recovery only)
+        type: string
+        default: ''
+      merge_sha:
+        description: Existing cut merge_sha (finalize retry only)
+        type: string
+        default: ''
+      bump_sha:
+        description: Existing cut bump_sha (finalize retry only)
+        type: string
+        default: ''
+      branch:
+        description: Existing cut branch (finalize retry only)
+        type: string
+        default: ''
+      version:
+        description: Existing cut version (finalize retry only)
+        type: string
+        default: ''
   pull_request:
-    types: [closed]
-    branches: [develop]
-
+    types:
+    - closed
+    branches:
+    - develop
+  repository_dispatch:
+    types:
+    - qbit-rc-published
 permissions:
   contents: write
-
+  actions: read
+  checks: read
 jobs:
   release-rc:
-    if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
-    uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
+    if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
+      || (github.event_name == 'pull_request' && github.event.pull_request.merged
+      == true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
+      'release:rc'))
+    uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
     with:
-      dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
+      dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
+        }}
+      finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
+        == 'workflow_dispatch' && inputs.finalize) }}
+      recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
+        }}
+      circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
+        || '' }}
+      merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
+        || inputs.merge_sha || '' }}
+      bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
+        || inputs.bump_sha || '' }}
+      branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
+        || inputs.branch || '' }}
+      version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
+        || inputs.version || '' }}
+    secrets: inherit

.qqq-ci.yml

@@ -0,0 +1,16 @@
+profile: qbit
+orb_version: 0.7.12
+default_branch: develop
+java_version: '21'
+browsers: true
+api_checks: false
+docs_only: false
+release_on_main: false
+release_tag_policy: stable
+test_branch_policy: non-release-no-integration
+publish:
+  snapshot: true
+  release_candidate: true
+  release: true
+  asciidoc: false
+release_completion: true

CHANGELOG.md

@@ -4,6 +4,15 @@ All notable changes are documented here, following [Keep a Changelog](https://ke
 
 ## [Unreleased]
 
+### Fixed: access-log lifecycle
+- The access-log worker and JVM shutdown hook hold the logger weakly, so an instance the host drops (tests, reloads) can be collected; its worker stops at the next tick.
+- `withRegisterShutdownHook(false)` for hosts that call `SecuredDataRuntime.close()` themselves; `SecuredDataAccessLogger.isWorkerRunning()`.
+- Virtual (computed) fields resolve through the table classification instead of throwing (#18).
+- Idle access-log ticks open no context, so the worker never validates an unfinished host instance, and a failed tick no longer stops the worker (#18).
+
+### Changed: UI API 1.6.0
+- UI API extension secured-data 1.1.0 (requires QQQ 4.1.0-RC.15, UI API 1.6.0): operationIds, standard error responses with BasicErrorResponseV1, extension tags, and retained SecuredDataErrorV1 schema deprecated since 1.1.0.
+
 ### Changed — direct core field access and audit protection
 - Remove all #1172/#1182 stand-ins and reflective bridges; use core field-access, classified-value token, audit protection, wipe flag and shredded-event APIs directly.
 - Register the resolver, key source and audit protector through core supplemental metadata.

README.md

@@ -4,7 +4,7 @@ Phases (b), (c), (e) and (f) of [SECURED-DATA](https://github.com/QRun-IO/qqq/bl
 
 ## Compatibility
 
-Java 21, Maven wrapper 3.9.11, and **QQQ 4.1.0-RC.7** or later (it ships field access #1172 and audit value protection #1182).
+Java 21, Maven wrapper 3.9.11, and **QQQ 4.1.0-RC.15** or later (UI API 1.6.0, field access #1172 and audit value protection #1182).
 
 Core classifications, AI metadata, client channels, table-feature permissions and UI API extension points are used directly. The UI API and its contract test are part of the main build; the contract test calls `UiApiContract` directly.
 
@@ -101,7 +101,7 @@ Policy/break-glass caches are bounded per instance and keyed by relevant user pe
 
 The extra `search_value_fingerprints` TEXT column persists JSON arrays of HMAC-SHA256 fingerprints for distinct-value detection after restarts and across nodes. Use a shared secret key supplier on multi-node deployments; the default is a random per-JVM key, so historical fingerprints from another JVM/key cannot be compared reliably.
 
-One daemon worker drains a bounded queue as a system session. Enqueueing does not wait for persistence. Full queues and failed writes warn without values and increment `getDroppedCount()`. The app can expose this counter later. Explicitly call the instance's `SecuredDataRuntime.close()` at host shutdown; a JVM shutdown hook also drains accepted rows. An abrupt process crash can lose queued rows.
+One daemon worker drains a bounded queue as a system session. Enqueueing does not wait for persistence. Full queues and failed writes warn without values and increment `getDroppedCount()`. The app can expose this counter later. Explicitly call the instance's `SecuredDataRuntime.close()` at host shutdown; a JVM shutdown hook also drains accepted rows unless `withRegisterShutdownHook(false)` turns it off (hosts that close the runtime themselves). Neither the worker nor the hook holds the instance strongly: an instance the host drops (tests, reloads) can be collected, and its worker stops at the next tick. An abrupt process crash can lose queued rows.
 
 Flagged rows raise the optional alerts producer signal `secured_data.enumeration`, deduplicated by `secured_data.enumeration:<user>:<table>.<field>:<UTC date>`. RC.5 already provides `QAlerts`; without an installed sink it logs the signal. Its catalog validator rejects underscores in type names, so this mandated type is sent through the producer without catalog registration until that core restriction is reconciled. No values or fingerprints are sent to the alert sink.
 
@@ -121,7 +121,7 @@ Configure `.withSchedulerName("yourScheduler")` to register the hidden `secured_
 
 ## UI API extension
 
-The `secured-data` extension is version **1.0.0**:
+The `secured-data` extension is version **1.1.0**:
 
 - `POST /qqq/v1/qbit/secured-data/reveal`
 - `GET /qqq/v1/qbit/secured-data/reveal/options?tableName=&fieldName=`
@@ -144,6 +144,7 @@ The classification field picker uses CUSTOM possible values, reading `table_name
 | `withFingerprintKeySupplier(Supplier<byte[]>)` | Random 32-byte JVM key; supplied key must be at least 16 bytes |
 | `withAccessLogQueueCapacity(int)` | 4096 rows |
 | `withAccessLogFlushIntervalMillis(long)` | 250 ms |
+| `withRegisterShutdownHook(boolean)` | true; JVM hook that drains the access log at shutdown |
 | `withExactMatchPerHour(int)` | 20; zero disables |
 | `withMissAlertConsecutive(int)` | 5; zero disables |
 | `withDistinctValueAlert(int)` | 15; zero disables |

pom.xml

@@ -22,8 +22,8 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 
    <properties>
-      <!-- QQQ 4.1.0-RC.7 ships field access (#1172) and audit value protection (#1182) -->
-      <qqq.snapshot.version>4.1.0-RC.7</qqq.snapshot.version>
+      <!-- QQQ 4.1.0-RC.15 ships UI API 1.6.0, field access and audit value protection -->
+      <qqq.snapshot.version>4.1.0-RC.15</qqq.snapshot.version>
       <revision>0.1.0-SNAPSHOT</revision>
       <maven.compiler.source>21</maven.compiler.source>
       <maven.compiler.target>21</maven.compiler.target>

src/main/java/com/kingsrook/qbits/secureddata/SecuredDataAccessLogger.java

@@ -21,6 +21,7 @@
 package com.kingsrook.qbits.secureddata;
 
 
+import java.lang.ref.WeakReference;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayDeque;
@@ -90,7 +91,12 @@ public class SecuredDataAccessLogger implements AutoCloseable
 
 
    /*******************************************************************************
-    ** One daemon drains this instance's queue; the JVM hook flushes on shutdown.
+    ** One daemon drains this instance's queue; the optional JVM hook flushes on
+    ** shutdown. Neither holds the logger strongly: once the host drops its
+    ** instance (a test, a reloaded instance), the logger, runtime and instance
+    ** can be collected, and the worker stops at its next tick. Hosts that close
+    ** the runtime themselves can turn the hook off
+    ** (SecuredDataQBitConfig.withRegisterShutdownHook(false)).
     *******************************************************************************/
    public SecuredDataAccessLogger(QInstance instance, SecuredDataRuntime runtime)
    {
@@ -103,10 +109,69 @@ public class SecuredDataAccessLogger implements AutoCloseable
          thread.setDaemon(true);
          return (thread);
       });
+      WeakReference<SecuredDataAccessLogger> self = new WeakReference<>(this);
       long interval = runtime.config().getAccessLogFlushIntervalMillis();
-      worker.scheduleWithFixedDelay(this::flush, interval, interval, TimeUnit.MILLISECONDS);
-      shutdownHook = new Thread(this::close, "secured-data-access-log-shutdown");
-      Runtime.getRuntime().addShutdownHook(shutdownHook);
+      worker.scheduleWithFixedDelay(new WeakTick(self, worker), interval, interval, TimeUnit.MILLISECONDS);
+      shutdownHook = runtime.config().getRegisterShutdownHook() ? new Thread(new WeakClose(self), "secured-data-access-log-shutdown") : null;
+      if(shutdownHook != null)
+      {
+         Runtime.getRuntime().addShutdownHook(shutdownHook);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The worker's task, holding the logger weakly: when the logger has been
+    ** collected the worker shuts itself down.
+    *******************************************************************************/
+   private record WeakTick(WeakReference<SecuredDataAccessLogger> logger, ScheduledExecutorService worker) implements Runnable
+   {
+      /*******************************************************************************
+       ** Drain, or stop once the logger is gone.
+       *******************************************************************************/
+      @Override
+      public void run()
+      {
+         SecuredDataAccessLogger target = logger.get();
+         if(target == null)
+         {
+            worker.shutdown();
+            return;
+         }
+         target.scheduledFlush();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The JVM shutdown hook, holding the logger weakly.
+    *******************************************************************************/
+   private record WeakClose(WeakReference<SecuredDataAccessLogger> logger) implements Runnable
+   {
+      /*******************************************************************************
+       ** Drain accepted rows at JVM shutdown, if the logger still exists.
+       *******************************************************************************/
+      @Override
+      public void run()
+      {
+         SecuredDataAccessLogger target = logger.get();
+         if(target != null)
+         {
+            target.close();
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether the background worker is still running (false after close).
+    *******************************************************************************/
+   public boolean isWorkerRunning()
+   {
+      return (!worker.isShutdown());
    }
 
 
@@ -332,13 +397,38 @@ public class SecuredDataAccessLogger implements AutoCloseable
 
 
 
+   /*******************************************************************************
+    ** The worker's tick. An exception would cancel every later run of a
+    ** scheduled task, so a failed drain is logged (value-free) and retried on
+    ** the next tick; the rows stay queued.
+    *******************************************************************************/
+   void scheduledFlush()
+   {
+      try
+      {
+         flush();
+      }
+      catch(RuntimeException e)
+      {
+         LOG.warn("Unable to drain the secured-data access log; retrying on the next tick");
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Drain safely both from the worker and explicit test/host shutdown calls.
+    ** An empty queue opens no context: QContext validates an unvalidated
+    ** instance, and the worker starts while the host is still building it.
     *******************************************************************************/
    public void flush()
    {
       synchronized(flushLock)
       {
+         if(queue.isEmpty())
+         {
+            return;
+         }
          QContext.withTemporaryContext(new CapturedContext(instance, new QSystemUserSession(instance)), () ->
          {
             PendingLog pending;
@@ -463,7 +553,7 @@ public class SecuredDataAccessLogger implements AutoCloseable
       }
       worker.shutdown();
       flush();
-      if(Thread.currentThread() != shutdownHook)
+      if(shutdownHook != null && Thread.currentThread() != shutdownHook)
       {
          try
          {

src/main/java/com/kingsrook/qbits/secureddata/SecuredDataClassifications.java

@@ -54,7 +54,11 @@ public final class SecuredDataClassifications
       {
          return (DataClassification.valueOf(row.getValueString("classification")));
       }
-      QFieldMetaData field = table.getField(fieldName);
+      ////////////////////////////////////////////////////////////////////
+      // getFields, not getField: a virtual field is not in the map and  //
+      // getField throws, while it still takes the table classification //
+      ////////////////////////////////////////////////////////////////////
+      QFieldMetaData field = table.getFields() == null || fieldName == null ? null : table.getFields().get(fieldName);
       DataClassification classification = field == null ? null : SecuredDataMetadata.classification(table, field);
       return (classification == null ? SecuredDataMetadata.classification(table, null) : classification);
    }

src/main/java/com/kingsrook/qbits/secureddata/SecuredDataQBitConfig.java

@@ -115,6 +115,7 @@ public class SecuredDataQBitConfig implements QBitConfig
    private Clock clock = Clock.systemUTC();
    private int accessLogQueueCapacity = 4096;
    private long accessLogFlushIntervalMillis = 250;
+   private boolean registerShutdownHook = true;
    private int exactMatchPerHour = 20;
    private int missAlertConsecutive = 5;
    private int distinctValueAlert = 15;
@@ -216,6 +217,29 @@ public class SecuredDataQBitConfig implements QBitConfig
 
 
 
+   /*******************************************************************************
+    ** Whether the access logger registers a JVM shutdown hook that drains
+    ** accepted rows (default true). Hosts that call SecuredDataRuntime.close()
+    ** from their own shutdown sequence turn it off.
+    *******************************************************************************/
+   public boolean getRegisterShutdownHook()
+   {
+      return (registerShutdownHook);
+   }
+
+
+
+   /*******************************************************************************
+    ** Configure registerShutdownHook.
+    *******************************************************************************/
+   public SecuredDataQBitConfig withRegisterShutdownHook(boolean registerShutdownHook)
+   {
+      this.registerShutdownHook = registerShutdownHook;
+      return (this);
+   }
+
+
+
    /*******************************************************************************
     ** Configure accessLogFlushIntervalMillis.
     *******************************************************************************/

src/main/java/com/kingsrook/qbits/secureddata/uiapi/SecuredDataExtension.java

@@ -37,11 +37,11 @@ public final class SecuredDataExtension
    }
 
    /*******************************************************************************
-    ** The secured-data UI API, version 1.0.0.
+    ** The secured-data UI API, version 1.1.0.
     *******************************************************************************/
    public static MiddlewareSpecExtension defineExtension()
    {
-      return (new MiddlewareSpecExtension().withName("secured-data").withSemanticVersion("1.0.0")
+      return (new MiddlewareSpecExtension().withName("secured-data").withSemanticVersion("1.1.0")
          .withDescription("Reveal grants and break-glass status")
          .withSpec(SecuredDataRevealSpecV1.class).withSpec(SecuredDataRevealOptionsSpecV1.class).withSpec(SecuredDataStatusSpecV1.class));
    }

src/main/java/com/kingsrook/qbits/secureddata/uiapi/SecuredDataRevealOptionsSpecV1.java

@@ -41,7 +41,7 @@ public class SecuredDataRevealOptionsSpecV1 extends AbstractJsonEndpointSpec
    public BasicOperation defineBasicOperation()
    {
       return (new BasicOperation().withPath("/qbit/secured-data/reveal/options").withHttpMethod(HttpMethod.GET)
-         .withTag(() -> "Secured data").withShortSummary("RevealOptions"));
+         .withTag(() -> "secured-data").withShortSummary("RevealOptions"));
    }
 
    /*******************************************************************************

src/main/java/com/kingsrook/qbits/secureddata/uiapi/SecuredDataRevealSpecV1.java

@@ -28,6 +28,7 @@ import com.kingsrook.qbits.secureddata.reveal.RevealService;
 import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException;
 import com.kingsrook.qqq.middleware.javalin.specs.BasicOperation;
 import com.kingsrook.qqq.middleware.javalin.specs.extensions.AbstractJsonEndpointSpec;
+import com.kingsrook.qqq.middleware.javalin.specs.v1.responses.BasicErrorResponseV1;
 import com.kingsrook.qqq.openapi.model.HttpMethod;
 import com.kingsrook.qqq.openapi.model.Parameter;
 import com.kingsrook.qqq.openapi.model.Schema;
@@ -44,7 +45,7 @@ public class SecuredDataRevealSpecV1 extends AbstractJsonEndpointSpec
    public BasicOperation defineBasicOperation()
    {
       return (new BasicOperation().withPath("/qbit/secured-data/reveal").withHttpMethod(HttpMethod.POST)
-         .withTag(() -> "Secured data").withShortSummary("Reveal"));
+         .withTag(() -> "secured-data").withShortSummary("Reveal"));
    }
 
    /*******************************************************************************
@@ -81,10 +82,10 @@ public class SecuredDataRevealSpecV1 extends AbstractJsonEndpointSpec
    @Override
    public List<com.kingsrook.qqq.middleware.javalin.specs.BasicResponse> defineAdditionalBasicResponses()
    {
-      return (List.of(new com.kingsrook.qqq.middleware.javalin.specs.BasicResponse(io.javalin.http.HttpStatus.BAD_REQUEST, "Invalid reveal request", "SecuredDataErrorV1"),
-         new com.kingsrook.qqq.middleware.javalin.specs.BasicResponse(io.javalin.http.HttpStatus.FORBIDDEN, "Reveal denied", "SecuredDataErrorV1"),
-         new com.kingsrook.qqq.middleware.javalin.specs.BasicResponse(io.javalin.http.HttpStatus.NOT_FOUND, "Record not found or locked", "SecuredDataErrorV1"),
-         new com.kingsrook.qqq.middleware.javalin.specs.BasicResponse(io.javalin.http.HttpStatus.TOO_MANY_REQUESTS, "Daily reveal cap reached", "SecuredDataErrorV1")));
+      return (List.of(new com.kingsrook.qqq.middleware.javalin.specs.BasicResponse(io.javalin.http.HttpStatus.BAD_REQUEST, "Invalid reveal request", BasicErrorResponseV1.class.getSimpleName()),
+         new com.kingsrook.qqq.middleware.javalin.specs.BasicResponse(io.javalin.http.HttpStatus.FORBIDDEN, "Reveal denied", BasicErrorResponseV1.class.getSimpleName()),
+         new com.kingsrook.qqq.middleware.javalin.specs.BasicResponse(io.javalin.http.HttpStatus.NOT_FOUND, "Record not found or locked", BasicErrorResponseV1.class.getSimpleName()),
+         new com.kingsrook.qqq.middleware.javalin.specs.BasicResponse(io.javalin.http.HttpStatus.TOO_MANY_REQUESTS, "Daily reveal cap reached", BasicErrorResponseV1.class.getSimpleName())));
    }
 
    /*******************************************************************************

src/main/java/com/kingsrook/qbits/secureddata/uiapi/SecuredDataSchemas.java

@@ -58,7 +58,8 @@ public final class SecuredDataSchemas
       schemas.put("SecuredDataRevealAllowanceV1", object("tableName", type(Type.STRING), "fieldName", type(Type.STRING), "remainingDailyAllowance", type(Type.INTEGER).withNullable(true)));
       schemas.put("SecuredDataStatusResponseV1", object("breakGlass", new Schema().withRefToSchema("SecuredDataBreakGlassV1").withNullable(true),
          "revealsUsedToday", type(Type.INTEGER), "revealAllowances", type(Type.ARRAY).withItems(new Schema().withRefToSchema("SecuredDataRevealAllowanceV1"))));
-      schemas.put("SecuredDataErrorV1", object("error", type(Type.STRING)));
+      schemas.put("SecuredDataErrorV1", object("error", type(Type.STRING))
+         .withDescription("Deprecated since 1.1.0: errors use BasicErrorResponseV1."));
       return (schemas);
    }
 

src/main/java/com/kingsrook/qbits/secureddata/uiapi/SecuredDataStatusSpecV1.java

@@ -41,7 +41,7 @@ public class SecuredDataStatusSpecV1 extends AbstractJsonEndpointSpec
    public BasicOperation defineBasicOperation()
    {
       return (new BasicOperation().withPath("/qbit/secured-data/status").withHttpMethod(HttpMethod.GET)
-         .withTag(() -> "Secured data").withShortSummary("Status"));
+         .withTag(() -> "secured-data").withShortSummary("Status"));
    }
 
    /*******************************************************************************

src/main/resources/openapi/qqq-v1/secured-data.yaml

@@ -1,5 +1,44 @@
 components:
   schemas:
+    BasicErrorResponseV1:
+      description: "The error body of every UI API operation (core and extensions)."
+      properties:
+        error:
+          description: "The error message (the same as message; kept for clients of\
+            \ UI API 1.0 to 1.5)."
+          type: "string"
+        code:
+          description: "A stable, machine-readable code. Well-known values: BAD_REQUEST\
+            \ (400), UNAUTHENTICATED (401),\nPERMISSION_DENIED (403), NOT_FOUND (404),\
+            \ CONFLICT (409), RATE_LIMITED (429), INTERNAL_ERROR (500),\nERROR (other).\
+            \ Extensions may add codes; treat an unknown code by the HTTP status.\
+            \ Since 1.6.0."
+          type: "string"
+        message:
+          description: "A message to show the user. Since 1.6.0."
+          type: "string"
+        fieldErrors:
+          description: "Errors about particular request fields, when known. Since\
+            \ 1.6.0."
+          items:
+            properties:
+              field:
+                description: "The field (or parameter) name."
+                type: "string"
+              message:
+                description: "The error about that field."
+                type: "string"
+              code:
+                description: "An optional machine-readable code, e.g. REQUIRED, TOO_LONG,\
+                  \ PATTERN."
+                type: "string"
+            type: "object"
+          type: "array"
+        traceId:
+          description: "Identifies this request in the server logs (also the X-QQQ-Trace-Id\
+            \ response header); quote it when reporting a problem. Since 1.6.0."
+          type: "string"
+      type: "object"
     SecuredDataBreakGlassV1:
       properties:
         id:
@@ -16,6 +55,7 @@ components:
           type: "string"
       type: "object"
     SecuredDataErrorV1:
+      description: "Deprecated since 1.1.0: errors use BasicErrorResponseV1."
       properties:
         error:
           type: "string"
@@ -96,19 +136,53 @@ components:
       type: "object"
   securitySchemes:
     sessionUuidCookie:
+      description: "The session cookie set by POST /qqq/v1/manageSession (web clients)."
       in: "cookie"
       name: "sessionUUID"
       type: "apiKey"
+    bearerAuth:
+      bearerFormat: "JWT or session UUID"
+      description: "`Authorization: Bearer <token>`, where the token is either\n-\
+        \ an OAuth2 access token (a signed JWT) issued by the app's identity provider\
+        \ to the app's client,\n  or to its native client (`values.nativeClientId`\
+        \ in GET /qqq/v1/metaData/authentication), for\n  example from the device-authorization\
+        \ grant or auth-code + PKCE. It is verified against the\n  provider's keys\
+        \ (issuer, audience, expiry, signature) and gives the same user and permissions\n\
+        \  as signing in on the web. When it expires (401), refresh it at the provider\
+        \ and retry; or\n- a session UUID returned by POST /qqq/v1/manageSession,\
+        \ which the server keeps refreshed."
+      scheme: "bearer"
+      type: "http"
+    oauth2:
+      description: "The app's OAuth2 / OpenID Connect provider. The real endpoints\
+        \ come from the provider's discovery\ndocument: `{values.baseUrl}.well-known/openid-configuration`,\
+        \ with values from\nGET /qqq/v1/metaData/authentication. Native clients use\
+        \ `values.nativeClientId` (a public client)\nwith auth-code + PKCE (loopback\
+        \ or custom-scheme redirect URI registered at the provider) or the\ndevice-authorization\
+        \ grant (`device_authorization_endpoint`, RFC 8628), then send the access\
+        \ token\nas `Authorization: Bearer`."
+      flows:
+        authorizationCode:
+          authorizationUrl: "/authorize"
+          scopes:
+            email: "Email address"
+            offline_access: "A refresh token, to renew the access token without signing\
+              \ in again"
+            openid: "OpenID Connect sign-in"
+            profile: "Name, username and groups (permissions come from the groups)"
+          tokenUrl: "/token"
+      type: "oauth2"
 info:
   contact:
     email: "contact@kingsrook.com"
   description: "Reveal grants and break-glass status"
   title: "QQQ Middleware API extension: secured-data"
-  version: "1.0.0"
+  version: "1.1.0"
 openapi: "3.0.3"
 paths:
   /qqq/v1/qbit/secured-data/reveal:
     post:
+      operationId: "postQbitSecuredDataReveal"
       requestBody:
         content:
           application/json:
@@ -120,19 +194,19 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/SecuredDataErrorV1"
+                $ref: "#/components/schemas/BasicErrorResponseV1"
           description: "Invalid reveal request"
         403:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/SecuredDataErrorV1"
+                $ref: "#/components/schemas/BasicErrorResponseV1"
           description: "Reveal denied"
         404:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/SecuredDataErrorV1"
+                $ref: "#/components/schemas/BasicErrorResponseV1"
           description: "Record not found or locked"
         200:
           content:
@@ -144,16 +218,37 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/SecuredDataErrorV1"
+                $ref: "#/components/schemas/BasicErrorResponseV1"
           description: "Daily reveal cap reached"
+        401:
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicErrorResponseV1"
+          description: "Not authenticated, or the session or token is invalid or expired\
+            \ (code UNAUTHENTICATED). Sign in again, or refresh the token."
+        409:
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicErrorResponseV1"
+          description: "Conflict with the current state, e.g. an edit conflict or\
+            \ an invalid transition (code CONFLICT)."
       security:
       - sessionUuidCookie:
         - "N/A"
+      - bearerAuth:
+        - "N/A"
+      - oauth2:
+        - "openid"
+        - "profile"
+        - "email"
       summary: "Reveal"
       tags:
-      - "Secured data"
+      - "secured-data"
   /qqq/v1/qbit/secured-data/reveal/options:
     get:
+      operationId: "getQbitSecuredDataRevealOptions"
       parameters:
       - in: "query"
         name: "tableName"
@@ -172,14 +267,61 @@ paths:
               schema:
                 $ref: "#/components/schemas/SecuredDataRevealOptionsResponseV1"
           description: "Successful Response"
+        400:
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicErrorResponseV1"
+          description: "Bad request: a parameter or the body is invalid (code BAD_REQUEST);\
+            \ fieldErrors may name the fields."
+        401:
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicErrorResponseV1"
+          description: "Not authenticated, or the session or token is invalid or expired\
+            \ (code UNAUTHENTICATED). Sign in again, or refresh the token."
+        403:
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicErrorResponseV1"
+          description: "The user (or the credential's scope) may not do this (code\
+            \ PERMISSION_DENIED)."
+        404:
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicErrorResponseV1"
+          description: "Not found, or not visible to this user (code NOT_FOUND)."
+        409:
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicErrorResponseV1"
+          description: "Conflict with the current state, e.g. an edit conflict or\
+            \ an invalid transition (code CONFLICT)."
+        429:
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicErrorResponseV1"
+          description: "Too many requests (code RATE_LIMITED); retry later."
       security:
       - sessionUuidCookie:
         - "N/A"
+      - bearerAuth:
+        - "N/A"
+      - oauth2:
+        - "openid"
+        - "profile"
+        - "email"
       summary: "RevealOptions"
       tags:
-      - "Secured data"
+      - "secured-data"
   /qqq/v1/qbit/secured-data/status:
     get:
+      operationId: "getQbitSecuredDataStatus"
       responses:
         200:
           content:
@@ -187,9 +329,61 @@ paths:
               schema:
                 $ref: "#/components/schemas/SecuredDataStatusResponseV1"
           description: "Successful Response"
+        400:
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicErrorResponseV1"
+          description: "Bad request: a parameter or the body is invalid (code BAD_REQUEST);\
+            \ fieldErrors may name the fields."
+        401:
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicErrorResponseV1"
+          description: "Not authenticated, or the session or token is invalid or expired\
+            \ (code UNAUTHENTICATED). Sign in again, or refresh the token."
+        403:
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicErrorResponseV1"
+          description: "The user (or the credential's scope) may not do this (code\
+            \ PERMISSION_DENIED)."
+        404:
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicErrorResponseV1"
+          description: "Not found, or not visible to this user (code NOT_FOUND)."
+        409:
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicErrorResponseV1"
+          description: "Conflict with the current state, e.g. an edit conflict or\
+            \ an invalid transition (code CONFLICT)."
+        429:
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicErrorResponseV1"
+          description: "Too many requests (code RATE_LIMITED); retry later."
       security:
       - sessionUuidCookie:
         - "N/A"
+      - bearerAuth:
+        - "N/A"
+      - oauth2:
+        - "openid"
+        - "profile"
+        - "email"
       summary: "Status"
       tags:
-      - "Secured data"
+      - "secured-data"
+servers:
+- description: "The QQQ server that serves this document (paths include /qqq/v1)."
+  url: "/"
+tags:
+- description: "Operations of the secured-data extension."
+  name: "secured-data"

src/test/java/com/kingsrook/qbits/secureddata/RetentionProofTest.java

@@ -84,13 +84,23 @@ class RetentionProofTest
 
 
 
+   private static final String CANARY_PREFIX     = "CANARY-PHI-";
+   private static final int    CANARY_RANDOM     = 24;
+   private static final int    MIN_LEAK_FRAGMENT = 12;
+
+
+
    /*******************************************************************************
-    ** Letters g-v only, so a marker can never coincide with hex digests or tokens.
+    ** A fixed prefix and 24 random letters g-v (so it never coincides with hex
+    ** digests). Leaks are matched on the whole marker or on 12-letter windows of
+    ** its random part: 16^12 possibilities, so ordinary text, JSON words or
+    ** base64 tokens cannot match by chance, while a truncated or re-encoded
+    ** fragment of a real leak still does.
     *******************************************************************************/
    private static String canary()
    {
-      StringBuilder marker = new StringBuilder("CANARY-PHI-");
-      for(int i = 0; i < 16; i++)
+      StringBuilder marker = new StringBuilder(CANARY_PREFIX);
+      for(int i = 0; i < CANARY_RANDOM; i++)
       {
          marker.append((char) ('g' + RANDOM.nextInt(16)));
       }
@@ -100,21 +110,25 @@ class RetentionProofTest
 
 
    /*******************************************************************************
-    ** The marker, or any substring of it longer than four characters.
+    ** The whole marker, or any MIN_LEAK_FRAGMENT-letter window of its random part.
     *******************************************************************************/
    private static List<String> leaks(String haystack, String... markers)
    {
       List<String> found = new ArrayList<>();
       for(String marker : markers)
       {
-         for(int length = 5; length <= marker.length(); length++)
+         if(haystack.contains(marker))
          {
-            for(int start = 0; start + length <= marker.length(); start++)
+            found.add("whole marker");
+            continue;
+         }
+
+         String random = marker.substring(CANARY_PREFIX.length());
+         for(int start = 0; start + MIN_LEAK_FRAGMENT <= random.length(); start++)
+         {
+            if(haystack.contains(random.substring(start, start + MIN_LEAK_FRAGMENT)))
             {
-               if(haystack.contains(marker.substring(start, start + length)))
-               {
-                  found.add("length " + length + " at " + start);
-               }
+               found.add("fragment at " + start);
             }
          }
       }
@@ -123,6 +137,23 @@ class RetentionProofTest
 
 
 
+   /*******************************************************************************
+    ** The leak scan: whole markers and 12-letter fragments are found; the shared
+    ** prefix and ordinary words made of the same letters are not.
+    *******************************************************************************/
+   @Test
+   void leakScanHasNoFalsePositives()
+   {
+      String marker = canary();
+      String random = marker.substring(CANARY_PREFIX.length());
+      assertThat(leaks("x" + marker + "x", marker)).containsExactly("whole marker");
+      assertThat(leaks("{\"v\":\"" + random.substring(5, 5 + MIN_LEAK_FRAGMENT) + "\"}", marker)).isNotEmpty();
+      assertThat(leaks(CANARY_PREFIX + " options joins strings rows points", marker)).isEmpty();
+      assertThat(leaks(random.substring(0, MIN_LEAK_FRAGMENT - 1), marker)).isEmpty();
+   }
+
+
+
    /*******************************************************************************
     ** Canary through CRUD, a failing validation, a wipe run, a masked export and
     ** the access log: it may appear only in the host table itself.

src/test/java/com/kingsrook/qbits/secureddata/SecuredDataAccessLoggerTest.java

@@ -22,6 +22,7 @@ package com.kingsrook.qbits.secureddata;
 
 
 import java.io.Serializable;
+import java.lang.ref.WeakReference;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
@@ -46,6 +47,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Test;
@@ -359,4 +361,92 @@ class SecuredDataAccessLoggerTest
       {
       }
    }
+
+
+   /*******************************************************************************
+    ** The worker starts while the host is still building its instance. An idle
+    ** tick opens no context, so it never validates (or enriches) an unfinished
+    ** instance on the worker thread, and a failing tick is logged instead of
+    ** cancelling every later tick.
+    *******************************************************************************/
+   @Test
+   void idleTicksNeverValidateAnUnfinishedInstance() throws Exception
+   {
+      setup();
+      SecuredDataRuntime runtime = SecuredDataRuntime.current();
+      QInstance unfinished = SecuredDataTestSupport.base();
+      unfinished.addTable(new QTableMetaData().withName("unfinished").withBackendName("not_registered_yet"));
+      SecuredDataAccessLogger idle = new SecuredDataAccessLogger(unfinished, runtime);
+      try
+      {
+         idle.flush();
+         idle.scheduledFlush();
+         assertThat(unfinished.getHasBeenValidated()).isFalse();
+      }
+      finally
+      {
+         idle.close();
+      }
+   }
+
+
+   /*******************************************************************************
+    ** Neither the worker thread nor the JVM shutdown hook keeps a discarded
+    ** instance alive: a host or test that drops its instance without closing
+    ** the runtime lets it be collected.
+    *******************************************************************************/
+   @Test
+   void aDiscardedInstanceCanBeCollected() throws Exception
+   {
+      close();
+      WeakReference<QInstance> discarded = discardedInstance();
+      long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
+      List<byte[]> pressure = new ArrayList<>();
+      while(discarded.get() != null && System.nanoTime() < deadline)
+      {
+         pressure.add(new byte[1 << 20]);
+         if(pressure.size() > 64)
+         {
+            pressure.clear();
+         }
+         System.gc();
+         Thread.sleep(20);
+      }
+      assertThat(discarded.get()).as("instance still reachable from the access logger").isNull();
+   }
+
+
+
+   /*******************************************************************************
+    ** Build, use and drop an instance whose worker ticks every 10 ms.
+    *******************************************************************************/
+   private static WeakReference<QInstance> discardedInstance() throws Exception
+   {
+      QInstance instance = SecuredDataTestSupport.base();
+      instance.addBackend(new com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData().withName("memory")
+         .withBackendType(com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule.class));
+      instance.addTable(person("memory"));
+      new SecuredDataQBitProducer().withQBitConfig(new SecuredDataQBitConfig().withAccessLogFlushIntervalMillis(10)).produce(instance);
+      Thread.sleep(50);
+      QContext.clear();
+      return (new WeakReference<>(instance));
+   }
+
+
+
+   /*******************************************************************************
+    ** Hosts that close the runtime themselves can skip the JVM hook; close
+    ** still drains and stops the worker.
+    *******************************************************************************/
+   @Test
+   void shutdownHookIsOptional() throws Exception
+   {
+      setup(config -> config.withRegisterShutdownHook(false));
+      var logger = SecuredDataRuntime.current().logger();
+      assertThat(logger.isWorkerRunning()).isTrue();
+      logger.onAccess(event(FieldAccessEventType.VIEW, null, null));
+      logger.close();
+      assertThat(logger.isWorkerRunning()).isFalse();
+      assertThat(logs()).hasSize(1);
+   }
 }

src/test/java/com/kingsrook/qbits/secureddata/SecuredDataHttpTest.java

@@ -109,11 +109,25 @@ class SecuredDataHttpTest
       var status = request("/qbit/secured-data/status", null, false);
       assertThat(status.statusCode()).as(status.body()).isEqualTo(200);
       assertThat(new JSONObject(status.body()).getInt("revealsUsedToday")).isEqualTo(1);
-      assertThat(request("/qbit/secured-data/reveal", body, false).statusCode()).isEqualTo(429);
-      assertThat(request("/qbit/secured-data/reveal", body, true).statusCode()).isEqualTo(403);
-      assertThat(request("/qbit/secured-data/reveal", "{}", false).statusCode()).isEqualTo(400);
-      assertThat(request("/qbit/secured-data/reveal", "{\"primaryKey\":1}", false).statusCode()).isEqualTo(400);
-      assertThat(request("/qbit/secured-data/reveal/options?tableName=person&fieldName=missing", null, false).statusCode()).isEqualTo(404);
+      assertErrorResponse(request("/qbit/secured-data/reveal", body, false), 429);
+      assertErrorResponse(request("/qbit/secured-data/reveal", body, true), 403);
+      assertErrorResponse(request("/qbit/secured-data/reveal", "{}", false), 400);
+      assertErrorResponse(request("/qbit/secured-data/reveal", "{\"primaryKey\":1}", false), 400);
+      assertErrorResponse(request("/qbit/secured-data/reveal/options?tableName=person&fieldName=missing", null, false), 404);
+   }
+
+   /*******************************************************************************
+    ** Verify standard error bodies without changing endpoint status semantics.
+    *******************************************************************************/
+   private void assertErrorResponse(HttpResponse<String> response, int status)
+   {
+      assertThat(response.statusCode()).as(response.body()).isEqualTo(status);
+      JSONObject body = new JSONObject(response.body());
+      assertThat(body.keySet()).contains("error", "code", "message", "traceId");
+      assertThat(body.getString("error")).isNotBlank();
+      assertThat(body.getString("code")).isNotBlank();
+      assertThat(body.getString("message")).isNotBlank();
+      assertThat(body.getString("traceId")).isNotBlank();
    }
 
    @Test

src/test/java/com/kingsrook/qbits/secureddata/SecuredDataResolverTest.java

@@ -33,6 +33,7 @@ import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QVirtualFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
@@ -326,4 +327,22 @@ class SecuredDataResolverTest
       CLOCK.now = NOW.plusSeconds(10);
       assertThat(resolver.isRevealActive("person", "email", 7)).isFalse();
    }
+
+
+   /*******************************************************************************
+    ** A virtual (computed) field is not in getFields, and getField throws for
+    ** it: it resolves through the table classification instead of failing the
+    ** whole read.
+    *******************************************************************************/
+   @Test
+   void virtualFieldsTakeTheTableClassification() throws Exception
+   {
+      QVirtualFieldMetaData display = new QVirtualFieldMetaData("displayName", QFieldType.STRING);
+      person.withVirtualFields(List.of(display));
+      assertThat(SecuredDataClassifications.effective(person, "displayName")).isNull();
+      assertThat(resolver.resolve(person, display).tier()).isEqualTo(FieldAccessTier.FULL);
+      person.withSupplementalMetaData(new AiTableMetaData().withClassification(DataClassification.PII));
+      assertThat(SecuredDataClassifications.effective(person, "displayName")).isEqualTo(DataClassification.PII);
+      assertThat(resolver.resolve(person, display).tier()).isEqualTo(FieldAccessTier.MASKED);
+   }
 }