ci: centralize QBit source scanning

Co-Authored-By: Codex <noreply@openai.com>

.circleci/config.yml

@@ -1,60 +1,14 @@
 version: 2.1
-
+setup: true
 orbs:
-   qqq-orb: kingsrook/qqq-orb@0.6.9
-
-commands:
-   configure-package-install:
-      steps:
-         - run:
-              name: Keep package installation unattended
-              command: |
-                 # These build machines are discarded after the job.
-                 sudo install -d /etc/needrestart/conf.d
-                 printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
-
+  qqq-orb: kingsrook/qqq-orb@0.7.12
 workflows:
-   test_only:
-      jobs:
-         - qqq-orb/mvn_test_only:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              filters:
-                 branches:
-                    ignore: /(develop|release\/.*)/
-
-   publish_snapshot:
-      jobs:
-         - qqq-orb/mvn_publish:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              branch_type: snapshot
-              filters:
-                 branches:
-                    only: [develop]
-
-   publish_release_candidate:
-      jobs:
-         - qqq-orb/mvn_publish:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              branch_type: release_candidate
-              filters:
-                 branches:
-                    only: [/release\/.*/]
-
-   publish_release:
-      jobs:
-         - qqq-orb/mvn_publish:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              branch_type: release
-              filters:
-                 branches:
-                    ignore: /.*/
-                 tags:
-                    only: [/^v\d+\.\d+\.\d+$/]
+  setup:
+    jobs:
+    - qqq-orb/generate_pipeline:
+        orb_version: 0.7.12
+        filters:
+          branches:
+            ignore: automation/qbit-rc-pending
+          tags:
+            only: /^v\d+\.\d+\.\d+$/

.github/workflows/codeql.yml

@@ -0,0 +1,23 @@
+name: CodeQL
+'on':
+  push:
+    branches:
+    - main
+    - develop
+    - release/**
+    - hotfix/**
+  pull_request: {}
+  schedule:
+  - cron: 41 16 * * 0
+  workflow_dispatch: {}
+permissions:
+  contents: read
+  actions: read
+  security-events: write
+  packages: read
+concurrency:
+  group: codeql-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+jobs:
+  analyze:
+    uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22

.github/workflows/release-rc.yml

@@ -1,25 +1,74 @@
-# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
-# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
-# the release line and the release/* push runs CircleCI publish_release_candidate.
 name: Release candidate
-
-on:
+'on':
   workflow_dispatch:
     inputs:
       dry_run:
-        description: Print the plan without merging or creating branches
+        description: Plan a cut without changing release state
         type: boolean
         default: false
+      finalize:
+        description: Retry finalization of an existing cut; do not cut another RC
+        type: boolean
+        default: false
+      recover:
+        description: Recover the same published RC after a deployment timeout; never
+          reupload
+        type: boolean
+        default: false
+      circle_build:
+        description: Original Circle publisher build number (recovery only)
+        type: string
+        default: ''
+      merge_sha:
+        description: Existing cut merge_sha (finalize retry only)
+        type: string
+        default: ''
+      bump_sha:
+        description: Existing cut bump_sha (finalize retry only)
+        type: string
+        default: ''
+      branch:
+        description: Existing cut branch (finalize retry only)
+        type: string
+        default: ''
+      version:
+        description: Existing cut version (finalize retry only)
+        type: string
+        default: ''
   pull_request:
-    types: [closed]
-    branches: [develop]
-
+    types:
+    - closed
+    branches:
+    - develop
+  repository_dispatch:
+    types:
+    - qbit-rc-published
 permissions:
   contents: write
-
+  actions: read
+  checks: read
 jobs:
   release-rc:
-    if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
-    uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
+    if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
+      || (github.event_name == 'pull_request' && github.event.pull_request.merged
+      == true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
+      'release:rc'))
+    uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
     with:
-      dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
+      dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
+        }}
+      finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
+        == 'workflow_dispatch' && inputs.finalize) }}
+      recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
+        }}
+      circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
+        || '' }}
+      merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
+        || inputs.merge_sha || '' }}
+      bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
+        || inputs.bump_sha || '' }}
+      branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
+        || inputs.branch || '' }}
+      version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
+        || inputs.version || '' }}
+    secrets: inherit

.qqq-ci.yml

@@ -0,0 +1,16 @@
+profile: qbit
+orb_version: 0.7.12
+default_branch: develop
+java_version: '21'
+browsers: true
+api_checks: false
+docs_only: false
+release_on_main: false
+release_tag_policy: stable
+test_branch_policy: non-release
+publish:
+  snapshot: true
+  release_candidate: true
+  release: true
+  asciidoc: false
+release_completion: true

CHANGELOG.md

@@ -27,9 +27,13 @@ All notable changes to this project are documented here. The format follows Keep
 - Conformance suite against MinIO and Garage (Testcontainers), service, index and action tests, and the index and actions on PostgreSQL.
 
 ### Removed
+- The QRun-IO/qqq#1165 stand-ins: `SessionCredentialsClient` (session tokens now go through core `S3BackendMetaData.sessionToken` / `S3ClientFactory`, including for the presigner) and the file reading in `SecretRefs` (core `QMetaDataVariableInterpreter` resolves `${file.PATH}`). **Behavior change:** core reads `${file.PATH}` only inside its allowed roots, which are `/var/run/secrets` and `/run/secrets` by default. A host that mounts storage secrets elsewhere must set `qqq.fileReference.roots` or `QQQ_FILE_REFERENCE_ROOTS`. The qbit's default allow-list (`file./var/run/secrets/storage/*`) is already inside the default roots.
 - None for the health migration; both Test connection record actions, including on-demand write tests, are retained.
 
 ### Changed
+- Multi-node deployments: the runtime caches are registered with QQQ `ClusterCaches`. Shared (cleared on every node): the bucket rows' presign limits (`qbit-object-storage.bucket-row-limits`, on a bucket row insert or update), the enabled header rules (`qbit-object-storage.header-rules`, on a header rule insert, update or delete), both through the new `ConfigRowChangeCustomizer`, and the capability statuses (`qbit-object-storage.capabilities`: a probe result recorded on one node clears that capability on the others, which reload it from the capability row; before, other nodes kept the old status until restart). The 15 s TTLs remain the safety net. Node-local, with reasons: the alias and connection row-id maps, the S3 client map, the process-permission map, the provider defaults, the running-scan guard and the per-pod presign rate limiter (`StorageBucket.PRESIGN_COUNTS`; per-node rate limits are accepted for now). `ObjectStorageClusterCachesTest` audits the package for unregistered static caches and covers remote and local invalidation.
+- Builds against QQQ `4.1.0-SNAPSHOT` (`qqq.snapshot.version`) for `ClusterCaches` until the next QQQ RC ships; a release must pin that RC.
+- Read the qbit metadata version from the Maven-filtered artifact version so release metadata matches the build and help bundle.
 - Rechecked the released RC.8 JARs: core still has no S3 session-token/provider hook or file-reference interpreter. Retained both compatibility shims and moved their review deadline to RC.9 under issue #10.
 - Raised the QQQ BOM baseline to released `4.1.0-RC.8` for the central health API.
 - Raised the QQQ BOM baseline to released `4.1.0-RC.7` for the core help-bundle registry and coverage assertions.

docs/S3-FACTORY-MIGRATION.md

@@ -26,16 +26,21 @@ The public qbit configuration API is unchanged.
 
 ## Remaining local code
 
-- Core RC.5 has no session-token field or credentials-provider customization
-  in `S3BackendMetaData` / `S3ClientFactory.buildClient`. The narrow
-  `SessionCredentialsClient` bridge adds session credentials to requests while
-  retaining their overrides; core still creates the client and HTTP transport.
-  Core needs session-token metadata (including enrichment and cache-key support)
-  and `AwsSessionCredentials` construction, or a caller-provided credentials hook.
-  Marker: `STAND-IN(core=QRun-IO/qqq#1165, removeBy=4.1.0-RC.7)`.
-- Core has no presigner factory. The local presigner retains its separate public
-  endpoint and least-privilege keys, along with existing session-token handling.
-- `SecretRefs` file resolution is unchanged and carries the same RC.7 marker.
+- Core has no presigner factory. The local presigner keeps its separate public
+  endpoint and least-privilege keys; its credentials (including the session
+  token) come from core `S3ClientFactory.buildCredentialsProvider`.
+- `SecretRefs` keeps only the qbit's allow-list; resolution, including
+  `${file.PATH}`, is core `QMetaDataVariableInterpreter`.
+
+## Stand-ins removed (QRun-IO/qqq#1165)
+
+- `SessionCredentialsClient` is gone: core `S3BackendMetaData.sessionToken`
+  makes `S3ClientFactory` sign with `AwsSessionCredentials`.
+- `SecretRefs` no longer reads files itself. Core reads `${file.PATH}`
+  read-only, and only inside its allowed roots: `/var/run/secrets` and
+  `/run/secrets` by default, or `qqq.fileReference.roots` /
+  `QQQ_FILE_REFERENCE_ROOTS` (comma-separated). The qbit's default allow-list
+  (`file./var/run/secrets/storage/*`) is inside the default roots.
 
 ## Resolved dependency diff
 

pom.xml

@@ -179,6 +179,7 @@
             <filtering>true</filtering>
             <includes>
                <include>help/*.yaml</include>
+               <include>qbit-object-storage.properties</include>
             </includes>
          </resource>
          <resource>
@@ -186,10 +187,20 @@
             <filtering>false</filtering>
             <excludes>
                <exclude>help/*.yaml</exclude>
+               <exclude>qbit-object-storage.properties</exclude>
             </excludes>
          </resource>
       </resources>
       <plugins>
+         <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <configuration>
+               <systemPropertyVariables combine.children="append">
+                  <qbit.artifactVersion>${project.version}</qbit.artifactVersion>
+               </systemPropertyVariables>
+            </configuration>
+         </plugin>
          <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-checkstyle-plugin</artifactId>
@@ -272,14 +283,14 @@
    <profiles>
       <!-- Built against qqq 4.1 (QBitMetaDataProducer). Until qbit-build-parent  -->
       <!-- imports the released 4.1 BOM, this profile is active by default and     -->
-      <!-- imports qqq-bom-pom 4.1.0-RC.7 ahead of the parent's 4.0.0 BOM.         -->
+      <!-- imports qqq-bom-pom 4.1.0-RC.15 ahead of the parent's 4.0.0 BOM.        -->
       <profile>
          <id>qqq-snapshot</id>
          <activation>
             <activeByDefault>true</activeByDefault>
          </activation>
          <properties>
-            <qqq.snapshot.version>4.1.0-RC.8</qqq.snapshot.version>
+            <qqq.snapshot.version>4.1.0-RC.15</qqq.snapshot.version>
          </properties>
          <dependencyManagement>
             <dependencies>

src/main/java/com/kingsrook/qbits/objectstorage/ObjectStorageQBitConfig.java

@@ -62,6 +62,7 @@ public class ObjectStorageQBitConfig implements QBitConfig
    private String appName = "objectStorageAdmin";
    private String appLabel = "Storage";
    private String parentAppName;
+   private String tablePathPrefix;
    private List<String> allowedSecretRefPatterns = new ArrayList<>(SecretRefs.DEFAULT_ALLOWED_PATTERNS);
    private Integer bulkCap = 10_000;
    private Boolean bucketScoping = true;
@@ -425,6 +426,39 @@ public class ObjectStorageQBitConfig implements QBitConfig
 
 
 
+   /********************************************************************************
+    ** Getter for tablePathPrefix: the front of the links the Storage home makes to
+    ** tables and records.  Defaults to /app, the route every QQQ frontend serves
+    ** tables at (/app/<tableName>).
+    *******************************************************************************/
+   public String getTablePathPrefix()
+   {
+      return (this.tablePathPrefix == null ? "/app" : this.tablePathPrefix);
+   }
+
+
+
+   /********************************************************************************
+    ** Setter for tablePathPrefix.
+    *******************************************************************************/
+   public void setTablePathPrefix(String tablePathPrefix)
+   {
+      this.tablePathPrefix = tablePathPrefix;
+   }
+
+
+
+   /********************************************************************************
+    ** Fluent setter for tablePathPrefix.
+    *******************************************************************************/
+   public ObjectStorageQBitConfig withTablePathPrefix(String tablePathPrefix)
+   {
+      this.tablePathPrefix = tablePathPrefix;
+      return (this);
+   }
+
+
+
    /********************************************************************************
     ** Getter for allowedSecretRefPatterns.
     *******************************************************************************/

src/main/java/com/kingsrook/qbits/objectstorage/ObjectStorageQBitProducer.java

@@ -21,8 +21,11 @@
 package com.kingsrook.qbits.objectstorage;
 
 
+import java.io.IOException;
+import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Properties;
 import com.kingsrook.qbits.objectstorage.app.StorageWidgets;
 import com.kingsrook.qbits.objectstorage.health.ObjectStorageConnectionChecks;
 import com.kingsrook.qbits.objectstorage.internal.ConfigSync;
@@ -60,13 +63,41 @@ public class ObjectStorageQBitProducer implements QBitMetaDataProducer<ObjectSto
 {
    public static final String GROUP_ID    = "com.kingsrook.qbits";
    public static final String ARTIFACT_ID = "qbit-object-storage";
-   public static final String VERSION     = "0.1.0";
+   public static final String VERSION     = loadVersion();
 
    private ObjectStorageQBitConfig qBitConfig;
    private String                  namespace;
 
 
 
+   /*******************************************************************************
+    ** Read the artifact version from the Maven-filtered resource.
+    *******************************************************************************/
+   private static String loadVersion()
+   {
+      try(InputStream inputStream = ObjectStorageQBitProducer.class.getClassLoader().getResourceAsStream("qbit-object-storage.properties"))
+      {
+         if(inputStream == null)
+         {
+            throw new IllegalStateException("Missing qbit-object-storage.properties; build with Maven to generate the artifact version");
+         }
+         Properties properties = new Properties();
+         properties.load(inputStream);
+         String version = properties.getProperty("version");
+         if(version == null || version.isBlank() || version.contains("${"))
+         {
+            throw new IllegalStateException("Missing or unfiltered version in qbit-object-storage.properties; build with Maven");
+         }
+         return (version);
+      }
+      catch(IOException e)
+      {
+         throw new IllegalStateException("Could not read artifact version from qbit-object-storage.properties", e);
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Fluent setter for the config.
     *******************************************************************************/

src/main/java/com/kingsrook/qbits/objectstorage/api/StorageBucket.java

@@ -46,6 +46,7 @@ import com.kingsrook.qbits.objectstorage.internal.PostPolicySigner;
 import com.kingsrook.qbits.objectstorage.internal.ProtectionRules;
 import com.kingsrook.qbits.objectstorage.internal.RateLimiter;
 import com.kingsrook.qqq.backend.core.actions.permissions.CorePermissionEnforcer;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -96,7 +97,8 @@ public final class StorageBucket
 {
    private static final QLogger LOG = QLogger.getLogger(StorageBucket.class);
 
-   private static final Map<String, long[]> PRESIGN_COUNTS = new ConcurrentHashMap<>();
+   private static final Map<String, long[]> PRESIGN_COUNTS = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
+      "per-pod presign rate limiter; per-node rate limits are accepted for now (cluster HA spec, owner decision 4)");
 
    private final StorageBucketConfig bucket;
    private final ConnectionRuntime   connection;

src/main/java/com/kingsrook/qbits/objectstorage/app/StorageWidgets.java

@@ -106,16 +106,12 @@ public final class StorageWidgets
 
 
    /*******************************************************************************
-    ** The prefix of table and record links.
+    ** The prefix of table and record links: /app, so a link is /app/<tableName>
+    ** (the frontend serves every table, in any app, at that one flat route).
     *******************************************************************************/
    public static String tablePathPrefix()
    {
-      if(!ObjectStorageRuntime.isInstalled())
-      {
-         return ("/objectStorageAdmin");
-      }
-      ObjectStorageQBitConfig config = ObjectStorageRuntime.getConfig();
-      return (config.getParentAppName() == null ? "/" + config.getAppName() : "/" + config.getParentAppName() + "/" + config.getAppName());
+      return (ObjectStorageRuntime.isInstalled() ? ObjectStorageRuntime.getConfig().getTablePathPrefix() : "/app");
    }
 
 

src/main/java/com/kingsrook/qbits/objectstorage/index/ObjectIndex.java

@@ -43,6 +43,7 @@ import com.kingsrook.qbits.objectstorage.internal.ConfigSync;
 import com.kingsrook.qbits.objectstorage.internal.Db;
 import com.kingsrook.qbits.objectstorage.internal.ObjectStorageRuntime;
 import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
@@ -78,7 +79,8 @@ public final class ObjectIndex
 
    public static final int PAGE_SIZE = 1000;
 
-   private static final Set<String> RUNNING = java.util.concurrent.ConcurrentHashMap.newKeySet();
+   private static final Set<String> RUNNING = ClusterCaches.nodeLocal(java.util.concurrent.ConcurrentHashMap.newKeySet(),
+      "the buckets this node is scanning now (an in-flight guard, not cached data)");
 
    /** Test seam: simulate a crash after this many pages of a scan (so resume can be tested). */
    static volatile Integer failAfterPages = null;

src/main/java/com/kingsrook/qbits/objectstorage/internal/CapabilityStore.java

@@ -30,6 +30,7 @@ import com.kingsrook.qbits.objectstorage.api.Capability;
 import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
 import com.kingsrook.qbits.objectstorage.config.StorageBucketConfig;
 import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 
@@ -47,7 +48,11 @@ public final class CapabilityStore
    public static final Set<Capability> CONNECTION_SCOPE = EnumSet.of(Capability.LIST_V2, Capability.PRESIGNED_GET, Capability.PRESIGNED_PUT,
       Capability.PRESIGNED_POST, Capability.USAGE_ADMIN_API, Capability.QUOTA_ADMIN_API);
 
-   private static final Map<String, CapabilityStatus> CACHE  = new ConcurrentHashMap<>();
+   /** Cluster cache name of the capability statuses (a probe result recorded on one node clears that key on every node). */
+   public static final String CACHE_NAME = "qbit-object-storage.capabilities";
+
+   private static final Map<String, CapabilityStatus> CACHE  = ClusterCaches.shared(CACHE_NAME, new ConcurrentHashMap<>(),
+      CapabilityStore::clearCache, CapabilityStore::clearKey);
    private static final Set<String>                   LOADED = ConcurrentHashMap.newKeySet();
 
 
@@ -62,7 +67,7 @@ public final class CapabilityStore
 
 
    /*******************************************************************************
-    ** Drop everything cached.
+    ** Drop everything cached on this node.
     *******************************************************************************/
    public static void clearCache()
    {
@@ -72,6 +77,28 @@ public final class CapabilityStore
 
 
 
+   /*******************************************************************************
+    ** Drop one cached status on this node; the next read reloads the missing
+    ** statuses from the capability rows (cached ones are kept).
+    *******************************************************************************/
+   static void clearKey(String key)
+   {
+      CACHE.remove(key);
+      LOADED.clear();
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether a status is cached on this node (tests).
+    *******************************************************************************/
+   static boolean isCached(StorageBucketConfig bucket, Capability capability)
+   {
+      return (CACHE.containsKey(scopeKey(bucket, capability)));
+   }
+
+
+
    /*******************************************************************************
     ** The status of a capability for a bucket alias.
     *******************************************************************************/
@@ -112,10 +139,24 @@ public final class CapabilityStore
 
    /*******************************************************************************
     ** Record one probe result (cache and, when the tables are there, the row).
+    ** The row is written first, then the key is cleared on every node (they
+    ** reload it from the row), then cached here.
     *******************************************************************************/
    public static void record(StorageBucketConfig bucket, Capability capability, CapabilityStatus status, String detail)
    {
-      CACHE.put(scopeKey(bucket, capability), status);
+      String key = scopeKey(bucket, capability);
+      persist(bucket, capability, status, detail);
+      ClusterCaches.invalidateKey(CACHE_NAME, key);
+      CACHE.put(key, status);
+   }
+
+
+
+   /*******************************************************************************
+    ** Write one probe result to its capability row; never throws.
+    *******************************************************************************/
+   private static void persist(StorageBucketConfig bucket, Capability capability, CapabilityStatus status, String detail)
+   {
       if(!Db.available(ObjectStorageTables.BUCKET_CAPABILITY))
       {
          return;

src/main/java/com/kingsrook/qbits/objectstorage/internal/ConfigRowChangeCustomizer.java

@@ -0,0 +1,95 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.objectstorage.internal;
+
+
+import java.util.List;
+import java.util.Optional;
+import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** Post insert, update and delete of the editable config rows that the runtime
+ ** caches: a bucket row (presign limits, ConfigSync) or a header rule
+ ** (DefaultObjectHeaderPolicy). Clears that cache on every node, so an admin's
+ ** edit takes effect everywhere at once; the short TTLs remain the safety net.
+ *******************************************************************************/
+public class ConfigRowChangeCustomizer implements TableCustomizerInterface
+{
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
+   {
+      invalidate(insertInput.getTableName());
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      invalidate(updateInput.getTableName());
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postDelete(DeleteInput deleteInput, List<QRecord> records) throws QException
+   {
+      invalidate(deleteInput.getTableName());
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    ** Clear the cache that holds the table's rows, on every node.
+    *******************************************************************************/
+   static void invalidate(String tableName)
+   {
+      if(ObjectStorageTables.BUCKET.equals(tableName))
+      {
+         ClusterCaches.invalidate(ConfigSync.ROW_LIMITS_CACHE_NAME);
+      }
+      else if(ObjectStorageTables.HEADER_RULE.equals(tableName))
+      {
+         ClusterCaches.invalidate(DefaultObjectHeaderPolicy.CACHE_NAME);
+      }
+   }
+}

src/main/java/com/kingsrook/qbits/objectstorage/internal/ConfigSync.java

@@ -30,6 +30,7 @@ import com.kingsrook.qbits.objectstorage.config.CredentialsMode;
 import com.kingsrook.qbits.objectstorage.config.StorageBucketConfig;
 import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig;
 import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -48,10 +49,16 @@ public final class ConfigSync
 {
    private static final QLogger LOG = QLogger.getLogger(ConfigSync.class);
 
-   private static final Map<String, Integer> BUCKET_IDS     = new ConcurrentHashMap<>();
-   private static final Map<String, Integer> CONNECTION_IDS = new ConcurrentHashMap<>();
+   /** Cluster cache name of the bucket rows' presign limits (cleared on every node when a bucket row changes). */
+   public static final String ROW_LIMITS_CACHE_NAME = "qbit-object-storage.bucket-row-limits";
 
-   private static final Map<String, RowLimits> ROW_LIMITS = new ConcurrentHashMap<>();
+   private static final Map<String, Integer> BUCKET_IDS     = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
+      "bucket alias to row id, written by the config sync; bucket rows cannot be inserted or deleted through the UI or API, so the ids are the same on every node");
+   private static final Map<String, Integer> CONNECTION_IDS = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
+      "connection name to row id, written by the config sync; connection rows cannot be inserted or deleted through the UI or API, so the ids are the same on every node");
+
+   private static final Map<String, RowLimits> ROW_LIMITS = ClusterCaches.shared(ROW_LIMITS_CACHE_NAME, new ConcurrentHashMap<>(),
+      ConfigSync::clearRowLimits, alias -> ConfigSync.ROW_LIMITS.remove(alias));
    private static final long                 ROW_LIMIT_CACHE_MILLIS = 15_000L;
 
    private static volatile boolean synced = false;
@@ -313,7 +320,8 @@ public final class ConfigSync
 
 
    /*******************************************************************************
-    ** Forget the cached row limits (after a row edit in a test; they also expire).
+    ** Forget the cached row limits on this node (a bucket row edit clears them on
+    ** every node through ConfigRowChangeCustomizer; they also expire).
     *******************************************************************************/
    public static void clearRowLimits()
    {

src/main/java/com/kingsrook/qbits/objectstorage/internal/ConnectionRuntime.java

@@ -22,7 +22,6 @@ package com.kingsrook.qbits.objectstorage.internal;
 
 
 import java.net.URI;
-import java.util.function.Function;
 import com.kingsrook.qbits.objectstorage.config.AddressingStyle;
 import com.kingsrook.qbits.objectstorage.config.CredentialsMode;
 import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig;
@@ -30,17 +29,10 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3ChecksumMode;
 import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3ClientFactory;
-import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
 import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
-import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
-import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration;
 import software.amazon.awssdk.regions.Region;
-import software.amazon.awssdk.services.s3.DelegatingS3Client;
 import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.S3Configuration;
-import software.amazon.awssdk.services.s3.model.S3Request;
 import software.amazon.awssdk.services.s3.presigner.S3Presigner;
 
 
@@ -198,16 +190,7 @@ public class ConnectionRuntime
     *******************************************************************************/
    private S3Client buildClient()
    {
-      S3BackendMetaData backend = backendMetaData();
-      String token = config.getCredentialsMode() == CredentialsMode.STATIC_REFS ? SecretRefs.resolve(config.getSessionTokenRef()) : null;
-      AwsCredentialsProvider sessionCredentials = null;
-      if(token != null && backend.getAccessKey() != null && backend.getSecretKey() != null)
-      {
-         sessionCredentials = StaticCredentialsProvider.create(AwsSessionCredentials.create(backend.getAccessKey(), backend.getSecretKey(), token));
-      }
-
-      S3Client coreClient = S3ClientFactory.buildClient(backend);
-      return (sessionCredentials == null ? coreClient : new SessionCredentialsClient(coreClient, sessionCredentials));
+      return (S3ClientFactory.buildClient(backendMetaData()));
    }
 
 
@@ -229,8 +212,7 @@ public class ConnectionRuntime
          .withSocketTimeoutMillis(config.getReadTimeoutMs());
       if(config.getCredentialsMode() == CredentialsMode.STATIC_REFS)
       {
-         backend.withAccessKey(SecretRefs.resolve(config.getAccessKeyRef()))
-            .withSecretKey(SecretRefs.resolve(config.getSecretKeyRef()));
+         withStaticCredentials(backend, config.getAccessKeyRef(), config.getSecretKeyRef());
       }
       return (backend);
    }
@@ -270,63 +252,33 @@ public class ConnectionRuntime
 
 
    /*******************************************************************************
-    ** Static keys from the references, or the SDK default chain.
+    ** Static keys (with the session token, when set) from the references, or the
+    ** SDK default chain; core builds the provider.
     *******************************************************************************/
    private AwsCredentialsProvider credentials(String accessRef, String secretRef)
    {
+      S3BackendMetaData credentials = new S3BackendMetaData();
       if(config.getCredentialsMode() == CredentialsMode.STATIC_REFS)
       {
-         String accessKey = SecretRefs.resolve(accessRef);
-         String secretKey = SecretRefs.resolve(secretRef);
-         if(accessKey != null && secretKey != null)
-         {
-            String token = SecretRefs.resolve(config.getSessionTokenRef());
-            return (StaticCredentialsProvider.create(token == null ? AwsBasicCredentials.create(accessKey, secretKey)
-               : AwsSessionCredentials.create(accessKey, secretKey, token)));
-         }
+         withStaticCredentials(credentials, accessRef, secretRef);
       }
-      return (DefaultCredentialsProvider.builder().build());
+      return (S3ClientFactory.buildCredentialsProvider(credentials));
    }
 
 
 
    /*******************************************************************************
-    ** STAND-IN(core=QRun-IO/qqq#1165, removeBy=4.1.0-RC.9)
-    ** RC.8 S3BackendMetaData and S3ClientFactory still lack a session token or
-    ** credentials-provider hook (verified against the released JAR).
-    ** Override request credentials only; core still owns the client and transport.
+    ** Resolve static keys onto core meta data. The session token applies only
+    ** with both keys, as before (core rejects a token without keys).
     *******************************************************************************/
-   private static class SessionCredentialsClient extends DelegatingS3Client
+   private void withStaticCredentials(S3BackendMetaData backend, String accessRef, String secretRef)
    {
-      private final AwsCredentialsProvider credentialsProvider;
-
-
-
-      /*******************************************************************************
-       ** Wrap a core-built client with the connection's session credentials.
-       *******************************************************************************/
-      private SessionCredentialsClient(S3Client delegate, AwsCredentialsProvider credentialsProvider)
+      String accessKey = SecretRefs.resolve(accessRef);
+      String secretKey = SecretRefs.resolve(secretRef);
+      backend.withAccessKey(accessKey).withSecretKey(secretKey);
+      if(accessKey != null && secretKey != null)
       {
-         super(delegate);
-         this.credentialsProvider = credentialsProvider;
-      }
-
-
-
-      /*******************************************************************************
-       ** Preserve request overrides and the concrete request type while adding the
-       ** session token to signing, including streaming and paginated operations.
-       *******************************************************************************/
-      @Override
-      @SuppressWarnings("unchecked") // toBuilder().build() retains the concrete request type T.
-      protected <T extends S3Request, ReturnT> ReturnT invokeOperation(T request, Function<T, ReturnT> operation)
-      {
-         AwsRequestOverrideConfiguration.Builder overrides = request.overrideConfiguration()
-            .map(AwsRequestOverrideConfiguration::toBuilder).orElseGet(AwsRequestOverrideConfiguration::builder);
-         T signedRequest = (T) request.toBuilder()
-            .overrideConfiguration(overrides.credentialsProvider(credentialsProvider).build())
-            .build();
-         return (operation.apply(signedRequest));
+         backend.withSessionToken(SecretRefs.resolve(config.getSessionTokenRef()));
       }
    }
 }

src/main/java/com/kingsrook/qbits/objectstorage/internal/DefaultObjectHeaderPolicy.java

@@ -23,12 +23,14 @@ package com.kingsrook.qbits.objectstorage.internal;
 
 import java.util.Comparator;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.regex.Pattern;
 import com.kingsrook.qbits.objectstorage.api.ObjectHeaderPolicy;
 import com.kingsrook.qbits.objectstorage.api.ObjectHeaders;
 import com.kingsrook.qbits.objectstorage.config.BucketRole;
 import com.kingsrook.qbits.objectstorage.config.StorageBucketConfig;
 import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -44,19 +46,31 @@ public class DefaultObjectHeaderPolicy implements ObjectHeaderPolicy
 {
    private static final QLogger LOG = QLogger.getLogger(DefaultObjectHeaderPolicy.class);
 
+   /** Cluster cache name of the enabled header rules (cleared on every node when a rule changes). */
+   public static final String CACHE_NAME = "qbit-object-storage.header-rules";
+
    private static final long CACHE_MILLIS = 15_000L;
 
-   private static volatile List<QRecord> cachedRules;
-   private static volatile long          cachedAt;
+   private static final AtomicReference<CachedRules> CACHED = ClusterCaches.shared(CACHE_NAME, new AtomicReference<>(), DefaultObjectHeaderPolicy::clearCache, null);
 
 
 
    /*******************************************************************************
-    ** Drop cached rules (after the rules change).
+    ** The enabled rules and when they were read.
+    *******************************************************************************/
+   private record CachedRules(List<QRecord> rules, long readAt)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Drop the cached rules on this node (a rule edit clears them on every node
+    ** through ConfigRowChangeCustomizer).
     *******************************************************************************/
    public static void clearCache()
    {
-      cachedRules = null;
+      CACHED.set(null);
    }
 
 
@@ -119,11 +133,12 @@ public class DefaultObjectHeaderPolicy implements ObjectHeaderPolicy
     *******************************************************************************/
    private static List<QRecord> rules()
    {
-      List<QRecord> rules = cachedRules;
-      if(rules != null && System.currentTimeMillis() - cachedAt < CACHE_MILLIS)
+      CachedRules cached = CACHED.get();
+      if(cached != null && System.currentTimeMillis() - cached.readAt() < CACHE_MILLIS)
       {
-         return (rules);
+         return (cached.rules());
       }
+      List<QRecord> rules;
       if(!Db.available(ObjectStorageTables.HEADER_RULE))
       {
          return (List.of());
@@ -133,8 +148,7 @@ public class DefaultObjectHeaderPolicy implements ObjectHeaderPolicy
          rules = Db.asSystem(() -> Db.query(ObjectStorageTables.HEADER_RULE, "enabled", true));
          rules.sort(Comparator.comparing((QRecord r) -> r.getValueInteger("priority") == null ? 100 : r.getValueInteger("priority"))
             .thenComparing(r -> r.getValueInteger("id") == null ? 0 : r.getValueInteger("id")));
-         cachedRules = rules;
-         cachedAt = System.currentTimeMillis();
+         CACHED.set(new CachedRules(rules, System.currentTimeMillis()));
          return (rules);
       }
       catch(Exception e)

src/main/java/com/kingsrook/qbits/objectstorage/internal/ObjectStorageRuntime.java

@@ -33,6 +33,7 @@ import com.kingsrook.qbits.objectstorage.api.StorageAlertSink;
 import com.kingsrook.qbits.objectstorage.api.StorageChangeListener;
 import com.kingsrook.qbits.objectstorage.config.StorageBucketConfig;
 import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
 
 
 /*******************************************************************************
@@ -43,9 +44,11 @@ import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig;
  *******************************************************************************/
 public final class ObjectStorageRuntime
 {
-   private static final Map<String, ConnectionRuntime>   CONNECTIONS = new ConcurrentHashMap<>();
+   private static final Map<String, ConnectionRuntime>   CONNECTIONS = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
+      "this node's S3 clients, built from the code config at first use; row overrides apply when a client is built, the same on every node");
    private static final List<StorageChangeListener>      LISTENERS   = new CopyOnWriteArrayList<>();
-   private static final Map<String, String>              PROCESS_PERMISSIONS = new ConcurrentHashMap<>();
+   private static final Map<String, String>              PROCESS_PERMISSIONS = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
+      "built from the qbit config at startup, the same on every node");
 
    private static volatile ObjectStorageQBitConfig config;
    private static volatile ObjectHeaderPolicy      headerPolicy;

src/main/java/com/kingsrook/qbits/objectstorage/internal/ProviderDefaults.java

@@ -26,6 +26,7 @@ import java.util.Map;
 import com.kingsrook.qbits.objectstorage.api.Capability;
 import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
 import com.kingsrook.qbits.objectstorage.config.ProviderType;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
 import static com.kingsrook.qbits.objectstorage.api.CapabilityStatus.PARTIAL;
 import static com.kingsrook.qbits.objectstorage.api.CapabilityStatus.SUPPORTED;
 import static com.kingsrook.qbits.objectstorage.api.CapabilityStatus.UNKNOWN;
@@ -39,7 +40,8 @@ import static com.kingsrook.qbits.objectstorage.api.CapabilityStatus.UNSUPPORTED
  *******************************************************************************/
 public final class ProviderDefaults
 {
-   private static final Map<ProviderType, Map<Capability, CapabilityStatus>> TABLE = new EnumMap<>(ProviderType.class);
+   private static final Map<ProviderType, Map<Capability, CapabilityStatus>> TABLE = ClusterCaches.nodeLocal(new EnumMap<>(ProviderType.class),
+      "provider defaults from code, the same on every node");
 
    static
    {

src/main/java/com/kingsrook/qbits/objectstorage/internal/SecretRefs.java

@@ -21,10 +21,6 @@
 package com.kingsrook.qbits.objectstorage.internal;
 
 
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
 import java.util.List;
 import java.util.regex.Pattern;
 import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
@@ -32,14 +28,12 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
 /*******************************************************************************
- ** Resolves secret references (${env.X}, ${prop.X}, ${file.PATH}) and enforces
- ** the allow-list. Resolved values live only in memory, never in a record, a log
- ** line or a process value.
- **
- ** STAND-IN(core=QRun-IO/qqq#1165, removeBy=4.1.0-RC.9)
- ** RC.8 QMetaDataVariableInterpreter still supports env/prop, not file references.
- ** ${file.PATH} moves to QMetaDataVariableInterpreter in QQQ core
- ** (spec section 8.2); then this class delegates entirely.
+ ** Enforces the qbit's secret-reference allow-list and resolves references
+ ** (${env.X}, ${prop.X}, ${file.PATH}) through core QMetaDataVariableInterpreter.
+ ** Core reads ${file.PATH} read-only from its allowed roots (default
+ ** /var/run/secrets and /run/secrets; qqq.fileReference.roots or
+ ** QQQ_FILE_REFERENCE_ROOTS). Resolved values live only in memory, never in a
+ ** record, a log line or a process value.
  *******************************************************************************/
 public final class SecretRefs
 {
@@ -93,19 +87,7 @@ public final class SecretRefs
       {
          return (null);
       }
-      String trimmed = ref.trim();
-      if(trimmed.startsWith("${file.") && trimmed.endsWith("}"))
-      {
-         try
-         {
-            return (Files.readString(Path.of(trimmed.substring("${file.".length(), trimmed.length() - 1)), StandardCharsets.UTF_8).trim());
-         }
-         catch(IOException e)
-         {
-            return (null);
-         }
-      }
-      String value = new QMetaDataVariableInterpreter().interpret(trimmed);
+      String value = new QMetaDataVariableInterpreter().interpret(ref.trim());
       return (value == null || value.startsWith("${") ? null : value);
    }
 

src/main/java/com/kingsrook/qbits/objectstorage/model/ObjectStorageTables.java

@@ -25,6 +25,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import com.kingsrook.qbits.objectstorage.ObjectStorageQBitConfig;
+import com.kingsrook.qbits.objectstorage.internal.ConfigRowChangeCustomizer;
 import com.kingsrook.qbits.objectstorage.permissions.ObjectStoragePermissionChecker;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.ChildRecordListRenderer;
@@ -564,7 +565,9 @@ public final class ObjectStorageTables
          .withField(ro("orphanedCount", QFieldType.LONG))
          .withField(ro("orphanedBytes", QFieldType.LONG))
          .withField(ro("missingCount", QFieldType.LONG))
-         .withField(ro("lastReferenceScanDate", QFieldType.DATE_TIME)), KEY_TYPE_BUCKET, "id", false));
+         .withField(ro("lastReferenceScanDate", QFieldType.DATE_TIME)), KEY_TYPE_BUCKET, "id", false)
+         .withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(ConfigRowChangeCustomizer.class))
+         .withCustomizer(TableCustomizers.POST_UPDATE_RECORD, new QCodeReference(ConfigRowChangeCustomizer.class)));
    }
 
 
@@ -701,7 +704,10 @@ public final class ObjectStorageTables
          .withField(str("cacheControl", 300))
          .withField(str("contentDisposition", 300))
          .withField(new QFieldMetaData("priority", QFieldType.INTEGER).withDefaultValue(100))
-         .withField(new QFieldMetaData("enabled", QFieldType.BOOLEAN).withDefaultValue(true)), KEY_TYPE_BUCKET, "bucketId", true));
+         .withField(new QFieldMetaData("enabled", QFieldType.BOOLEAN).withDefaultValue(true)), KEY_TYPE_BUCKET, "bucketId", true)
+         .withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(ConfigRowChangeCustomizer.class))
+         .withCustomizer(TableCustomizers.POST_UPDATE_RECORD, new QCodeReference(ConfigRowChangeCustomizer.class))
+         .withCustomizer(TableCustomizers.POST_DELETE_RECORD, new QCodeReference(ConfigRowChangeCustomizer.class)));
    }
 
 

src/main/resources/qbit-object-storage.properties

@@ -0,0 +1 @@
+version=${project.version}

src/test/java/com/kingsrook/qbits/objectstorage/ObjectStorageHelpBundleTest.java

@@ -59,7 +59,13 @@ class ObjectStorageHelpBundleTest
 
       assertThat(instance.getTable("helpContent")).isNull();
       assertThat(HelpBundles.of(instance).getBundles()).singleElement()
-         .satisfies(bundle -> assertThat(bundle.getId()).isEqualTo("qbit-object-storage"));
+         .satisfies(bundle ->
+         {
+            assertThat(bundle.getId()).isEqualTo("qbit-object-storage");
+            assertThat(System.getProperty("qbit.artifactVersion")).isNotBlank();
+            assertThat(bundle.getVersion()).isEqualTo(System.getProperty("qbit.artifactVersion"))
+               .isEqualTo(ObjectStorageQBitProducer.VERSION);
+         });
       HelpBundles.syncOnStartup(instance);
       Set<String> keys = HelpBundles.of(instance).getBundles().get(0).getEntries().stream()
          .map(entry -> entry.getKey()).collect(Collectors.toSet());

src/test/java/com/kingsrook/qbits/objectstorage/ObjectStorageQBitProducerTest.java

@@ -44,6 +44,10 @@ class ObjectStorageQBitProducerTest
       ObjectStorageQBitConfig config    = new ObjectStorageQBitConfig().withBackendName("rdbms");
       ObjectStorageQBitProducer producer  = new ObjectStorageQBitProducer().withQBitConfig(config).withNamespace("object_storage");
 
+      String artifactVersion = System.getProperty("qbit.artifactVersion");
+      assertThat(artifactVersion).isNotBlank();
+      assertThat(ObjectStorageQBitProducer.VERSION).isEqualTo(artifactVersion);
+      assertThat(producer.getQBitMetaData().getVersion()).isEqualTo(artifactVersion);
       assertThat(producer.isEnabled()).isTrue();
       assertThat(producer.getQBitConfig()).isSameAs(config);
       assertThat(producer.getNamespace()).isEqualTo("object_storage");

src/test/java/com/kingsrook/qbits/objectstorage/index/MinioIndexTest.java

@@ -574,4 +574,40 @@ class MinioIndexTest
          .contains("Usage trend").contains("collecting since")
          .contains("Connections").contains("Never tested");
    }
+
+   /*******************************************************************************
+    ** Every link the home emits is a /app/<table>[...] route that the Next frontend
+    ** resolves, never an /<appName>/<table> path, with data and without.
+    *******************************************************************************/
+   @Test
+   void everyHomeLinkUsesTheAppRoute() throws Exception
+   {
+      String prefix = com.kingsrook.qbits.objectstorage.app.StorageWidgets.tablePathPrefix();
+      assertThat(prefix).isEqualTo("/app");
+      java.util.List<String> hrefs = new java.util.ArrayList<>(hrefs(com.kingsrook.qbits.objectstorage.app.StorageHome.render(prefix, Instant.now())));
+
+      rawPut(target.indexBucket, "media/1/a.jpg", 2048);
+      rawPut(target.indexBucket, "media/9/orphan.jpg", 1024);
+      ObjectIndex.scan("indexed");
+      ObjectIndex.headBacklog("indexed", 10);
+      ReferenceScan.run("indexed");
+      ObjectStorage.bucket("indexed").put(PutRequest.of("media/1/c.jpg", new byte[3]));
+      hrefs.addAll(hrefs(com.kingsrook.qbits.objectstorage.app.StorageHome.render(prefix, Instant.now())));
+
+      assertThat(hrefs).isNotEmpty().allSatisfy(href -> assertThat(href).startsWith("/app/"));
+      assertThat(hrefs).noneMatch(href -> href.contains("objectStorageAdmin"));
+   }
+
+
+
+   private static java.util.List<String> hrefs(String html)
+   {
+      java.util.List<String> hrefs = new java.util.ArrayList<>();
+      java.util.regex.Matcher m = java.util.regex.Pattern.compile("href=\"([^\"]*)\"").matcher(html);
+      while(m.find())
+      {
+         hrefs.add(m.group(1).replace("&amp;", "&"));
+      }
+      return (hrefs);
+   }
 }

src/test/java/com/kingsrook/qbits/objectstorage/internal/ConnectionRuntimeTest.java

@@ -38,6 +38,7 @@ import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMe
 import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3ChecksumMode;
 import com.sun.net.httpserver.Headers;
 import com.sun.net.httpserver.HttpServer;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
 import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
@@ -58,9 +59,21 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
  *******************************************************************************/
 class ConnectionRuntimeTest
 {
+   /*******************************************************************************
+    ** Core reads ${file.PATH} only inside its allowed roots; reference() points
+    ** them at each test's temporary directory.
+    *******************************************************************************/
+   @AfterEach
+   void afterEach()
+   {
+      System.clearProperty("qqq.fileReference.roots");
+   }
+
+
+
    /*******************************************************************************
     ** Every core-supported setting is mapped after resolving references. Session
-    ** tokens and presign settings have no core fields and are exercised below.
+    ** tokens and presign settings are exercised below.
     *******************************************************************************/
    @Test
    void testBackendMetaDataCarriesEverySetting(@TempDir Path dir) throws Exception
@@ -199,8 +212,8 @@ class ConnectionRuntimeTest
 
 
    /*******************************************************************************
-    ** Session credentials sign streaming reads/writes and paginator requests.
-    ** Existing request headers survive the bridge's credentials override.
+    ** Session credentials (core sessionToken) sign streaming reads/writes and
+    ** paginator requests; request overrides are untouched.
     *******************************************************************************/
    @Test
    void testSessionSigningAndRequestOverrides(@TempDir Path dir) throws Exception
@@ -313,6 +326,7 @@ class ConnectionRuntimeTest
     *******************************************************************************/
    private static String reference(Path dir, String name, String value) throws Exception
    {
+      System.setProperty("qqq.fileReference.roots", dir.toString());
       Path file = dir.resolve(name);
       Files.writeString(file, value);
       return ("${file." + file + "}");

src/test/java/com/kingsrook/qbits/objectstorage/internal/InternalsTest.java

@@ -83,10 +83,23 @@ class InternalsTest
       assertThat(SecretRefs.resolve(null)).isNull();
       assertThat(SecretRefs.resolveSetting("")).isNull();
 
+      //////////////////////////////////////////////////////////////////////
+      // core reads ${file.PATH} only inside its allowed roots (by default //
+      // /var/run/secrets and /run/secrets); point them at the temp dir.   //
+      //////////////////////////////////////////////////////////////////////
       Path file = dir.resolve("key");
       Files.writeString(file, "  from-file\n");
-      assertThat(SecretRefs.resolve("${file." + file + "}")).isEqualTo("from-file");
-      assertThat(SecretRefs.resolve("${file." + dir.resolve("missing") + "}")).isNull();
+      assertThat(SecretRefs.resolve("${file." + file + "}")).as("outside the allowed roots").isNull();
+      System.setProperty("qqq.fileReference.roots", dir.toString());
+      try
+      {
+         assertThat(SecretRefs.resolve("${file." + file + "}")).isEqualTo("from-file");
+         assertThat(SecretRefs.resolve("${file." + dir.resolve("missing") + "}")).isNull();
+      }
+      finally
+      {
+         System.clearProperty("qqq.fileReference.roots");
+      }
    }
 
 

src/test/java/com/kingsrook/qbits/objectstorage/internal/ObjectStorageClusterCachesTest.java

@@ -0,0 +1,240 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.objectstorage.internal;
+
+
+import java.util.EnumSet;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.function.Consumer;
+import com.kingsrook.qbits.objectstorage.ObjectStorageQBitConfig;
+import com.kingsrook.qbits.objectstorage.api.Capability;
+import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
+import com.kingsrook.qbits.objectstorage.config.BucketRole;
+import com.kingsrook.qbits.objectstorage.config.ProviderType;
+import com.kingsrook.qbits.objectstorage.config.StorageBucketConfig;
+import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig;
+import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
+import com.kingsrook.qbits.objectstorage.testsupport.StorageTestSupport;
+import com.kingsrook.qqq.backend.core.cluster.ClusterBus;
+import com.kingsrook.qqq.backend.core.cluster.ClusterBusInterface;
+import com.kingsrook.qqq.backend.core.cluster.ClusterBusMetaData;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCacheAudit;
+import com.kingsrook.qqq.backend.core.cluster.ClusterEvent;
+import com.kingsrook.qqq.backend.core.cluster.QNodeRoles;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** The Object Storage caches are correct on several nodes: every static cache
+ ** is registered with ClusterCaches, a probe result recorded on one node clears
+ ** that capability on the others, and an edited bucket row or header rule
+ ** clears its cache on every node.
+ *******************************************************************************/
+class ObjectStorageClusterCachesTest
+{
+   /** the "wire": events every node's transport sees */
+   static final List<Consumer<ClusterEvent>> WIRE = new CopyOnWriteArrayList<>();
+
+
+
+   /*******************************************************************************
+    ** A transport over the static WIRE, so a "remote" publish reaches us.
+    *******************************************************************************/
+   public static class WireBus implements ClusterBusInterface
+   {
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public void publish(ClusterEvent event)
+      {
+         WIRE.forEach(listener -> listener.accept(event));
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public AutoCloseable subscribe(Consumer<ClusterEvent> listener)
+      {
+         WIRE.add(listener);
+         return (() -> WIRE.remove(listener));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** A synced instance whose buckets never touch a real store, on the wire as
+    ** node-a.
+    *******************************************************************************/
+   @BeforeEach
+   void setUp() throws Exception
+   {
+      System.setProperty("os.unit.ak", "ak");
+      System.setProperty("os.unit.sk", "sk");
+      StorageTestSupport.setUp(new ObjectStorageQBitConfig().withBackendName(StorageTestSupport.MEMORY).withBucketScoping(false).withOpenWhenAnonymous(true).withAdminApp(true)
+            .withAllowedSecretRefPatterns(List.of("prop.os.unit.*"))
+            .withConnection(new StorageConnectionConfig("main").withProviderType(ProviderType.MINIO).withEndpoint("http://localhost:1").withRegion("us-east-1")
+               .withAccessKeyRef("${prop.os.unit.ak}").withSecretKeyRef("${prop.os.unit.sk}"))
+            .withBucket(new StorageBucketConfig("media").withConnection("main").withBucketName("media-bucket").withRole(BucketRole.PRIVATE).withIndexed(false)),
+         instance -> instance.withSupplementalMetaData(new ClusterBusMetaData().withBus(new QCodeReference(WireBus.class))));
+      QNodeRoles.set(new QNodeRoles.Settings(EnumSet.allOf(QNodeRoles.Role.class), true, true, true, null, "node-a"));
+      ClusterBus.connect(QContext.getQInstance());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @AfterEach
+   void tearDown()
+   {
+      ClusterBus.close(QContext.getQInstance());
+      WIRE.clear();
+      QNodeRoles.set(null);
+      StorageTestSupport.tearDown();
+   }
+
+
+
+   /*******************************************************************************
+    ** Every static Memoization in the QBit is registered (shared or node-local).
+    *******************************************************************************/
+   @Test
+   void testNoUnregisteredStaticCaches() throws Exception
+   {
+      assertThat(ClusterCacheAudit.unregisteredStaticCaches("com.kingsrook.qbits.objectstorage")).isEmpty();
+   }
+
+
+
+   /*******************************************************************************
+    ** Another node's probe result clears the capability here; the next read
+    ** reloads it from the capability row.
+    *******************************************************************************/
+   @Test
+   void testRemoteProbeResultClearsCapability() throws Exception
+   {
+      StorageBucketConfig bucket = ObjectStorageRuntime.bucketConfig("media");
+      CapabilityStore.record(bucket, Capability.VERSIONING, CapabilityStatus.UNSUPPORTED, "probe here");
+      assertThat(CapabilityStore.isCached(bucket, Capability.VERSIONING)).isTrue();
+
+      ////////////////////////////////////////////////////////////
+      // node-b probes again and writes SUPPORTED to the row    //
+      ////////////////////////////////////////////////////////////
+      Integer bucketId = ConfigSync.bucketId("media");
+      Db.asSystem(() ->
+      {
+         QRecord row = Db.first(ObjectStorageTables.BUCKET_CAPABILITY, "bucketId", bucketId, "capability", Capability.VERSIONING.name());
+         Db.update(ObjectStorageTables.BUCKET_CAPABILITY, List.of(new QRecord().withValue("id", row.getValue("id")).withValue("status", CapabilityStatus.SUPPORTED.name())));
+         return (null);
+      });
+      ClusterEvent remote = new ClusterEvent(ClusterEvent.TYPE_MEMO_INVALIDATE, CapabilityStore.CACHE_NAME, "b:media:" + Capability.VERSIONING);
+      remote.setOriginNodeId("node-b");
+      WIRE.forEach(listener -> listener.accept(remote));
+
+      assertThat(CapabilityStore.isCached(bucket, Capability.VERSIONING)).isFalse();
+      assertThat(CapabilityStore.status("media", Capability.VERSIONING)).isEqualTo(CapabilityStatus.SUPPORTED);
+   }
+
+
+
+   /*******************************************************************************
+    ** A probe result recorded here tells the other nodes, and stays cached here.
+    *******************************************************************************/
+   @Test
+   void testProbeResultPublishesInvalidation()
+   {
+      List<ClusterEvent> sent = sentEvents();
+      StorageBucketConfig bucket = ObjectStorageRuntime.bucketConfig("media");
+      CapabilityStore.record(bucket, Capability.VERSIONING, CapabilityStatus.SUPPORTED, "probe here");
+
+      assertThat(CapabilityStore.status("media", Capability.VERSIONING)).isEqualTo(CapabilityStatus.SUPPORTED);
+      assertThat(sent).anySatisfy(event -> assertInvalidation(event, CapabilityStore.CACHE_NAME, "b:media:VERSIONING"));
+   }
+
+
+
+   /*******************************************************************************
+    ** An edited bucket row takes effect at once and tells the other nodes.
+    *******************************************************************************/
+   @Test
+   void testBucketRowEditInvalidatesRowLimitsEverywhere() throws Exception
+   {
+      StorageBucketConfig bucket = ObjectStorageRuntime.bucketConfig("media");
+      assertThat(ConfigSync.effectiveMaxPresignGetSeconds(bucket)).isEqualTo(3600);
+
+      List<ClusterEvent> sent = sentEvents();
+      Db.update(ObjectStorageTables.BUCKET, List.of(new QRecord().withValue("id", ConfigSync.bucketId("media")).withValue("maxPresignGetSeconds", 60)));
+
+      assertThat(ConfigSync.effectiveMaxPresignGetSeconds(bucket)).as("no wait for the 15 s cache").isEqualTo(60);
+      assertThat(sent).anySatisfy(event -> assertInvalidation(event, ConfigSync.ROW_LIMITS_CACHE_NAME, null));
+   }
+
+
+
+   /*******************************************************************************
+    ** An edited header rule tells the other nodes.
+    *******************************************************************************/
+   @Test
+   void testHeaderRuleEditInvalidatesRulesEverywhere() throws Exception
+   {
+      List<ClusterEvent> sent = sentEvents();
+      StorageTestSupport.insert(ObjectStorageTables.HEADER_RULE, new QRecord().withValue("name", "all").withValue("keyGlob", "**").withValue("cacheControl", "no-store"));
+      assertThat(sent).anySatisfy(event -> assertInvalidation(event, DefaultObjectHeaderPolicy.CACHE_NAME, null));
+   }
+
+
+
+   /*******************************************************************************
+    ** Collect what this node puts on the wire.
+    *******************************************************************************/
+   private static List<ClusterEvent> sentEvents()
+   {
+      List<ClusterEvent> sent = new CopyOnWriteArrayList<>();
+      WIRE.add(sent::add);
+      return (sent);
+   }
+
+
+
+   /*******************************************************************************
+    ** A memo invalidation from node-a for a cache (and key).
+    *******************************************************************************/
+   private static void assertInvalidation(ClusterEvent event, String name, String key)
+   {
+      assertThat(event.getType()).isEqualTo(ClusterEvent.TYPE_MEMO_INVALIDATE);
+      assertThat(event.getName()).isEqualTo(name);
+      assertThat(event.getKey()).isEqualTo(key);
+      assertThat(event.getOriginNodeId()).isEqualTo("node-a");
+   }
+}

src/test/java/com/kingsrook/qbits/objectstorage/internal/TablesAndConfigTest.java

@@ -205,7 +205,7 @@ class TablesAndConfigTest
       new com.kingsrook.qbits.objectstorage.ObjectStorageQBitProducer().withQBitConfig(config).produce(instance);
       assertThat(instance.getApp("objectStorageAdmin").getParentAppName()).isEqualTo("admin");
       assertThat(instance.getApp("admin").getChildren()).extracting(c -> c.getName()).contains("objectStorageAdmin");
-      assertThat(com.kingsrook.qbits.objectstorage.app.StorageWidgets.tablePathPrefix()).isEqualTo("/admin/objectStorageAdmin");
+      assertThat(com.kingsrook.qbits.objectstorage.app.StorageWidgets.tablePathPrefix()).isEqualTo("/app");
    }