ci: centralize QBit source scanning

Co-Authored-By: Codex <noreply@openai.com>
ci: adopt central orb 0.7.12 and stable gates (#35 )
2026-10-21 10:19:15 -05:00 · 2026-10-21 08:58:38 -05:00 · 2026-10-21 02:30:37 -05:00 · 2026-10-21 01:02:00 -05:00 · 2026-10-20 18:49:56 -05:00 · 2026-10-20 18:42:41 -05:00
36 changed files with 1477 additions and 232 deletions
@@ -1,60 +1,14 @@
 version: 2.1
-
+setup: true
 orbs:
-   qqq-orb: kingsrook/qqq-orb@0.6.9
-
-commands:
-   configure-package-install:
-      steps:
-         - run:
-              name: Keep package installation unattended
-              command: |
-                 # These build machines are discarded after the job.
-                 sudo install -d /etc/needrestart/conf.d
-                 printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
-
+  qqq-orb: kingsrook/qqq-orb@0.7.12
 workflows:
-   test_only:
-      jobs:
-         - qqq-orb/mvn_test_only:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              filters:
-                 branches:
-                    ignore: /(develop|release\/.*)/
-
-   publish_snapshot:
-      jobs:
-         - qqq-orb/mvn_publish:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              branch_type: snapshot
-              filters:
-                 branches:
-                    only: [develop]
-
-   publish_release_candidate:
-      jobs:
-         - qqq-orb/mvn_publish:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              branch_type: release_candidate
-              filters:
-                 branches:
-                    only: [/release\/.*/]
-
-   publish_release:
-      jobs:
-         - qqq-orb/mvn_publish:
-              pre-steps:
-                 - configure-package-install
-              context: [qqq-maven-registry-credentials]
-              branch_type: release
-              filters:
-                 branches:
-                    ignore: /.*/
-                 tags:
-                    only: [/^v\d+\.\d+\.\d+$/]
+  setup:
+    jobs:
+    - qqq-orb/generate_pipeline:
+        orb_version: 0.7.12
+        filters:
+          branches:
+            ignore: automation/qbit-rc-pending
+          tags:
+            only: /^v\d+\.\d+\.\d+$/
@@ -0,0 +1,23 @@
+name: CodeQL
+'on':
+  push:
+    branches:
+    - main
+    - develop
+    - release/**
+    - hotfix/**
+  pull_request: {}
+  schedule:
+  - cron: 41 16 * * 0
+  workflow_dispatch: {}
+permissions:
+  contents: read
+  actions: read
+  security-events: write
+  packages: read
+concurrency:
+  group: codeql-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+jobs:
+  analyze:
+    uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22
@@ -1,25 +1,74 @@
-# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
-# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
-# the release line and the release/* push runs CircleCI publish_release_candidate.
 name: Release candidate
-
-on:
+'on':
  workflow_dispatch:
    inputs:
      dry_run:
-        description: Print the plan without merging or creating branches
+        description: Plan a cut without changing release state
        type: boolean
        default: false
+      finalize:
+        description: Retry finalization of an existing cut; do not cut another RC
+        type: boolean
+        default: false
+      recover:
+        description: Recover the same published RC after a deployment timeout; never
+          reupload
+        type: boolean
+        default: false
+      circle_build:
+        description: Original Circle publisher build number (recovery only)
+        type: string
+        default: ''
+      merge_sha:
+        description: Existing cut merge_sha (finalize retry only)
+        type: string
+        default: ''
+      bump_sha:
+        description: Existing cut bump_sha (finalize retry only)
+        type: string
+        default: ''
+      branch:
+        description: Existing cut branch (finalize retry only)
+        type: string
+        default: ''
+      version:
+        description: Existing cut version (finalize retry only)
+        type: string
+        default: ''
  pull_request:
-    types: [closed]
-    branches: [develop]
-
+    types:
+    - closed
+    branches:
+    - develop
+  repository_dispatch:
+    types:
+    - qbit-rc-published
 permissions:
  contents: write
-
+  actions: read
+  checks: read
 jobs:
  release-rc:
-    if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
-    uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
+    if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
+      || (github.event_name == 'pull_request' && github.event.pull_request.merged
+      == true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
+      'release:rc'))
+    uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
    with:
-      dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
+      dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
+        }}
+      finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
+        == 'workflow_dispatch' && inputs.finalize) }}
+      recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
+        }}
+      circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
+        || '' }}
+      merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
+        || inputs.merge_sha || '' }}
+      bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
+        || inputs.bump_sha || '' }}
+      branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
+        || inputs.branch || '' }}
+      version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
+        || inputs.version || '' }}
+    secrets: inherit
@@ -0,0 +1,16 @@
+profile: qbit
+orb_version: 0.7.12
+default_branch: develop
+java_version: '21'
+browsers: true
+api_checks: false
+docs_only: false
+release_on_main: false
+release_tag_policy: stable
+test_branch_policy: non-release
+publish:
+  snapshot: true
+  release_candidate: true
+  release: true
+  asciidoc: false
+release_completion: true
@@ -5,6 +5,8 @@ All notable changes to this project are documented here. The format follows Keep
 ## [0.1.0] - Unreleased

 ### Added
+- Central health supplier `object_storage.connection.<id>` (DEPENDENCY/STORAGE, 300s interval, 20000ms timeout), reusing bounded read-only capability probes with safe problem kinds and clock-skew details. Turn a connection's check off in the central `health_check` table.
+- In-memory health tests for registration, refresh, UP/DOWN/DEGRADED/UNKNOWN outcomes, deadlines, read-only execution and sentinel-secret exclusion.
 - Expanded the host-neutral in-app help bundle to cover storage fields, actions, widgets and possible values, with live references and a `HelpBundleAssertions` coverage gate. Registered it for QQQ startup sync and HOST overrides; retained inline table/app help for hosts without help-content tables.
 - Every table explains itself: a help bundle (`help/qbit-object-storage.yaml`, QQQ help-bundle format) says what each `object_storage_*` table is for and how its rows get created; it shows on the table screen and when the table is empty. Applied inline by `ObjectStorageHelp` for hosts without help-content tables; a host's own help is kept.
 - A Setup card on the Storage home (connections, buckets, header rules, protection rules) with the state and next step of each, resolved the way the runtime resolves them.
@@ -24,7 +26,16 @@ All notable changes to this project are documented here. The format follows Keep
 - Foreign keys on every relation between the qbit tables (`fk_<table>_<column>`), generated by `ObjectStorageSchema`: owned observed state (capabilities, folders, objects, versions, usage) cascades from its bucket or object, `object.folder_id` and `object_reference.object_id` are set null, and a bucket's connection, header and protection rules and operation log restrict. Each key is added by an idempotent block that first records rows whose parent is gone in `qqq_fk_quarantine` and clears (nullable column) or deletes (required column) them, so it also upgrades an existing schema. New indexes `ix_object_storage_bucket_1` (connection_id), `ix_object_storage_header_rule_1` and `ix_object_storage_protection_rule_1` (bucket_id) back the keys. Upgrade: run the generated statements again (or the host's copy of them).
 - Conformance suite against MinIO and Garage (Testcontainers), service, index and action tests, and the index and actions on PostgreSQL.

+### Removed
+- The QRun-IO/qqq#1165 stand-ins: `SessionCredentialsClient` (session tokens now go through core `S3BackendMetaData.sessionToken` / `S3ClientFactory`, including for the presigner) and the file reading in `SecretRefs` (core `QMetaDataVariableInterpreter` resolves `${file.PATH}`). **Behavior change:** core reads `${file.PATH}` only inside its allowed roots, which are `/var/run/secrets` and `/run/secrets` by default. A host that mounts storage secrets elsewhere must set `qqq.fileReference.roots` or `QQQ_FILE_REFERENCE_ROOTS`. The qbit's default allow-list (`file./var/run/secrets/storage/*`) is already inside the default roots.
+- None for the health migration; both Test connection record actions, including on-demand write tests, are retained.
+
 ### Changed
+- Multi-node deployments: the runtime caches are registered with QQQ `ClusterCaches`. Shared (cleared on every node): the bucket rows' presign limits (`qbit-object-storage.bucket-row-limits`, on a bucket row insert or update), the enabled header rules (`qbit-object-storage.header-rules`, on a header rule insert, update or delete), both through the new `ConfigRowChangeCustomizer`, and the capability statuses (`qbit-object-storage.capabilities`: a probe result recorded on one node clears that capability on the others, which reload it from the capability row; before, other nodes kept the old status until restart). The 15 s TTLs remain the safety net. Node-local, with reasons: the alias and connection row-id maps, the S3 client map, the process-permission map, the provider defaults, the running-scan guard and the per-pod presign rate limiter (`StorageBucket.PRESIGN_COUNTS`; per-node rate limits are accepted for now). `ObjectStorageClusterCachesTest` audits the package for unregistered static caches and covers remote and local invalidation.
+- Builds against QQQ `4.1.0-SNAPSHOT` (`qqq.snapshot.version`) for `ClusterCaches` until the next QQQ RC ships; a release must pin that RC.
+- Read the qbit metadata version from the Maven-filtered artifact version so release metadata matches the build and help bundle.
+- Rechecked the released RC.8 JARs: core still has no S3 session-token/provider hook or file-reference interpreter. Retained both compatibility shims and moved their review deadline to RC.9 under issue #10.
+- Raised the QQQ BOM baseline to released `4.1.0-RC.8` for the central health API.
 - Raised the QQQ BOM baseline to released `4.1.0-RC.7` for the core help-bundle registry and coverage assertions.
 - Raised the QQQ BOM baseline to released `4.1.0-RC.5` and delegated S3 client construction to core `S3ClientFactory.buildClient`. Removed local HTTP client and trust-manager implementations; existing endpoint, region, addressing, checksum, TLS/CA, credentials and timeout settings remain supported.
 - Clients remain owned by each connection runtime and are rebuilt on reset/reinstall. Retry behavior now follows core/SDK defaults instead of forcing `ADAPTIVE_V2`; hosts can select the SDK retry mode through `AWS_RETRY_MODE` or `aws.retryMode`.
@@ -15,7 +15,7 @@ S3-compatible storage as a capability for any QQQ app, and an admin app to run i
 ## Requirements

 - Java 21.
- QQQ `4.1.0-RC.7` (released BOM), including `qqq-backend-module-filesystem`. `ConnectionRuntime` maps connection settings to core's `S3BackendMetaData` and uses `S3ClientFactory.buildClient`; core owns HTTP, TLS and checksum setup. The unused SFTP/SSH dependency is excluded. See the [dependency diff](docs/S3-FACTORY-MIGRATION.md).
+- QQQ `4.1.0-RC.8` (released BOM), including `qqq-backend-module-filesystem`. `ConnectionRuntime` maps connection settings to core's `S3BackendMetaData` and uses `S3ClientFactory.buildClient`; core owns HTTP, TLS and checksum setup. The unused SFTP/SSH dependency is excluded. See the [dependency diff](docs/S3-FACTORY-MIGRATION.md).
 - A QQQ backend for the qbit's tables (PostgreSQL is the intended store). `ObjectStorageSchema.postgresScript(config)` generates the DDL from the same metadata the app runs on, for your Liquibase or Flyway migration.

 ## Install
@@ -92,6 +92,21 @@ Every write is a standard streamed ETL: **Review** (the dry run, with summary li

 Permissions: `object_storage.view`, `.write`, `.delete`, `.admin` (admin implies the rest). Per-bucket scoping uses record security locks on the `object_storage_bucket` and `object_storage_connection` security key types (switch off with `withBucketScoping(false)` for a single-administrator app).

+## Health checks
+
+The producer registers `object_storage.connection.<id>` for each configured
+`object_storage_connection` row: **DEPENDENCY**, **STORAGE**, every **300s**,
+with a **20000ms** timeout and component `object_storage`. The central health runner
+refreshes the supplier as rows change. The existing `CapabilityProbe` runs only
+read-only calls within the deadline; it never runs write or presign tests or stores
+free-form capability diagnostics. Connection failures are DOWN, clock skew above
+900 seconds is DEGRADED, and connections without buckets are UNKNOWN. Details
+contain only problem kind, clock skew seconds and bucket count.
+
+Turn a connection's check off or change its interval in the central `health_check`
+table. Both **Test connection** record actions remain available, including
+on-demand write tests.
+
 ## Setup and starter defaults

 The Storage home has a **Setup** card: connections and buckets (declared in code, synced into their tables), header rules, and protection rules, each with its state and next step. Every table has help (`help/qbit-object-storage.yaml`) saying what it is for and how its rows get created, which also shows when the table is empty.
@@ -26,16 +26,21 @@ The public qbit configuration API is unchanged.

 ## Remaining local code

- Core RC.5 has no session-token field or credentials-provider customization
-  in `S3BackendMetaData` / `S3ClientFactory.buildClient`. The narrow
-  `SessionCredentialsClient` bridge adds session credentials to requests while
-  retaining their overrides; core still creates the client and HTTP transport.
-  Core needs session-token metadata (including enrichment and cache-key support)
-  and `AwsSessionCredentials` construction, or a caller-provided credentials hook.
-  Marker: `STAND-IN(core=QRun-IO/qqq#1165, removeBy=4.1.0-RC.7)`.
- Core has no presigner factory. The local presigner retains its separate public
-  endpoint and least-privilege keys, along with existing session-token handling.
- `SecretRefs` file resolution is unchanged and carries the same RC.7 marker.
+- Core has no presigner factory. The local presigner keeps its separate public
+  endpoint and least-privilege keys; its credentials (including the session
+  token) come from core `S3ClientFactory.buildCredentialsProvider`.
+- `SecretRefs` keeps only the qbit's allow-list; resolution, including
+  `${file.PATH}`, is core `QMetaDataVariableInterpreter`.
+
+## Stand-ins removed (QRun-IO/qqq#1165)
+
+- `SessionCredentialsClient` is gone: core `S3BackendMetaData.sessionToken`
+  makes `S3ClientFactory` sign with `AwsSessionCredentials`.
+- `SecretRefs` no longer reads files itself. Core reads `${file.PATH}`
+  read-only, and only inside its allowed roots: `/var/run/secrets` and
+  `/run/secrets` by default, or `qqq.fileReference.roots` /
+  `QQQ_FILE_REFERENCE_ROOTS` (comma-separated). The qbit's default allow-list
+  (`file./var/run/secrets/storage/*`) is inside the default roots.

 ## Resolved dependency diff

@@ -172,7 +172,35 @@
   </dependencies>

   <build>
+      <resources>
+         <!-- help bundles carry the artifact version (QQQ help bundle provenance) -->
+         <resource>
+            <directory>src/main/resources</directory>
+            <filtering>true</filtering>
+            <includes>
+               <include>help/*.yaml</include>
+               <include>qbit-object-storage.properties</include>
+            </includes>
+         </resource>
+         <resource>
+            <directory>src/main/resources</directory>
+            <filtering>false</filtering>
+            <excludes>
+               <exclude>help/*.yaml</exclude>
+               <exclude>qbit-object-storage.properties</exclude>
+            </excludes>
+         </resource>
+      </resources>
      <plugins>
+         <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <configuration>
+               <systemPropertyVariables combine.children="append">
+                  <qbit.artifactVersion>${project.version}</qbit.artifactVersion>
+               </systemPropertyVariables>
+            </configuration>
+         </plugin>
         <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-checkstyle-plugin</artifactId>
@@ -255,14 +283,14 @@
   <profiles>
      <!-- Built against qqq 4.1 (QBitMetaDataProducer). Until qbit-build-parent  -->
      <!-- imports the released 4.1 BOM, this profile is active by default and     -->
-      <!-- imports qqq-bom-pom 4.1.0-RC.7 ahead of the parent's 4.0.0 BOM.         -->
+      <!-- imports qqq-bom-pom 4.1.0-RC.15 ahead of the parent's 4.0.0 BOM.        -->
      <profile>
         <id>qqq-snapshot</id>
         <activation>
            <activeByDefault>true</activeByDefault>
         </activation>
         <properties>
-            <qqq.snapshot.version>4.1.0-RC.7</qqq.snapshot.version>
+            <qqq.snapshot.version>4.1.0-RC.15</qqq.snapshot.version>
         </properties>
         <dependencyManagement>
            <dependencies>
@@ -62,6 +62,7 @@ public class ObjectStorageQBitConfig implements QBitConfig
   private String appName = "objectStorageAdmin";
   private String appLabel = "Storage";
   private String parentAppName;
+   private String tablePathPrefix;
   private List<String> allowedSecretRefPatterns = new ArrayList<>(SecretRefs.DEFAULT_ALLOWED_PATTERNS);
   private Integer bulkCap = 10_000;
   private Boolean bucketScoping = true;
@@ -425,6 +426,39 @@ public class ObjectStorageQBitConfig implements QBitConfig



+   /********************************************************************************
+    ** Getter for tablePathPrefix: the front of the links the Storage home makes to
+    ** tables and records.  Defaults to /app, the route every QQQ frontend serves
+    ** tables at (/app/<tableName>).
+    *******************************************************************************/
+   public String getTablePathPrefix()
+   {
+      return (this.tablePathPrefix == null ? "/app" : this.tablePathPrefix);
+   }
+
+
+
+   /********************************************************************************
+    ** Setter for tablePathPrefix.
+    *******************************************************************************/
+   public void setTablePathPrefix(String tablePathPrefix)
+   {
+      this.tablePathPrefix = tablePathPrefix;
+   }
+
+
+
+   /********************************************************************************
+    ** Fluent setter for tablePathPrefix.
+    *******************************************************************************/
+   public ObjectStorageQBitConfig withTablePathPrefix(String tablePathPrefix)
+   {
+      this.tablePathPrefix = tablePathPrefix;
+      return (this);
+   }
+
+
+
   /********************************************************************************
    ** Getter for allowedSecretRefPatterns.
    *******************************************************************************/
@@ -21,9 +21,13 @@
 package com.kingsrook.qbits.objectstorage;


+import java.io.IOException;
+import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Properties;
 import com.kingsrook.qbits.objectstorage.app.StorageWidgets;
+import com.kingsrook.qbits.objectstorage.health.ObjectStorageConnectionChecks;
 import com.kingsrook.qbits.objectstorage.internal.ConfigSync;
 import com.kingsrook.qbits.objectstorage.internal.DefaultObjectHeaderPolicy;
 import com.kingsrook.qbits.objectstorage.internal.ObjectStorageRuntime;
@@ -31,8 +35,11 @@ import com.kingsrook.qbits.objectstorage.model.ObjectStorageHelp;
 import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
 import com.kingsrook.qbits.objectstorage.processes.ObjectStorageProcesses;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.health.HealthCheckSupplierMetaData;
+import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundles;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
@@ -56,13 +63,41 @@ public class ObjectStorageQBitProducer implements QBitMetaDataProducer<ObjectSto
 {
   public static final String GROUP_ID    = "com.kingsrook.qbits";
   public static final String ARTIFACT_ID = "qbit-object-storage";
-   public static final String VERSION     = "0.1.0";
+   public static final String VERSION     = loadVersion();

   private ObjectStorageQBitConfig qBitConfig;
   private String                  namespace;



+   /*******************************************************************************
+    ** Read the artifact version from the Maven-filtered resource.
+    *******************************************************************************/
+   private static String loadVersion()
+   {
+      try(InputStream inputStream = ObjectStorageQBitProducer.class.getClassLoader().getResourceAsStream("qbit-object-storage.properties"))
+      {
+         if(inputStream == null)
+         {
+            throw new IllegalStateException("Missing qbit-object-storage.properties; build with Maven to generate the artifact version");
+         }
+         Properties properties = new Properties();
+         properties.load(inputStream);
+         String version = properties.getProperty("version");
+         if(version == null || version.isBlank() || version.contains("${"))
+         {
+            throw new IllegalStateException("Missing or unfiltered version in qbit-object-storage.properties; build with Maven");
+         }
+         return (version);
+      }
+      catch(IOException e)
+      {
+         throw new IllegalStateException("Could not read artifact version from qbit-object-storage.properties", e);
+      }
+   }
+
+
+
   /*******************************************************************************
    ** Fluent setter for the config.
    *******************************************************************************/
@@ -188,6 +223,9 @@ public class ObjectStorageQBitProducer implements QBitMetaDataProducer<ObjectSto
         output.add(app);
      }
      output.addSelfToInstance(qInstance);
+      HealthChecksMetaData.ofOrWithNew(qInstance).withSupplier(new HealthCheckSupplierMetaData()
+         .withNamePrefix("object_storage.connection").withComponent("object_storage")
+         .withSupplier(new QCodeReference(ObjectStorageConnectionChecks.class)));
      if(Boolean.TRUE.equals(qBitConfig.getAdminApp()) && qBitConfig.getParentAppName() != null && qInstance.getApp(qBitConfig.getParentAppName()) != null)
      {
         ///////////////////////////////////////////////////////////////////////////////
@@ -46,6 +46,7 @@ import com.kingsrook.qbits.objectstorage.internal.PostPolicySigner;
 import com.kingsrook.qbits.objectstorage.internal.ProtectionRules;
 import com.kingsrook.qbits.objectstorage.internal.RateLimiter;
 import com.kingsrook.qqq.backend.core.actions.permissions.CorePermissionEnforcer;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -96,7 +97,8 @@ public final class StorageBucket
 {
   private static final QLogger LOG = QLogger.getLogger(StorageBucket.class);

-   private static final Map<String, long[]> PRESIGN_COUNTS = new ConcurrentHashMap<>();
+   private static final Map<String, long[]> PRESIGN_COUNTS = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
+      "per-pod presign rate limiter; per-node rate limits are accepted for now (cluster HA spec, owner decision 4)");

   private final StorageBucketConfig bucket;
   private final ConnectionRuntime   connection;
@@ -106,16 +106,12 @@ public final class StorageWidgets


   /*******************************************************************************
-    ** The prefix of table and record links.
+    ** The prefix of table and record links: /app, so a link is /app/<tableName>
+    ** (the frontend serves every table, in any app, at that one flat route).
    *******************************************************************************/
   public static String tablePathPrefix()
   {
-      if(!ObjectStorageRuntime.isInstalled())
-      {
-         return ("/objectStorageAdmin");
-      }
-      ObjectStorageQBitConfig config = ObjectStorageRuntime.getConfig();
-      return (config.getParentAppName() == null ? "/" + config.getAppName() : "/" + config.getParentAppName() + "/" + config.getAppName());
+      return (ObjectStorageRuntime.isInstalled() ? ObjectStorageRuntime.getConfig().getTablePathPrefix() : "/app");
   }


@@ -0,0 +1,121 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.objectstorage.health;
+
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import com.kingsrook.qbits.objectstorage.internal.CapabilityProbe;
+import com.kingsrook.qbits.objectstorage.internal.ConfigSync;
+import com.kingsrook.qbits.objectstorage.internal.Db;
+import com.kingsrook.qbits.objectstorage.internal.ObjectStorageRuntime;
+import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
+import com.kingsrook.qqq.backend.core.alerts.AlertSeverity;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.health.HealthCategory;
+import com.kingsrook.qqq.backend.core.health.HealthCheckInput;
+import com.kingsrook.qqq.backend.core.health.HealthCheckKind;
+import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData;
+import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome;
+import com.kingsrook.qqq.backend.core.health.HealthCheckSupplier;
+import com.kingsrook.qqq.backend.core.health.HealthDetails;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+
+
+/*******************************************************************************
+ ** Dynamic, read-only checks for configured connection rows. Diagnostic text and
+ ** resolved settings never cross the health boundary.
+ *******************************************************************************/
+public class ObjectStorageConnectionChecks implements HealthCheckSupplier
+{
+   /*******************************************************************************
+    ** Resolve current rows on every refresh; removed configuration is not probed.
+    *******************************************************************************/
+   @Override
+   public List<HealthCheckMetaData> supply(QInstance qInstance) throws QException
+   {
+      ConfigSync.ensureSynced();
+      List<HealthCheckMetaData> checks = new ArrayList<>();
+      for(QRecord row : Db.asSystem(() -> Db.query(ObjectStorageTables.CONNECTION)))
+      {
+         if(ObjectStorageRuntime.getConfig().findConnection(row.getValueString("name")) == null)
+         {
+            continue;
+         }
+         Integer id = row.getValueInteger("id");
+         String name = "object_storage.connection." + id.toString().toLowerCase(Locale.ROOT);
+         checks.add(new HealthCheckMetaData().withName(name).withLabel("Storage connection " + id)
+            .withComponent("object_storage").withCategory(HealthCategory.STORAGE).withKind(HealthCheckKind.DEPENDENCY)
+            .withIntervalSeconds(300).withTimeoutMs(20000).withSeverity(AlertSeverity.ERROR)
+            .withCheckInstance(input -> run(id, input)));
+      }
+      return (checks);
+   }
+
+
+
+   /*******************************************************************************
+    ** Check the current row, reuse the bounded capability probe, and expose only
+    ** numeric facts and a fixed problem vocabulary (never exception messages).
+    *******************************************************************************/
+   private static HealthCheckOutcome run(Integer id, HealthCheckInput input) throws QException
+   {
+      if(!Instant.now().isBefore(input.getDeadline()) || Thread.currentThread().isInterrupted())
+      {
+         return (HealthCheckOutcome.down("Storage probe timed out").withDetails(new HealthDetails().withValue("problemKind", "timeout")));
+      }
+      QRecord row = Db.asSystem(() -> Db.first(ObjectStorageTables.CONNECTION, "id", id));
+      if(row == null || ObjectStorageRuntime.getConfig().findConnection(row.getValueString("name")) == null)
+      {
+         return (HealthCheckOutcome.unknown("Storage connection is disabled or no longer configured"));
+      }
+      try
+      {
+         CapabilityProbe.Report report = CapabilityProbe.probeConnection(row.getValueString("name"), input.getDeadline());
+         HealthDetails details = new HealthDetails().withValue("bucketsProbed", report.getByBucket().size());
+         if(report.getClockSkewSeconds() != null)
+         {
+            details.withValue("clockSkewSeconds", report.getClockSkewSeconds());
+         }
+         if(report.getProblemKind() != null)
+         {
+            return (HealthCheckOutcome.down("Storage connection probe failed").withDetails(details.withValue("problemKind", report.getProblemKind())));
+         }
+         if(report.getClockSkewSeconds() != null && report.getClockSkewSeconds() > CapabilityProbe.CLOCK_SKEW_THRESHOLD_SECONDS)
+         {
+            return (HealthCheckOutcome.degraded("Storage clock skew exceeds the signing threshold").withDetails(details));
+         }
+         if(report.getByBucket().isEmpty())
+         {
+            return (HealthCheckOutcome.unknown("No storage buckets configured to probe").withDetails(details));
+         }
+         return (HealthCheckOutcome.up("Storage connection is reachable").withDetails(details));
+      }
+      catch(RuntimeException e)
+      {
+         return (HealthCheckOutcome.down("Storage connection probe failed")
+            .withDetails(new HealthDetails().withValue("problemKind", CapabilityProbe.problemKind(e))));
+      }
+   }
+}
@@ -43,6 +43,7 @@ import com.kingsrook.qbits.objectstorage.internal.ConfigSync;
 import com.kingsrook.qbits.objectstorage.internal.Db;
 import com.kingsrook.qbits.objectstorage.internal.ObjectStorageRuntime;
 import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
@@ -78,7 +79,8 @@ public final class ObjectIndex

   public static final int PAGE_SIZE = 1000;

-   private static final Set<String> RUNNING = java.util.concurrent.ConcurrentHashMap.newKeySet();
+   private static final Set<String> RUNNING = ClusterCaches.nodeLocal(java.util.concurrent.ConcurrentHashMap.newKeySet(),
+      "the buckets this node is scanning now (an in-flight guard, not cached data)");

   /** Test seam: simulate a crash after this many pages of a scan (so resume can be tested). */
   static volatile Integer failAfterPages = null;
@@ -21,7 +21,9 @@
 package com.kingsrook.qbits.objectstorage.internal;


+import javax.net.ssl.SSLException;
 import java.net.URI;
+import java.net.UnknownHostException;
 import java.net.http.HttpClient;
 import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
@@ -35,6 +37,7 @@ import java.util.EnumMap;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
+import java.util.function.Function;
 import com.kingsrook.qbits.objectstorage.api.Capability;
 import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
 import com.kingsrook.qbits.objectstorage.api.PresignedPost;
@@ -45,7 +48,9 @@ import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig;
 import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration;
 import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.DelegatingS3Client;
 import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
 import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;
@@ -61,6 +66,7 @@ import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
 import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
 import software.amazon.awssdk.services.s3.model.PutObjectRequest;
 import software.amazon.awssdk.services.s3.model.S3Exception;
+import software.amazon.awssdk.services.s3.model.S3Request;
 import software.amazon.awssdk.services.s3.model.UploadPartRequest;
 import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
 import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest;
@@ -79,6 +85,7 @@ public final class CapabilityProbe
   private static final QLogger LOG = QLogger.getLogger(CapabilityProbe.class);

   public static final String PROBE_PREFIX = ".qqq-probe/";
+   public static final int CLOCK_SKEW_THRESHOLD_SECONDS = 900;

   private static final int MIN_PART_BYTES = 5 * 1024 * 1024;

@@ -94,6 +101,17 @@ public final class CapabilityProbe
      private final List<String>                                   problems = new ArrayList<>();
      private Long                                                 clockSkewSeconds;
      private Boolean                                              presignHostReachable;
+      private String                                               problemKind;
+
+
+
+      /*******************************************************************************
+       ** Safe connection failure category, without provider messages or identifiers.
+       *******************************************************************************/
+      public String getProblemKind()
+      {
+         return (problemKind);
+      }



@@ -209,7 +227,7 @@ public final class CapabilityProbe
    *******************************************************************************/
   public static Report probeConnection(String connectionName, boolean includeWrite)
   {
-      details.clear();
+      details.get().clear();
      Report                  report     = new Report();
      ConnectionRuntime       connection = ObjectStorageRuntime.connection(connectionName);
      StorageConnectionConfig config     = connection.getConfig();
@@ -230,7 +248,7 @@ public final class CapabilityProbe
         List<String> notes = new ArrayList<>();
         report.notes.put(bucket.getAlias(), notes);

-         readProbes(connection, bucket, results, report);
+         readProbes(connection, bucket, results, report, null);
         recordVersioningState(bucket);

         if(includeWrite)
@@ -251,7 +269,7 @@ public final class CapabilityProbe
         }
         for(Map.Entry<Capability, CapabilityStatus> e : results.entrySet())
         {
-            CapabilityStore.record(bucket, e.getKey(), e.getValue(), details.getOrDefault(bucket.getAlias() + ":" + e.getKey(), null));
+            CapabilityStore.record(bucket, e.getKey(), e.getValue(), details.get().getOrDefault(bucket.getAlias() + ":" + e.getKey(), null));
         }
      }

@@ -262,7 +280,7 @@ public final class CapabilityProbe
         report.byBucket.get(writable.getAlias()).putAll(presign);
         for(Map.Entry<Capability, CapabilityStatus> e : presign.entrySet())
         {
-            CapabilityStore.record(writable, e.getKey(), e.getValue(), details.get(writable.getAlias() + ":" + e.getKey()));
+            CapabilityStore.record(writable, e.getKey(), e.getValue(), details.get().get(writable.getAlias() + ":" + e.getKey()));
         }
      }

@@ -281,7 +299,116 @@ public final class CapabilityProbe
      return (report);
   }

-   private static final Map<String, String> details = new java.util.concurrent.ConcurrentHashMap<>();
+   private static final ThreadLocal<Map<String, String>> details = ThreadLocal.withInitial(java.util.HashMap::new);
+
+
+
+   /*******************************************************************************
+    ** Reuse the read probes within a health deadline, without storing diagnostics
+    ** or running write/presign probes. Only the returned report is consumed.
+    *******************************************************************************/
+   public static Report probeConnection(String connectionName, Instant deadline)
+   {
+      try
+      {
+         return (probeReadOnly(connectionName, deadline));
+      }
+      finally
+      {
+         details.remove();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Run the shared read-only calls without updating capability or bucket rows.
+    *******************************************************************************/
+   private static Report probeReadOnly(String connectionName, Instant deadline)
+   {
+      Report report = new Report();
+      ConnectionRuntime connection = ObjectStorageRuntime.connection(connectionName);
+      for(StorageBucketConfig bucket : ObjectStorageRuntime.getConfig().getBuckets())
+      {
+         if(!connectionName.equals(bucket.getConnection()))
+         {
+            continue;
+         }
+         if(!Instant.now().isBefore(deadline) || Thread.currentThread().isInterrupted())
+         {
+            report.problemKind = "timeout";
+            break;
+         }
+         Map<Capability, CapabilityStatus> results = new EnumMap<>(Capability.class);
+         report.byBucket.put(bucket.getAlias(), results);
+         readProbes(connection, bucket, results, report, deadline);
+      }
+      if(!Instant.now().isBefore(deadline))
+      {
+         report.problemKind = "timeout";
+      }
+      return (report);
+   }
+
+
+
+   /*******************************************************************************
+    ** Classify failures using types and known protocol codes, never message text.
+    *******************************************************************************/
+   public static String problemKind(Throwable error)
+   {
+      for(Throwable cause = error; cause != null; cause = cause.getCause())
+      {
+         if(cause instanceof UnknownHostException)
+         {
+            return ("dns");
+         }
+         if(cause instanceof SSLException)
+         {
+            return ("tls");
+         }
+         String code = ErrorMapper.errorCode(cause);
+         if("RequestTimeTooSkewed".equals(code) || "RequestExpired".equals(code))
+         {
+            return ("clock");
+         }
+         if(cause instanceof S3Exception s3 && (s3.statusCode() == 401 || s3.statusCode() == 403
+            || "InvalidAccessKeyId".equals(code) || "SignatureDoesNotMatch".equals(code)))
+         {
+            return ("auth");
+         }
+      }
+      return ("connection");
+   }
+
+
+
+   /*******************************************************************************
+    ** Bound each SDK call (including retries) by the remaining probe budget.
+    *******************************************************************************/
+   private static S3Client boundedClient(S3Client client, Instant deadline)
+   {
+      return (new DelegatingS3Client(client)
+      {
+         /*******************************************************************************
+          ** Preserve credentials and other overrides while limiting the request.
+          *******************************************************************************/
+         @Override
+         @SuppressWarnings("unchecked") // toBuilder retains the concrete request type.
+         protected <T extends S3Request, ReturnT> ReturnT invokeOperation(T request, Function<T, ReturnT> operation)
+         {
+            Duration remaining = Duration.between(Instant.now(), deadline);
+            if(remaining.isNegative() || remaining.isZero() || Thread.currentThread().isInterrupted())
+            {
+               throw (new IllegalStateException("Storage probe deadline exceeded"));
+            }
+            AwsRequestOverrideConfiguration.Builder overrides = request.overrideConfiguration()
+               .map(AwsRequestOverrideConfiguration::toBuilder).orElseGet(AwsRequestOverrideConfiguration::builder);
+            T bounded = (T) request.toBuilder().overrideConfiguration(overrides.apiCallTimeout(remaining).apiCallAttemptTimeout(remaining).build()).build();
+            return (operation.apply(bounded));
+         }
+      });
+   }



@@ -291,7 +418,7 @@ public final class CapabilityProbe
    *******************************************************************************/
   private static void recordVersioningState(StorageBucketConfig bucket)
   {
-      String seen = details.get(bucket.getAlias() + ":" + Capability.VERSIONING);
+      String seen = details.get().get(bucket.getAlias() + ":" + Capability.VERSIONING);
      if(seen == null || !Db.available(ObjectStorageTables.BUCKET))
      {
         return;
@@ -317,14 +444,14 @@ public final class CapabilityProbe
   /*******************************************************************************
    ** The read-only probes of one bucket.
    *******************************************************************************/
-   private static void readProbes(ConnectionRuntime connection, StorageBucketConfig bucket, Map<Capability, CapabilityStatus> results, Report report)
+   private static void readProbes(ConnectionRuntime connection, StorageBucketConfig bucket, Map<Capability, CapabilityStatus> results, Report report, Instant deadline)
   {
-      S3Client client = connection.client();
+      S3Client client = deadline == null ? connection.client() : boundedClient(connection.client(), deadline);
      String   name   = name(bucket);
      String   alias  = bucket.getAlias();
      String   sampleKey = null;

-      try(RateLimiter.Permit ignored = connection.getRateLimiter().acquire())
+      try(RateLimiter.Permit ignored = deadline == null ? connection.getRateLimiter().acquire() : connection.getRateLimiter().acquire(deadline))
      {
         ListObjectsV2Response r = client.listObjectsV2(ListObjectsV2Request.builder().bucket(name).maxKeys(1).build());
         results.put(Capability.LIST_V2, CapabilityStatus.SUPPORTED);
@@ -352,6 +479,7 @@ public final class CapabilityProbe
      {
         results.put(Capability.LIST_V2, classify(alias, Capability.LIST_V2, e));
         report.problems.add(alias + ": listing failed (" + shortMessage(e) + ")");
+         report.problemKind = problemKind(e);
      }

      probe(results, alias, Capability.VERSIONING, () ->
@@ -359,10 +487,10 @@ public final class CapabilityProbe
         String status = client.getBucketVersioning(b -> b.bucket(name)).statusAsString();
         if(status == null && ProviderDefaults.defaultStatus(connection.getConfig().getProviderType(), Capability.VERSIONING) == CapabilityStatus.UNSUPPORTED)
         {
-            details.put(alias + ":" + Capability.VERSIONING, "the provider reports versioning off and documents it as a stub");
+            details.get().put(alias + ":" + Capability.VERSIONING, "the provider reports versioning off and documents it as a stub");
            return (CapabilityStatus.UNSUPPORTED);
         }
-         details.put(alias + ":" + Capability.VERSIONING, status == null ? "not enabled" : status);
+         details.get().put(alias + ":" + Capability.VERSIONING, status == null ? "not enabled" : status);
         return (CapabilityStatus.SUPPORTED);
      });
      probe(results, alias, Capability.OBJECT_LOCK, () ->
@@ -376,7 +504,7 @@ public final class CapabilityProbe
         {
            if("ObjectLockConfigurationNotFoundError".equals(ErrorMapper.errorCode(e)))
            {
-               details.put(alias + ":" + Capability.OBJECT_LOCK, "object lock is not enabled on this bucket");
+               details.get().put(alias + ":" + Capability.OBJECT_LOCK, "object lock is not enabled on this bucket");
               return (CapabilityStatus.UNSUPPORTED);
            }
            throw (e);
@@ -391,7 +519,7 @@ public final class CapabilityProbe
      if(key == null)
      {
         results.put(Capability.OBJECT_TAGGING, CapabilityStatus.UNKNOWN);
-         details.put(alias + ":" + Capability.OBJECT_TAGGING, "no object in the bucket to test");
+         details.get().put(alias + ":" + Capability.OBJECT_TAGGING, "no object in the bucket to test");
      }
      else
      {
@@ -505,7 +633,7 @@ public final class CapabilityProbe
            try
            {
               client.putObject(PutObjectRequest.builder().bucket(name).key(key).ifNoneMatch("*").build(), RequestBody.fromString("2"));
-               details.put(alias + ":" + Capability.CONDITIONAL_WRITE, "If-None-Match was ignored");
+               details.get().put(alias + ":" + Capability.CONDITIONAL_WRITE, "If-None-Match was ignored");
               return (CapabilityStatus.UNSUPPORTED);
            }
            catch(S3Exception e)
@@ -564,7 +692,7 @@ public final class CapabilityProbe
            report.presignHostReachable = true;
            if(r.statusCode() == 200 && !"probe".equals(r.body()))
            {
-               details.put(alias + ":" + Capability.PRESIGNED_GET, "the presigned GET returned unexpected content (does it reach the same store?)");
+               details.get().put(alias + ":" + Capability.PRESIGNED_GET, "the presigned GET returned unexpected content (does it reach the same store?)");
               return (CapabilityStatus.UNKNOWN);
            }
            return (httpStatus(alias, Capability.PRESIGNED_GET, r.statusCode(), r.body()));
@@ -618,7 +746,7 @@ public final class CapabilityProbe
      {
         return (CapabilityStatus.SUPPORTED);
      }
-      details.put(alias + ":" + capability, "HTTP " + status + (body == null ? "" : " " + body.replaceAll("\\s+", " ").substring(0, Math.min(200, body.length()))));
+      details.get().put(alias + ":" + capability, "HTTP " + status + (body == null ? "" : " " + body.replaceAll("\\s+", " ").substring(0, Math.min(200, body.length()))));
      if(status == 403)
      {
         return (CapabilityStatus.NOT_PERMITTED);
@@ -669,7 +797,7 @@ public final class CapabilityProbe
   static CapabilityStatus classify(String alias, Capability capability, Exception e)
   {
      String detail = shortMessage(e);
-      details.put(alias + ":" + capability, detail);
+      details.get().put(alias + ":" + capability, detail);
      if(e instanceof S3Exception s3)
      {
         String code = ErrorMapper.errorCode(s3);
@@ -30,6 +30,7 @@ import com.kingsrook.qbits.objectstorage.api.Capability;
 import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
 import com.kingsrook.qbits.objectstorage.config.StorageBucketConfig;
 import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;

@@ -47,7 +48,11 @@ public final class CapabilityStore
   public static final Set<Capability> CONNECTION_SCOPE = EnumSet.of(Capability.LIST_V2, Capability.PRESIGNED_GET, Capability.PRESIGNED_PUT,
      Capability.PRESIGNED_POST, Capability.USAGE_ADMIN_API, Capability.QUOTA_ADMIN_API);

-   private static final Map<String, CapabilityStatus> CACHE  = new ConcurrentHashMap<>();
+   /** Cluster cache name of the capability statuses (a probe result recorded on one node clears that key on every node). */
+   public static final String CACHE_NAME = "qbit-object-storage.capabilities";
+
+   private static final Map<String, CapabilityStatus> CACHE  = ClusterCaches.shared(CACHE_NAME, new ConcurrentHashMap<>(),
+      CapabilityStore::clearCache, CapabilityStore::clearKey);
   private static final Set<String>                   LOADED = ConcurrentHashMap.newKeySet();


@@ -62,7 +67,7 @@ public final class CapabilityStore


   /*******************************************************************************
-    ** Drop everything cached.
+    ** Drop everything cached on this node.
    *******************************************************************************/
   public static void clearCache()
   {
@@ -72,6 +77,28 @@ public final class CapabilityStore



+   /*******************************************************************************
+    ** Drop one cached status on this node; the next read reloads the missing
+    ** statuses from the capability rows (cached ones are kept).
+    *******************************************************************************/
+   static void clearKey(String key)
+   {
+      CACHE.remove(key);
+      LOADED.clear();
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether a status is cached on this node (tests).
+    *******************************************************************************/
+   static boolean isCached(StorageBucketConfig bucket, Capability capability)
+   {
+      return (CACHE.containsKey(scopeKey(bucket, capability)));
+   }
+
+
+
   /*******************************************************************************
    ** The status of a capability for a bucket alias.
    *******************************************************************************/
@@ -112,10 +139,24 @@ public final class CapabilityStore

   /*******************************************************************************
    ** Record one probe result (cache and, when the tables are there, the row).
+    ** The row is written first, then the key is cleared on every node (they
+    ** reload it from the row), then cached here.
    *******************************************************************************/
   public static void record(StorageBucketConfig bucket, Capability capability, CapabilityStatus status, String detail)
   {
-      CACHE.put(scopeKey(bucket, capability), status);
+      String key = scopeKey(bucket, capability);
+      persist(bucket, capability, status, detail);
+      ClusterCaches.invalidateKey(CACHE_NAME, key);
+      CACHE.put(key, status);
+   }
+
+
+
+   /*******************************************************************************
+    ** Write one probe result to its capability row; never throws.
+    *******************************************************************************/
+   private static void persist(StorageBucketConfig bucket, Capability capability, CapabilityStatus status, String detail)
+   {
      if(!Db.available(ObjectStorageTables.BUCKET_CAPABILITY))
      {
         return;
@@ -0,0 +1,95 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.objectstorage.internal;
+
+
+import java.util.List;
+import java.util.Optional;
+import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** Post insert, update and delete of the editable config rows that the runtime
+ ** caches: a bucket row (presign limits, ConfigSync) or a header rule
+ ** (DefaultObjectHeaderPolicy). Clears that cache on every node, so an admin's
+ ** edit takes effect everywhere at once; the short TTLs remain the safety net.
+ *******************************************************************************/
+public class ConfigRowChangeCustomizer implements TableCustomizerInterface
+{
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
+   {
+      invalidate(insertInput.getTableName());
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      invalidate(updateInput.getTableName());
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    ** {@inheritDoc}
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postDelete(DeleteInput deleteInput, List<QRecord> records) throws QException
+   {
+      invalidate(deleteInput.getTableName());
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    ** Clear the cache that holds the table's rows, on every node.
+    *******************************************************************************/
+   static void invalidate(String tableName)
+   {
+      if(ObjectStorageTables.BUCKET.equals(tableName))
+      {
+         ClusterCaches.invalidate(ConfigSync.ROW_LIMITS_CACHE_NAME);
+      }
+      else if(ObjectStorageTables.HEADER_RULE.equals(tableName))
+      {
+         ClusterCaches.invalidate(DefaultObjectHeaderPolicy.CACHE_NAME);
+      }
+   }
+}
@@ -30,6 +30,7 @@ import com.kingsrook.qbits.objectstorage.config.CredentialsMode;
 import com.kingsrook.qbits.objectstorage.config.StorageBucketConfig;
 import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig;
 import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -48,10 +49,16 @@ public final class ConfigSync
 {
   private static final QLogger LOG = QLogger.getLogger(ConfigSync.class);

-   private static final Map<String, Integer> BUCKET_IDS     = new ConcurrentHashMap<>();
-   private static final Map<String, Integer> CONNECTION_IDS = new ConcurrentHashMap<>();
+   /** Cluster cache name of the bucket rows' presign limits (cleared on every node when a bucket row changes). */
+   public static final String ROW_LIMITS_CACHE_NAME = "qbit-object-storage.bucket-row-limits";

-   private static final Map<String, RowLimits> ROW_LIMITS = new ConcurrentHashMap<>();
+   private static final Map<String, Integer> BUCKET_IDS     = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
+      "bucket alias to row id, written by the config sync; bucket rows cannot be inserted or deleted through the UI or API, so the ids are the same on every node");
+   private static final Map<String, Integer> CONNECTION_IDS = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
+      "connection name to row id, written by the config sync; connection rows cannot be inserted or deleted through the UI or API, so the ids are the same on every node");
+
+   private static final Map<String, RowLimits> ROW_LIMITS = ClusterCaches.shared(ROW_LIMITS_CACHE_NAME, new ConcurrentHashMap<>(),
+      ConfigSync::clearRowLimits, alias -> ConfigSync.ROW_LIMITS.remove(alias));
   private static final long                 ROW_LIMIT_CACHE_MILLIS = 15_000L;

   private static volatile boolean synced = false;
@@ -313,7 +320,8 @@ public final class ConfigSync


   /*******************************************************************************
-    ** Forget the cached row limits (after a row edit in a test; they also expire).
+    ** Forget the cached row limits on this node (a bucket row edit clears them on
+    ** every node through ConfigRowChangeCustomizer; they also expire).
    *******************************************************************************/
   public static void clearRowLimits()
   {
@@ -22,7 +22,6 @@ package com.kingsrook.qbits.objectstorage.internal;


 import java.net.URI;
-import java.util.function.Function;
 import com.kingsrook.qbits.objectstorage.config.AddressingStyle;
 import com.kingsrook.qbits.objectstorage.config.CredentialsMode;
 import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig;
@@ -30,17 +29,10 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3ChecksumMode;
 import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3ClientFactory;
-import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
 import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
-import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
-import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration;
 import software.amazon.awssdk.regions.Region;
-import software.amazon.awssdk.services.s3.DelegatingS3Client;
 import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.S3Configuration;
-import software.amazon.awssdk.services.s3.model.S3Request;
 import software.amazon.awssdk.services.s3.presigner.S3Presigner;


@@ -198,16 +190,7 @@ public class ConnectionRuntime
    *******************************************************************************/
   private S3Client buildClient()
   {
-      S3BackendMetaData backend = backendMetaData();
-      String token = config.getCredentialsMode() == CredentialsMode.STATIC_REFS ? SecretRefs.resolve(config.getSessionTokenRef()) : null;
-      AwsCredentialsProvider sessionCredentials = null;
-      if(token != null && backend.getAccessKey() != null && backend.getSecretKey() != null)
-      {
-         sessionCredentials = StaticCredentialsProvider.create(AwsSessionCredentials.create(backend.getAccessKey(), backend.getSecretKey(), token));
-      }
-
-      S3Client coreClient = S3ClientFactory.buildClient(backend);
-      return (sessionCredentials == null ? coreClient : new SessionCredentialsClient(coreClient, sessionCredentials));
+      return (S3ClientFactory.buildClient(backendMetaData()));
   }


@@ -229,8 +212,7 @@ public class ConnectionRuntime
         .withSocketTimeoutMillis(config.getReadTimeoutMs());
      if(config.getCredentialsMode() == CredentialsMode.STATIC_REFS)
      {
-         backend.withAccessKey(SecretRefs.resolve(config.getAccessKeyRef()))
-            .withSecretKey(SecretRefs.resolve(config.getSecretKeyRef()));
+         withStaticCredentials(backend, config.getAccessKeyRef(), config.getSecretKeyRef());
      }
      return (backend);
   }
@@ -270,63 +252,33 @@ public class ConnectionRuntime


   /*******************************************************************************
-    ** Static keys from the references, or the SDK default chain.
+    ** Static keys (with the session token, when set) from the references, or the
+    ** SDK default chain; core builds the provider.
    *******************************************************************************/
   private AwsCredentialsProvider credentials(String accessRef, String secretRef)
   {
+      S3BackendMetaData credentials = new S3BackendMetaData();
      if(config.getCredentialsMode() == CredentialsMode.STATIC_REFS)
      {
-         String accessKey = SecretRefs.resolve(accessRef);
-         String secretKey = SecretRefs.resolve(secretRef);
-         if(accessKey != null && secretKey != null)
-         {
-            String token = SecretRefs.resolve(config.getSessionTokenRef());
-            return (StaticCredentialsProvider.create(token == null ? AwsBasicCredentials.create(accessKey, secretKey)
-               : AwsSessionCredentials.create(accessKey, secretKey, token)));
-         }
+         withStaticCredentials(credentials, accessRef, secretRef);
      }
-      return (DefaultCredentialsProvider.builder().build());
+      return (S3ClientFactory.buildCredentialsProvider(credentials));
   }



   /*******************************************************************************
-    ** STAND-IN(core=QRun-IO/qqq#1165, removeBy=4.1.0-RC.8)
-    ** RC.7 S3BackendMetaData and S3ClientFactory still lack a session token or
-    ** credentials-provider hook (verified against the released JAR).
-    ** Override request credentials only; core still owns the client and transport.
+    ** Resolve static keys onto core meta data. The session token applies only
+    ** with both keys, as before (core rejects a token without keys).
    *******************************************************************************/
-   private static class SessionCredentialsClient extends DelegatingS3Client
+   private void withStaticCredentials(S3BackendMetaData backend, String accessRef, String secretRef)
   {
-      private final AwsCredentialsProvider credentialsProvider;
-
-
-
-      /*******************************************************************************
-       ** Wrap a core-built client with the connection's session credentials.
-       *******************************************************************************/
-      private SessionCredentialsClient(S3Client delegate, AwsCredentialsProvider credentialsProvider)
+      String accessKey = SecretRefs.resolve(accessRef);
+      String secretKey = SecretRefs.resolve(secretRef);
+      backend.withAccessKey(accessKey).withSecretKey(secretKey);
+      if(accessKey != null && secretKey != null)
      {
-         super(delegate);
-         this.credentialsProvider = credentialsProvider;
-      }
-
-
-
-      /*******************************************************************************
-       ** Preserve request overrides and the concrete request type while adding the
-       ** session token to signing, including streaming and paginated operations.
-       *******************************************************************************/
-      @Override
-      @SuppressWarnings("unchecked") // toBuilder().build() retains the concrete request type T.
-      protected <T extends S3Request, ReturnT> ReturnT invokeOperation(T request, Function<T, ReturnT> operation)
-      {
-         AwsRequestOverrideConfiguration.Builder overrides = request.overrideConfiguration()
-            .map(AwsRequestOverrideConfiguration::toBuilder).orElseGet(AwsRequestOverrideConfiguration::builder);
-         T signedRequest = (T) request.toBuilder()
-            .overrideConfiguration(overrides.credentialsProvider(credentialsProvider).build())
-            .build();
-         return (operation.apply(signedRequest));
+         backend.withSessionToken(SecretRefs.resolve(config.getSessionTokenRef()));
      }
   }
 }
@@ -23,12 +23,14 @@ package com.kingsrook.qbits.objectstorage.internal;

 import java.util.Comparator;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.regex.Pattern;
 import com.kingsrook.qbits.objectstorage.api.ObjectHeaderPolicy;
 import com.kingsrook.qbits.objectstorage.api.ObjectHeaders;
 import com.kingsrook.qbits.objectstorage.config.BucketRole;
 import com.kingsrook.qbits.objectstorage.config.StorageBucketConfig;
 import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -44,19 +46,31 @@ public class DefaultObjectHeaderPolicy implements ObjectHeaderPolicy
 {
   private static final QLogger LOG = QLogger.getLogger(DefaultObjectHeaderPolicy.class);

+   /** Cluster cache name of the enabled header rules (cleared on every node when a rule changes). */
+   public static final String CACHE_NAME = "qbit-object-storage.header-rules";
+
   private static final long CACHE_MILLIS = 15_000L;

-   private static volatile List<QRecord> cachedRules;
-   private static volatile long          cachedAt;
+   private static final AtomicReference<CachedRules> CACHED = ClusterCaches.shared(CACHE_NAME, new AtomicReference<>(), DefaultObjectHeaderPolicy::clearCache, null);



   /*******************************************************************************
-    ** Drop cached rules (after the rules change).
+    ** The enabled rules and when they were read.
+    *******************************************************************************/
+   private record CachedRules(List<QRecord> rules, long readAt)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Drop the cached rules on this node (a rule edit clears them on every node
+    ** through ConfigRowChangeCustomizer).
    *******************************************************************************/
   public static void clearCache()
   {
-      cachedRules = null;
+      CACHED.set(null);
   }


@@ -119,11 +133,12 @@ public class DefaultObjectHeaderPolicy implements ObjectHeaderPolicy
    *******************************************************************************/
   private static List<QRecord> rules()
   {
-      List<QRecord> rules = cachedRules;
-      if(rules != null && System.currentTimeMillis() - cachedAt < CACHE_MILLIS)
+      CachedRules cached = CACHED.get();
+      if(cached != null && System.currentTimeMillis() - cached.readAt() < CACHE_MILLIS)
      {
-         return (rules);
+         return (cached.rules());
      }
+      List<QRecord> rules;
      if(!Db.available(ObjectStorageTables.HEADER_RULE))
      {
         return (List.of());
@@ -133,8 +148,7 @@ public class DefaultObjectHeaderPolicy implements ObjectHeaderPolicy
         rules = Db.asSystem(() -> Db.query(ObjectStorageTables.HEADER_RULE, "enabled", true));
         rules.sort(Comparator.comparing((QRecord r) -> r.getValueInteger("priority") == null ? 100 : r.getValueInteger("priority"))
            .thenComparing(r -> r.getValueInteger("id") == null ? 0 : r.getValueInteger("id")));
-         cachedRules = rules;
-         cachedAt = System.currentTimeMillis();
+         CACHED.set(new CachedRules(rules, System.currentTimeMillis()));
         return (rules);
      }
      catch(Exception e)
@@ -33,6 +33,7 @@ import com.kingsrook.qbits.objectstorage.api.StorageAlertSink;
 import com.kingsrook.qbits.objectstorage.api.StorageChangeListener;
 import com.kingsrook.qbits.objectstorage.config.StorageBucketConfig;
 import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;


 /*******************************************************************************
@@ -43,9 +44,11 @@ import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig;
 *******************************************************************************/
 public final class ObjectStorageRuntime
 {
-   private static final Map<String, ConnectionRuntime>   CONNECTIONS = new ConcurrentHashMap<>();
+   private static final Map<String, ConnectionRuntime>   CONNECTIONS = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
+      "this node's S3 clients, built from the code config at first use; row overrides apply when a client is built, the same on every node");
   private static final List<StorageChangeListener>      LISTENERS   = new CopyOnWriteArrayList<>();
-   private static final Map<String, String>              PROCESS_PERMISSIONS = new ConcurrentHashMap<>();
+   private static final Map<String, String>              PROCESS_PERMISSIONS = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
+      "built from the qbit config at startup, the same on every node");

   private static volatile ObjectStorageQBitConfig config;
   private static volatile ObjectHeaderPolicy      headerPolicy;
@@ -26,6 +26,7 @@ import java.util.Map;
 import com.kingsrook.qbits.objectstorage.api.Capability;
 import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
 import com.kingsrook.qbits.objectstorage.config.ProviderType;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
 import static com.kingsrook.qbits.objectstorage.api.CapabilityStatus.PARTIAL;
 import static com.kingsrook.qbits.objectstorage.api.CapabilityStatus.SUPPORTED;
 import static com.kingsrook.qbits.objectstorage.api.CapabilityStatus.UNKNOWN;
@@ -39,7 +40,8 @@ import static com.kingsrook.qbits.objectstorage.api.CapabilityStatus.UNSUPPORTED
 *******************************************************************************/
 public final class ProviderDefaults
 {
-   private static final Map<ProviderType, Map<Capability, CapabilityStatus>> TABLE = new EnumMap<>(ProviderType.class);
+   private static final Map<ProviderType, Map<Capability, CapabilityStatus>> TABLE = ClusterCaches.nodeLocal(new EnumMap<>(ProviderType.class),
+      "provider defaults from code, the same on every node");

   static
   {
@@ -21,7 +21,10 @@
 package com.kingsrook.qbits.objectstorage.internal;


+import java.time.Duration;
+import java.time.Instant;
 import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;


 /*******************************************************************************
@@ -57,17 +60,52 @@ public class RateLimiter
    *******************************************************************************/
   public Permit acquire()
   {
+      return (acquire(null));
+   }
+
+
+
+   /*******************************************************************************
+    ** Wait no later than a caller's deadline, releasing the slot on interruption.
+    *******************************************************************************/
+   public Permit acquire(Instant deadline)
+   {
+      boolean acquired = false;
      try
      {
-         concurrency.acquire();
-         waitForToken();
+         if(deadline == null)
+         {
+            concurrency.acquire();
+            acquired = true;
+         }
+         else
+         {
+            acquired = concurrency.tryAcquire(Math.max(0, Duration.between(Instant.now(), deadline).toNanos()), TimeUnit.NANOSECONDS);
+            if(!acquired)
+            {
+               throw (new IllegalStateException("Storage probe deadline exceeded"));
+            }
+         }
+         waitForToken(deadline);
         return (new Permit(concurrency));
      }
      catch(InterruptedException e)
      {
+         if(acquired)
+         {
+            concurrency.release();
+         }
         Thread.currentThread().interrupt();
         throw (new IllegalStateException("Interrupted while waiting for the storage rate limiter", e));
      }
+      catch(RuntimeException e)
+      {
+         if(acquired)
+         {
+            concurrency.release();
+         }
+         throw (e);
+      }
   }


@@ -75,10 +113,14 @@ public class RateLimiter
   /*******************************************************************************
    ** Block until a token is available.
    *******************************************************************************/
-   private void waitForToken() throws InterruptedException
+   private void waitForToken(Instant deadline) throws InterruptedException
   {
      while(true)
      {
+         if(deadline != null && !Instant.now().isBefore(deadline))
+         {
+            throw (new IllegalStateException("Storage probe deadline exceeded"));
+         }
         long sleepMillis;
         synchronized(this)
         {
@@ -93,7 +135,7 @@ public class RateLimiter
            throttledWaits++;
            sleepMillis = Math.max(1, (long) ((1.0 - tokens) / opsPerSecond * 1000));
         }
-         Thread.sleep(sleepMillis);
+         Thread.sleep(deadline == null ? sleepMillis : Math.min(sleepMillis, Math.max(1, Duration.between(Instant.now(), deadline).toMillis())));
      }
   }

@@ -21,10 +21,6 @@
 package com.kingsrook.qbits.objectstorage.internal;


-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
 import java.util.List;
 import java.util.regex.Pattern;
 import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
@@ -32,14 +28,12 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;


 /*******************************************************************************
- ** Resolves secret references (${env.X}, ${prop.X}, ${file.PATH}) and enforces
- ** the allow-list. Resolved values live only in memory, never in a record, a log
- ** line or a process value.
- **
- ** STAND-IN(core=QRun-IO/qqq#1165, removeBy=4.1.0-RC.8)
- ** RC.7 QMetaDataVariableInterpreter still supports env/prop, not file references.
- ** ${file.PATH} moves to QMetaDataVariableInterpreter in QQQ core
- ** (spec section 8.2); then this class delegates entirely.
+ ** Enforces the qbit's secret-reference allow-list and resolves references
+ ** (${env.X}, ${prop.X}, ${file.PATH}) through core QMetaDataVariableInterpreter.
+ ** Core reads ${file.PATH} read-only from its allowed roots (default
+ ** /var/run/secrets and /run/secrets; qqq.fileReference.roots or
+ ** QQQ_FILE_REFERENCE_ROOTS). Resolved values live only in memory, never in a
+ ** record, a log line or a process value.
 *******************************************************************************/
 public final class SecretRefs
 {
@@ -93,19 +87,7 @@ public final class SecretRefs
      {
         return (null);
      }
-      String trimmed = ref.trim();
-      if(trimmed.startsWith("${file.") && trimmed.endsWith("}"))
-      {
-         try
-         {
-            return (Files.readString(Path.of(trimmed.substring("${file.".length(), trimmed.length() - 1)), StandardCharsets.UTF_8).trim());
-         }
-         catch(IOException e)
-         {
-            return (null);
-         }
-      }
-      String value = new QMetaDataVariableInterpreter().interpret(trimmed);
+      String value = new QMetaDataVariableInterpreter().interpret(ref.trim());
      return (value == null || value.startsWith("${") ? null : value);
   }

@@ -25,6 +25,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import com.kingsrook.qbits.objectstorage.ObjectStorageQBitConfig;
+import com.kingsrook.qbits.objectstorage.internal.ConfigRowChangeCustomizer;
 import com.kingsrook.qbits.objectstorage.permissions.ObjectStoragePermissionChecker;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.ChildRecordListRenderer;
@@ -564,7 +565,9 @@ public final class ObjectStorageTables
         .withField(ro("orphanedCount", QFieldType.LONG))
         .withField(ro("orphanedBytes", QFieldType.LONG))
         .withField(ro("missingCount", QFieldType.LONG))
-         .withField(ro("lastReferenceScanDate", QFieldType.DATE_TIME)), KEY_TYPE_BUCKET, "id", false));
+         .withField(ro("lastReferenceScanDate", QFieldType.DATE_TIME)), KEY_TYPE_BUCKET, "id", false)
+         .withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(ConfigRowChangeCustomizer.class))
+         .withCustomizer(TableCustomizers.POST_UPDATE_RECORD, new QCodeReference(ConfigRowChangeCustomizer.class)));
   }


@@ -701,7 +704,10 @@ public final class ObjectStorageTables
         .withField(str("cacheControl", 300))
         .withField(str("contentDisposition", 300))
         .withField(new QFieldMetaData("priority", QFieldType.INTEGER).withDefaultValue(100))
-         .withField(new QFieldMetaData("enabled", QFieldType.BOOLEAN).withDefaultValue(true)), KEY_TYPE_BUCKET, "bucketId", true));
+         .withField(new QFieldMetaData("enabled", QFieldType.BOOLEAN).withDefaultValue(true)), KEY_TYPE_BUCKET, "bucketId", true)
+         .withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(ConfigRowChangeCustomizer.class))
+         .withCustomizer(TableCustomizers.POST_UPDATE_RECORD, new QCodeReference(ConfigRowChangeCustomizer.class))
+         .withCustomizer(TableCustomizers.POST_DELETE_RECORD, new QCodeReference(ConfigRowChangeCustomizer.class)));
   }


@@ -227,7 +227,7 @@ public final class TestConnectionProcess
            }
            if(report.getClockSkewSeconds() != null)
            {
-               boolean skewed = report.getClockSkewSeconds() > 900;
+               boolean skewed = report.getClockSkewSeconds() > CapabilityProbe.CLOCK_SKEW_THRESHOLD_SECONDS;
               ok = ok && !skewed;
               lines.add(new ProcessSummaryLine(skewed ? Status.ERROR : Status.INFO, "Clock skew against the store: " + report.getClockSkewSeconds() + " seconds"
                  + (skewed ? " (SigV4 fails beyond 15 minutes)" : "")));
@@ -1,5 +1,5 @@
 bundle: qbit-object-storage
-version: 0.1.0-SNAPSHOT
+version: ${project.version}
 layer: QBIT
 entries:
 - key: app:objectStorageAdmin;slot:help
@@ -0,0 +1 @@
+version=${project.version}
@@ -59,7 +59,13 @@ class ObjectStorageHelpBundleTest

      assertThat(instance.getTable("helpContent")).isNull();
      assertThat(HelpBundles.of(instance).getBundles()).singleElement()
-         .satisfies(bundle -> assertThat(bundle.getId()).isEqualTo("qbit-object-storage"));
+         .satisfies(bundle ->
+         {
+            assertThat(bundle.getId()).isEqualTo("qbit-object-storage");
+            assertThat(System.getProperty("qbit.artifactVersion")).isNotBlank();
+            assertThat(bundle.getVersion()).isEqualTo(System.getProperty("qbit.artifactVersion"))
+               .isEqualTo(ObjectStorageQBitProducer.VERSION);
+         });
      HelpBundles.syncOnStartup(instance);
      Set<String> keys = HelpBundles.of(instance).getBundles().get(0).getEntries().stream()
         .map(entry -> entry.getKey()).collect(Collectors.toSet());
@@ -44,6 +44,10 @@ class ObjectStorageQBitProducerTest
      ObjectStorageQBitConfig config    = new ObjectStorageQBitConfig().withBackendName("rdbms");
      ObjectStorageQBitProducer producer  = new ObjectStorageQBitProducer().withQBitConfig(config).withNamespace("object_storage");

+      String artifactVersion = System.getProperty("qbit.artifactVersion");
+      assertThat(artifactVersion).isNotBlank();
+      assertThat(ObjectStorageQBitProducer.VERSION).isEqualTo(artifactVersion);
+      assertThat(producer.getQBitMetaData().getVersion()).isEqualTo(artifactVersion);
      assertThat(producer.isEnabled()).isTrue();
      assertThat(producer.getQBitConfig()).isSameAs(config);
      assertThat(producer.getNamespace()).isEqualTo("object_storage");
@@ -0,0 +1,364 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.objectstorage.health;
+
+
+import javax.net.ssl.SSLHandshakeException;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Proxy;
+import java.net.UnknownHostException;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qbits.objectstorage.ObjectStorageQBitConfig;
+import com.kingsrook.qbits.objectstorage.config.StorageBucketConfig;
+import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig;
+import com.kingsrook.qbits.objectstorage.internal.CapabilityProbe;
+import com.kingsrook.qbits.objectstorage.internal.ConfigSync;
+import com.kingsrook.qbits.objectstorage.internal.ConnectionRuntime;
+import com.kingsrook.qbits.objectstorage.internal.Db;
+import com.kingsrook.qbits.objectstorage.internal.ObjectStorageRuntime;
+import com.kingsrook.qbits.objectstorage.internal.RateLimiter;
+import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
+import com.kingsrook.qbits.objectstorage.processes.TestConnectionProcess;
+import com.kingsrook.qbits.objectstorage.testsupport.StorageTestSupport;
+import com.kingsrook.qqq.backend.core.health.HealthCategory;
+import com.kingsrook.qqq.backend.core.health.HealthCheckInput;
+import com.kingsrook.qqq.backend.core.health.HealthCheckKind;
+import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData;
+import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome;
+import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData;
+import com.kingsrook.qqq.backend.core.health.HealthStatus;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import software.amazon.awssdk.awscore.exception.AwsErrorDetails;
+import software.amazon.awssdk.http.SdkHttpResponse;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+import software.amazon.awssdk.services.s3.model.S3Request;
+import software.amazon.awssdk.utils.builder.SdkBuilder;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+
+/*******************************************************************************
+ ** Exercise the real probe using an in-memory S3 client and QQQ record store.
+ *******************************************************************************/
+class ObjectStorageConnectionChecksTest
+{
+   private static final String SECRET = "S3NT1NEL-secret";
+   private QInstance instance;
+   private ObjectStorageQBitConfig config;
+   private final List<String> calls = new ArrayList<>();
+   private RuntimeException failure;
+   private long skew;
+   private boolean expireAfterList;
+   private Instant deadline;
+
+
+
+   /*******************************************************************************
+    ** Poison every credential reference; the fake prevents all network access.
+    *******************************************************************************/
+   @BeforeEach
+   void setUp() throws Exception
+   {
+      System.setProperty("os.health.secret", SECRET);
+      String ref = "${prop.os.health.secret}";
+      config = new ObjectStorageQBitConfig().withBackendName(StorageTestSupport.MEMORY)
+         .withAllowedSecretRefPatterns(List.of("prop.os.health.*"))
+         .withConnection(new StorageConnectionConfig("PRIMARY").withEndpoint("https://user:" + SECRET + "@private.example")
+            .withAccessKeyRef(ref).withSecretKeyRef(ref).withSessionTokenRef(ref)
+            .withPresignAccessKeyRef(ref).withPresignSecretKeyRef(ref).withCaBundleRef(ref).withMaxConcurrency(1))
+         .withBucket(new StorageBucketConfig("files").withConnection("PRIMARY").withBucketName("private-bucket").withIndexed(false));
+      instance = StorageTestSupport.setUp(config);
+      S3Client client = (S3Client) Proxy.newProxyInstance(S3Client.class.getClassLoader(), new Class<?>[] { S3Client.class }, (proxy, method, args) ->
+      {
+         if(method.getName().equals("close"))
+         {
+            return (null);
+         }
+         if(args != null && args.length == 1 && args[0] instanceof java.util.function.Consumer<?>)
+         {
+            return (InvocationHandler.invokeDefault(proxy, method, args));
+         }
+         calls.add(method.getName());
+         assertThat(method.getName()).matches("listObjectsV2|getBucket.*|getObjectLockConfiguration|getObjectTagging");
+         S3Request request = (S3Request) args[0];
+         Duration timeout = request.overrideConfiguration().orElseThrow().apiCallTimeout().orElseThrow();
+         assertThat(timeout).isPositive().isLessThanOrEqualTo(Duration.ofSeconds(20));
+         assertThat(request.overrideConfiguration().orElseThrow().apiCallAttemptTimeout()).contains(timeout);
+         if(failure != null)
+         {
+            throw (failure);
+         }
+         if(method.getName().equals("listObjectsV2"))
+         {
+            if(expireAfterList)
+            {
+               Thread.sleep(Math.max(1, Duration.between(Instant.now(), deadline).toMillis() + 10));
+            }
+            return (ListObjectsV2Response.builder().sdkHttpResponse(SdkHttpResponse.builder().statusCode(200)
+               .putHeader("Date", DateTimeFormatter.RFC_1123_DATE_TIME.format(Instant.now().minusSeconds(skew).atZone(ZoneOffset.UTC))).build()).build());
+         }
+         return (((SdkBuilder<?, ?>) method.getReturnType().getMethod("builder").invoke(null)).build());
+      });
+      Field field = ConnectionRuntime.class.getDeclaredField("client");
+      field.setAccessible(true);
+      field.set(ObjectStorageRuntime.connection("PRIMARY"), client);
+   }
+
+
+
+   /*******************************************************************************
+    ** Restore process-wide fixtures and the poisoned property.
+    *******************************************************************************/
+   @AfterEach
+   void tearDown()
+   {
+      StorageTestSupport.tearDown();
+      System.clearProperty("os.health.secret");
+   }
+
+
+
+   /*******************************************************************************
+    ** Producer registration includes the required metadata and keeps both actions.
+    *******************************************************************************/
+   @Test
+   void registrationAndRefresh() throws Exception
+   {
+      var registry = HealthChecksMetaData.ofOrWithNew(instance);
+      assertThat(registry.getSuppliers()).singleElement().satisfies(supplier ->
+      {
+         assertThat(supplier.getNamePrefix()).isEqualTo("object_storage.connection");
+         assertThat(supplier.getComponent()).isEqualTo("object_storage");
+      });
+      HealthCheckMetaData check = check();
+      assertThat(check.getName()).isEqualTo("object_storage.connection." + ConfigSync.connectionId("PRIMARY")).matches("[a-z0-9]+([._-][a-z0-9]+)*");
+      assertThat(check.getKind()).isEqualTo(HealthCheckKind.DEPENDENCY);
+      assertThat(check.getCategory()).isEqualTo(HealthCategory.STORAGE);
+      assertThat(check.getComponent()).isEqualTo("object_storage");
+      assertThat(check.getIntervalSeconds()).isEqualTo(300);
+      assertThat(check.getTimeoutMs()).isEqualTo(20000);
+      assertThat(instance.getProcess(TestConnectionProcess.READ_NAME)).isNotNull();
+      assertThat(instance.getProcess(TestConnectionProcess.WRITE_NAME)).isNotNull();
+      ConfigSync.sync();
+      assertThat(registry.resolveAll(instance)).hasSize(1);
+      config.withConnection(new StorageConnectionConfig("SECOND"));
+      ConfigSync.sync();
+      assertThat(registry.resolveAll(instance)).hasSize(2);
+      config.getConnections().clear();
+      assertThat(registry.resolveAll(instance)).isEmpty();
+      assertClean(check.getCheckInstance().run(input()), HealthStatus.UNKNOWN);
+   }
+
+
+
+   /*******************************************************************************
+    ** Successful checks read buckets without persisting any capability diagnostics.
+    *******************************************************************************/
+   @Test
+   void upReadOnlyAndNoSecrets() throws Exception
+   {
+      HealthCheckOutcome outcome = run();
+      assertClean(outcome, HealthStatus.UP);
+      assertThat(outcome.getDetails().getValues()).containsEntry("bucketsProbed", 1);
+      assertThat(calls).contains("listObjectsV2", "getBucketVersioning", "getBucketAcl");
+      assertThat(Db.asSystem(() -> Db.query(ObjectStorageTables.CONNECTION_CAPABILITY))).isEmpty();
+      assertThat(Db.asSystem(() -> Db.query(ObjectStorageTables.BUCKET_CAPABILITY))).isEmpty();
+      assertThat(Db.asSystem(() -> Db.first(ObjectStorageTables.CONNECTION, "name", "PRIMARY")).getValue("lastTestedAt")).isNull();
+   }
+
+
+
+   /*******************************************************************************
+    ** Clock skew uses the same threshold as the record action.
+    *******************************************************************************/
+   @Test
+   void clockSkew() throws Exception
+   {
+      skew = CapabilityProbe.CLOCK_SKEW_THRESHOLD_SECONDS - 2;
+      assertClean(run(), HealthStatus.UP);
+      skew = CapabilityProbe.CLOCK_SKEW_THRESHOLD_SECONDS + 10;
+      HealthCheckOutcome outcome = run();
+      assertClean(outcome, HealthStatus.DEGRADED);
+      assertThat((Long) outcome.getDetails().getValues().get("clockSkewSeconds")).isGreaterThan(900);
+   }
+
+
+
+   /*******************************************************************************
+    ** Expected failures carry only a fixed kind even when errors contain secrets.
+    *******************************************************************************/
+   @ParameterizedTest
+   @ValueSource(strings = { "auth", "tls", "dns", "clock", "connection" })
+   void downAndNoSecrets(String kind) throws Exception
+   {
+      failure = switch(kind)
+      {
+         case "auth" -> (S3Exception) S3Exception.builder().statusCode(403).message(SECRET).build();
+         case "tls" -> new IllegalStateException(SECRET, new SSLHandshakeException(SECRET));
+         case "dns" -> new IllegalStateException(SECRET, new UnknownHostException(SECRET));
+         case "clock" -> (S3Exception) S3Exception.builder().statusCode(403).message(SECRET)
+            .awsErrorDetails(AwsErrorDetails.builder().errorCode("RequestTimeTooSkewed").build()).build();
+         default -> new IllegalStateException(SECRET);
+      };
+      skew = 1000;
+      HealthCheckOutcome outcome = run();
+      assertClean(outcome, HealthStatus.DOWN);
+      assertThat(outcome.getDetails().getValues()).containsEntry("problemKind", kind);
+      assertThat(Db.asSystem(() -> Db.query(ObjectStorageTables.CONNECTION_CAPABILITY))).isEmpty();
+   }
+
+
+
+   /*******************************************************************************
+    ** An empty configuration cannot establish connectivity.
+    *******************************************************************************/
+   @Test
+   void noBuckets() throws Exception
+   {
+      config.getBuckets().clear();
+      assertClean(run(), HealthStatus.UNKNOWN);
+      assertThat(calls).isEmpty();
+   }
+
+
+
+   /*******************************************************************************
+    ** Client initialization errors also cannot expose poisoned configuration.
+    *******************************************************************************/
+   @Test
+   @SuppressWarnings("unchecked")
+   void clientInitializationFailure() throws Exception
+   {
+      Field connections = ObjectStorageRuntime.class.getDeclaredField("CONNECTIONS");
+      connections.setAccessible(true);
+      ((Map<String, ConnectionRuntime>) connections.get(null)).put("PRIMARY", new ConnectionRuntime(config.getConnections().getFirst())
+      {
+         /*******************************************************************************
+          ** Simulate a credential or endpoint failure before any request is sent.
+          *******************************************************************************/
+         @Override
+         public synchronized S3Client client()
+         {
+            throw (new IllegalStateException(SECRET));
+         }
+      });
+      assertClean(run(), HealthStatus.DOWN);
+      assertThat(calls).isEmpty();
+   }
+
+
+
+   /*******************************************************************************
+    ** Expired checks perform no I/O; a deadline reached mid-probe stops SDK calls.
+    *******************************************************************************/
+   @Test
+   void deadlines() throws Exception
+   {
+      assertClean(check().getCheckInstance().run(input().withDeadline(Instant.now().minusSeconds(1))), HealthStatus.DOWN);
+      assertThat(calls).isEmpty();
+      expireAfterList = true;
+      deadline = Instant.now().plusMillis(500);
+      HealthCheckOutcome outcome = check().getCheckInstance().run(input().withDeadline(deadline));
+      assertClean(outcome, HealthStatus.DOWN);
+      assertThat(outcome.getDetails().getValues()).containsEntry("problemKind", "timeout");
+      assertThat(calls).containsExactly("listObjectsV2");
+   }
+
+
+
+   /*******************************************************************************
+    ** Rate limiting also respects the budget, both for slots and depleted tokens.
+    *******************************************************************************/
+   @Test
+   void rateLimiterDeadline() throws Exception
+   {
+      RateLimiter limiter = ObjectStorageRuntime.connection("PRIMARY").getRateLimiter();
+      try(var held = limiter.acquire())
+      {
+         assertClean(check().getCheckInstance().run(input().withDeadline(Instant.now().plusMillis(50))), HealthStatus.DOWN);
+         assertThat(calls).isEmpty();
+      }
+      RateLimiter onePerSecond = new RateLimiter(1, 1);
+      onePerSecond.acquire().close();
+      assertThatThrownBy(() -> onePerSecond.acquire(Instant.now().plusMillis(20))).isInstanceOf(IllegalStateException.class);
+      try(var permit = onePerSecond.acquire(Instant.now().plusSeconds(2)))
+      {
+         assertThat(permit).isNotNull();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Fetch checks through the registry, exercising the supplier code reference.
+    *******************************************************************************/
+   private HealthCheckMetaData check() throws Exception
+   {
+      return (HealthChecksMetaData.ofOrWithNew(instance).resolveAll(instance).getFirst());
+   }
+
+
+
+   /*******************************************************************************
+    ** Default bounded input.
+    *******************************************************************************/
+   private HealthCheckInput input()
+   {
+      return (new HealthCheckInput().withQInstance(instance).withDeadline(Instant.now().plusSeconds(20)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Run the registered check.
+    *******************************************************************************/
+   private HealthCheckOutcome run() throws Exception
+   {
+      return (check().getCheckInstance().run(input()));
+   }
+
+
+
+   /*******************************************************************************
+    ** HealthRunner.sanitize is package-private; enforce cleanliness on raw output.
+    *******************************************************************************/
+   private void assertClean(HealthCheckOutcome outcome, HealthStatus expected)
+   {
+      assertThat(outcome.getStatus()).isEqualTo(expected);
+      assertThat(outcome.getMessage() + outcome.getDetails().toJson()).doesNotContain(SECRET, "private.example", "private-bucket");
+      assertThat(outcome.getMessage()).doesNotContain("\n", "\r");
+   }
+}
@@ -574,4 +574,40 @@ class MinioIndexTest
         .contains("Usage trend").contains("collecting since")
         .contains("Connections").contains("Never tested");
   }
+
+   /*******************************************************************************
+    ** Every link the home emits is a /app/<table>[...] route that the Next frontend
+    ** resolves, never an /<appName>/<table> path, with data and without.
+    *******************************************************************************/
+   @Test
+   void everyHomeLinkUsesTheAppRoute() throws Exception
+   {
+      String prefix = com.kingsrook.qbits.objectstorage.app.StorageWidgets.tablePathPrefix();
+      assertThat(prefix).isEqualTo("/app");
+      java.util.List<String> hrefs = new java.util.ArrayList<>(hrefs(com.kingsrook.qbits.objectstorage.app.StorageHome.render(prefix, Instant.now())));
+
+      rawPut(target.indexBucket, "media/1/a.jpg", 2048);
+      rawPut(target.indexBucket, "media/9/orphan.jpg", 1024);
+      ObjectIndex.scan("indexed");
+      ObjectIndex.headBacklog("indexed", 10);
+      ReferenceScan.run("indexed");
+      ObjectStorage.bucket("indexed").put(PutRequest.of("media/1/c.jpg", new byte[3]));
+      hrefs.addAll(hrefs(com.kingsrook.qbits.objectstorage.app.StorageHome.render(prefix, Instant.now())));
+
+      assertThat(hrefs).isNotEmpty().allSatisfy(href -> assertThat(href).startsWith("/app/"));
+      assertThat(hrefs).noneMatch(href -> href.contains("objectStorageAdmin"));
+   }
+
+
+
+   private static java.util.List<String> hrefs(String html)
+   {
+      java.util.List<String> hrefs = new java.util.ArrayList<>();
+      java.util.regex.Matcher m = java.util.regex.Pattern.compile("href=\"([^\"]*)\"").matcher(html);
+      while(m.find())
+      {
+         hrefs.add(m.group(1).replace("&amp;", "&"));
+      }
+      return (hrefs);
+   }
 }
@@ -38,6 +38,7 @@ import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMe
 import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3ChecksumMode;
 import com.sun.net.httpserver.Headers;
 import com.sun.net.httpserver.HttpServer;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
 import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
@@ -58,9 +59,21 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
 *******************************************************************************/
 class ConnectionRuntimeTest
 {
+   /*******************************************************************************
+    ** Core reads ${file.PATH} only inside its allowed roots; reference() points
+    ** them at each test's temporary directory.
+    *******************************************************************************/
+   @AfterEach
+   void afterEach()
+   {
+      System.clearProperty("qqq.fileReference.roots");
+   }
+
+
+
   /*******************************************************************************
    ** Every core-supported setting is mapped after resolving references. Session
-    ** tokens and presign settings have no core fields and are exercised below.
+    ** tokens and presign settings are exercised below.
    *******************************************************************************/
   @Test
   void testBackendMetaDataCarriesEverySetting(@TempDir Path dir) throws Exception
@@ -199,8 +212,8 @@ class ConnectionRuntimeTest


   /*******************************************************************************
-    ** Session credentials sign streaming reads/writes and paginator requests.
-    ** Existing request headers survive the bridge's credentials override.
+    ** Session credentials (core sessionToken) sign streaming reads/writes and
+    ** paginator requests; request overrides are untouched.
    *******************************************************************************/
   @Test
   void testSessionSigningAndRequestOverrides(@TempDir Path dir) throws Exception
@@ -313,6 +326,7 @@ class ConnectionRuntimeTest
    *******************************************************************************/
   private static String reference(Path dir, String name, String value) throws Exception
   {
+      System.setProperty("qqq.fileReference.roots", dir.toString());
      Path file = dir.resolve(name);
      Files.writeString(file, value);
      return ("${file." + file + "}");
@@ -83,10 +83,23 @@ class InternalsTest
      assertThat(SecretRefs.resolve(null)).isNull();
      assertThat(SecretRefs.resolveSetting("")).isNull();

+      //////////////////////////////////////////////////////////////////////
+      // core reads ${file.PATH} only inside its allowed roots (by default //
+      // /var/run/secrets and /run/secrets); point them at the temp dir.   //
+      //////////////////////////////////////////////////////////////////////
      Path file = dir.resolve("key");
      Files.writeString(file, "  from-file\n");
-      assertThat(SecretRefs.resolve("${file." + file + "}")).isEqualTo("from-file");
-      assertThat(SecretRefs.resolve("${file." + dir.resolve("missing") + "}")).isNull();
+      assertThat(SecretRefs.resolve("${file." + file + "}")).as("outside the allowed roots").isNull();
+      System.setProperty("qqq.fileReference.roots", dir.toString());
+      try
+      {
+         assertThat(SecretRefs.resolve("${file." + file + "}")).isEqualTo("from-file");
+         assertThat(SecretRefs.resolve("${file." + dir.resolve("missing") + "}")).isNull();
+      }
+      finally
+      {
+         System.clearProperty("qqq.fileReference.roots");
+      }
   }


@@ -0,0 +1,240 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.objectstorage.internal;
+
+
+import java.util.EnumSet;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.function.Consumer;
+import com.kingsrook.qbits.objectstorage.ObjectStorageQBitConfig;
+import com.kingsrook.qbits.objectstorage.api.Capability;
+import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
+import com.kingsrook.qbits.objectstorage.config.BucketRole;
+import com.kingsrook.qbits.objectstorage.config.ProviderType;
+import com.kingsrook.qbits.objectstorage.config.StorageBucketConfig;
+import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig;
+import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
+import com.kingsrook.qbits.objectstorage.testsupport.StorageTestSupport;
+import com.kingsrook.qqq.backend.core.cluster.ClusterBus;
+import com.kingsrook.qqq.backend.core.cluster.ClusterBusInterface;
+import com.kingsrook.qqq.backend.core.cluster.ClusterBusMetaData;
+import com.kingsrook.qqq.backend.core.cluster.ClusterCacheAudit;
+import com.kingsrook.qqq.backend.core.cluster.ClusterEvent;
+import com.kingsrook.qqq.backend.core.cluster.QNodeRoles;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** The Object Storage caches are correct on several nodes: every static cache
+ ** is registered with ClusterCaches, a probe result recorded on one node clears
+ ** that capability on the others, and an edited bucket row or header rule
+ ** clears its cache on every node.
+ *******************************************************************************/
+class ObjectStorageClusterCachesTest
+{
+   /** the "wire": events every node's transport sees */
+   static final List<Consumer<ClusterEvent>> WIRE = new CopyOnWriteArrayList<>();
+
+
+
+   /*******************************************************************************
+    ** A transport over the static WIRE, so a "remote" publish reaches us.
+    *******************************************************************************/
+   public static class WireBus implements ClusterBusInterface
+   {
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public void publish(ClusterEvent event)
+      {
+         WIRE.forEach(listener -> listener.accept(event));
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public AutoCloseable subscribe(Consumer<ClusterEvent> listener)
+      {
+         WIRE.add(listener);
+         return (() -> WIRE.remove(listener));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** A synced instance whose buckets never touch a real store, on the wire as
+    ** node-a.
+    *******************************************************************************/
+   @BeforeEach
+   void setUp() throws Exception
+   {
+      System.setProperty("os.unit.ak", "ak");
+      System.setProperty("os.unit.sk", "sk");
+      StorageTestSupport.setUp(new ObjectStorageQBitConfig().withBackendName(StorageTestSupport.MEMORY).withBucketScoping(false).withOpenWhenAnonymous(true).withAdminApp(true)
+            .withAllowedSecretRefPatterns(List.of("prop.os.unit.*"))
+            .withConnection(new StorageConnectionConfig("main").withProviderType(ProviderType.MINIO).withEndpoint("http://localhost:1").withRegion("us-east-1")
+               .withAccessKeyRef("${prop.os.unit.ak}").withSecretKeyRef("${prop.os.unit.sk}"))
+            .withBucket(new StorageBucketConfig("media").withConnection("main").withBucketName("media-bucket").withRole(BucketRole.PRIVATE).withIndexed(false)),
+         instance -> instance.withSupplementalMetaData(new ClusterBusMetaData().withBus(new QCodeReference(WireBus.class))));
+      QNodeRoles.set(new QNodeRoles.Settings(EnumSet.allOf(QNodeRoles.Role.class), true, true, true, null, "node-a"));
+      ClusterBus.connect(QContext.getQInstance());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @AfterEach
+   void tearDown()
+   {
+      ClusterBus.close(QContext.getQInstance());
+      WIRE.clear();
+      QNodeRoles.set(null);
+      StorageTestSupport.tearDown();
+   }
+
+
+
+   /*******************************************************************************
+    ** Every static Memoization in the QBit is registered (shared or node-local).
+    *******************************************************************************/
+   @Test
+   void testNoUnregisteredStaticCaches() throws Exception
+   {
+      assertThat(ClusterCacheAudit.unregisteredStaticCaches("com.kingsrook.qbits.objectstorage")).isEmpty();
+   }
+
+
+
+   /*******************************************************************************
+    ** Another node's probe result clears the capability here; the next read
+    ** reloads it from the capability row.
+    *******************************************************************************/
+   @Test
+   void testRemoteProbeResultClearsCapability() throws Exception
+   {
+      StorageBucketConfig bucket = ObjectStorageRuntime.bucketConfig("media");
+      CapabilityStore.record(bucket, Capability.VERSIONING, CapabilityStatus.UNSUPPORTED, "probe here");
+      assertThat(CapabilityStore.isCached(bucket, Capability.VERSIONING)).isTrue();
+
+      ////////////////////////////////////////////////////////////
+      // node-b probes again and writes SUPPORTED to the row    //
+      ////////////////////////////////////////////////////////////
+      Integer bucketId = ConfigSync.bucketId("media");
+      Db.asSystem(() ->
+      {
+         QRecord row = Db.first(ObjectStorageTables.BUCKET_CAPABILITY, "bucketId", bucketId, "capability", Capability.VERSIONING.name());
+         Db.update(ObjectStorageTables.BUCKET_CAPABILITY, List.of(new QRecord().withValue("id", row.getValue("id")).withValue("status", CapabilityStatus.SUPPORTED.name())));
+         return (null);
+      });
+      ClusterEvent remote = new ClusterEvent(ClusterEvent.TYPE_MEMO_INVALIDATE, CapabilityStore.CACHE_NAME, "b:media:" + Capability.VERSIONING);
+      remote.setOriginNodeId("node-b");
+      WIRE.forEach(listener -> listener.accept(remote));
+
+      assertThat(CapabilityStore.isCached(bucket, Capability.VERSIONING)).isFalse();
+      assertThat(CapabilityStore.status("media", Capability.VERSIONING)).isEqualTo(CapabilityStatus.SUPPORTED);
+   }
+
+
+
+   /*******************************************************************************
+    ** A probe result recorded here tells the other nodes, and stays cached here.
+    *******************************************************************************/
+   @Test
+   void testProbeResultPublishesInvalidation()
+   {
+      List<ClusterEvent> sent = sentEvents();
+      StorageBucketConfig bucket = ObjectStorageRuntime.bucketConfig("media");
+      CapabilityStore.record(bucket, Capability.VERSIONING, CapabilityStatus.SUPPORTED, "probe here");
+
+      assertThat(CapabilityStore.status("media", Capability.VERSIONING)).isEqualTo(CapabilityStatus.SUPPORTED);
+      assertThat(sent).anySatisfy(event -> assertInvalidation(event, CapabilityStore.CACHE_NAME, "b:media:VERSIONING"));
+   }
+
+
+
+   /*******************************************************************************
+    ** An edited bucket row takes effect at once and tells the other nodes.
+    *******************************************************************************/
+   @Test
+   void testBucketRowEditInvalidatesRowLimitsEverywhere() throws Exception
+   {
+      StorageBucketConfig bucket = ObjectStorageRuntime.bucketConfig("media");
+      assertThat(ConfigSync.effectiveMaxPresignGetSeconds(bucket)).isEqualTo(3600);
+
+      List<ClusterEvent> sent = sentEvents();
+      Db.update(ObjectStorageTables.BUCKET, List.of(new QRecord().withValue("id", ConfigSync.bucketId("media")).withValue("maxPresignGetSeconds", 60)));
+
+      assertThat(ConfigSync.effectiveMaxPresignGetSeconds(bucket)).as("no wait for the 15 s cache").isEqualTo(60);
+      assertThat(sent).anySatisfy(event -> assertInvalidation(event, ConfigSync.ROW_LIMITS_CACHE_NAME, null));
+   }
+
+
+
+   /*******************************************************************************
+    ** An edited header rule tells the other nodes.
+    *******************************************************************************/
+   @Test
+   void testHeaderRuleEditInvalidatesRulesEverywhere() throws Exception
+   {
+      List<ClusterEvent> sent = sentEvents();
+      StorageTestSupport.insert(ObjectStorageTables.HEADER_RULE, new QRecord().withValue("name", "all").withValue("keyGlob", "**").withValue("cacheControl", "no-store"));
+      assertThat(sent).anySatisfy(event -> assertInvalidation(event, DefaultObjectHeaderPolicy.CACHE_NAME, null));
+   }
+
+
+
+   /*******************************************************************************
+    ** Collect what this node puts on the wire.
+    *******************************************************************************/
+   private static List<ClusterEvent> sentEvents()
+   {
+      List<ClusterEvent> sent = new CopyOnWriteArrayList<>();
+      WIRE.add(sent::add);
+      return (sent);
+   }
+
+
+
+   /*******************************************************************************
+    ** A memo invalidation from node-a for a cache (and key).
+    *******************************************************************************/
+   private static void assertInvalidation(ClusterEvent event, String name, String key)
+   {
+      assertThat(event.getType()).isEqualTo(ClusterEvent.TYPE_MEMO_INVALIDATE);
+      assertThat(event.getName()).isEqualTo(name);
+      assertThat(event.getKey()).isEqualTo(key);
+      assertThat(event.getOriginNodeId()).isEqualTo("node-a");
+   }
+}
@@ -205,7 +205,7 @@ class TablesAndConfigTest
      new com.kingsrook.qbits.objectstorage.ObjectStorageQBitProducer().withQBitConfig(config).produce(instance);
      assertThat(instance.getApp("objectStorageAdmin").getParentAppName()).isEqualTo("admin");
      assertThat(instance.getApp("admin").getChildren()).extracting(c -> c.getName()).contains("objectStorageAdmin");
-      assertThat(com.kingsrook.qbits.objectstorage.app.StorageWidgets.tablePathPrefix()).isEqualTo("/admin/objectStorageAdmin");
+      assertThat(com.kingsrook.qbits.objectstorage.app.StorageWidgets.tablePathPrefix()).isEqualTo("/app");
   }
Author	SHA1	Message	Date
james.maesandCodex	022505d130	ci: centralize QBit source scanning Co-Authored-By: Codex <noreply@openai.com>	2026-10-21 10:19:15 -05:00
james.maesandCodex	03ed383587	ci: adopt central orb 0.7.12 and stable gates (#35 ) Co-authored-by: Codex <noreply@openai.com>	2026-10-21 08:58:38 -05:00
james.maesandCodex	85e9ab1316	ci: centralize rc release completion in qqq orb (#34 ) Co-authored-by: Codex <noreply@openai.com>	2026-10-21 02:30:37 -05:00
james.maesandCodex	03215cab62	ci: adopt shared orb 0.7.11 (#33 ) Co-authored-by: Codex <noreply@openai.com>	2026-10-21 01:02:00 -05:00
james.maes	6866e6653f	Merge pull request #32 from QRun-IO/feature/central-ci ci: adopt verified central QQQ orb 0.7.1	2026-10-20 18:49:56 -05:00
james.maesandCodex	eceb2a6d59	ci: adopt verified central QQQ orb 0.7.2 Co-Authored-By: Codex <noreply@openai.com>	2026-10-20 18:42:41 -05:00
james.maes	3f8ca0ef1f	Merge pull request #30 from QRun-IO/fix/object-storage-app-links fix(app): Storage home links use the /app/<table> route	2026-10-16 10:19:59 -05:00
james.maesandClaude Opus 5.5	e341a90df3	fix(app): Storage home links use the /app/<table> route The home emitted /objectStorageAdmin/<table> (or /<parent>/objectStorageAdmin/ <table>), which Next does not serve. Links now use /app/<table>, configurable through ObjectStorageQBitConfig.tablePathPrefix (default /app), matching the other qbits. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-16 09:12:28 -05:00
james.maes	a7e6a40924	Merge pull request #29 from QRun-IO/ci/release-rc-secrets-inherit ci(release-rc): pass secrets so the shared workflow can sign the RC tag	2026-10-16 08:03:32 -05:00
james.maes	0222074cf9	Merge pull request #27 from QRun-IO/feature/cluster-caches Register object-storage caches with ClusterCaches (multi-node)	2026-10-15 16:29:04 -05:00
james.maesandClaude Opus 5.5	4033590078	ci(release-rc): pass secrets to the shared workflow so it can sign the RC tag QRun-IO/.github#2 adds a tag-and-release job that signs vX with the QRUN_TAG_SIGNING_KEY secret after Maven Central confirms the artifact. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-15 16:19:37 -05:00
james.maes	75b4647d08	Merge remote-tracking branch 'origin/develop' into feature/cluster-caches # Conflicts: # pom.xml	2026-10-15 16:17:05 -05:00
james.maes	9590619e0d	Merge pull request #28 from QRun-IO/feature/core-s3-session-file-refs refactor: core S3 session credentials and ${file.PATH} refs; drop qqq#1165 stand-ins (QQQ RC.15)	2026-10-15 16:07:37 -05:00
james.maesandClaude Opus 5.5	4261382567	build: pin QQQ 4.1.0-RC.15 (core S3 session credentials and ${file.PATH}, qqq#1165) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-15 16:00:18 -05:00
james.maesandClaude Opus 5.5	88f7f33fe7	build: QQQ 4.1.0-RC.14 (ships ClusterCaches) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-15 14:05:09 -05:00
james.maesandClaude Opus 5.5	2b5a84e805	refactor: drop the qqq#1165 stand-ins for core session credentials and file refs SessionCredentialsClient is gone: the session token now rides core S3BackendMetaData.sessionToken, and the presigner takes its credentials from core S3ClientFactory.buildCredentialsProvider. SecretRefs keeps only the qbit's allow-list; ${file.PATH} resolution is core QMetaDataVariableInterpreter (read-only, allowed roots /var/run/secrets and /run/secrets by default). Tests point qqq.fileReference.roots at their temp dirs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-15 13:16:47 -05:00
james.maesandClaude Opus 5.5	878270503e	feat(cluster): register object-storage caches with ClusterCaches Shared (cleared on every node): bucket-row presign limits and enabled header rules, invalidated by the new ConfigRowChangeCustomizer on the bucket and header-rule tables, and capability statuses, invalidated per key when a probe result is recorded (other nodes reload from the row). The 15 s TTLs stay as the safety net. Row-id maps, S3 clients, process permissions, provider defaults, the running-scan guard and the per-pod presign rate limiter are node-local, with reasons. Builds against QQQ 4.1.0-SNAPSHOT (qqq.snapshot.version) until the QQQ RC with ClusterCaches is published; a release must pin that RC. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-15 12:20:02 -05:00
james.maes	7c825b05d5	Merge pull request #26 from QRun-IO/fix/producer-version-from-build fix: qbit metadata version comes from the build	2026-10-15 01:42:56 -05:00
james.maesandClaude Opus 5.5	5401070321	fix: the qbit metadata version comes from the build, not a hardcoded constant The producer read a literal version, so released artifacts reported a stale or -SNAPSHOT version. qbit-object-storage.properties (version=${project.version}) is Maven-filtered and the producer loads it; tests assert producer == bundle == artifact version. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-15 01:34:11 -05:00
james.maes	75bf74967d	Merge pull request #25 from QRun-IO/fix/help-bundle-version fix(help): help bundle version comes from the build, not a hardcoded SNAPSHOT	2026-10-14 22:37:21 -05:00
james.maesandClaude Opus 5.5	eb04621d4e	fix(help): the help bundle carries the artifact version (Maven-filtered) instead of a hardcoded SNAPSHOT On release branches the bundle's version and the producer's differed, so the bundle test that asserts they match failed, and released bundles recorded SNAPSHOT as their provenance. help/*.yaml is now filtered: version: ${project.version}. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-14 22:07:58 -05:00
james.maes	38e74cc07f	Merge pull request #17 from QRun-IO/feature/health-checks feat(health): register object storage connection checks with core central health	2026-10-14 21:27:01 -05:00
james.maesandClaude Opus 5.5	8aa4511839	Merge origin/develop into feature/health-checks; pin QQQ 4.1.0-RC.8 RC.8 ships the central health kernel (QRun-IO/qqq#1207). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-14 21:18:29 -05:00
james.maes	9b2158bf97	Merge pull request #24 from QRun-IO/feature/help-bundle feat(help): qbit-object-storage help bundle via QQQ help bundles, with coverage gate	2026-10-14 20:45:30 -05:00
james.maesandClaude Opus 5.5	fd81c88f32	docs: object storage health checks Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-14 13:39:40 -05:00
james.maesandClaude Opus 5.5	63c441e6cd	feat(health): register object storage connection checks with core central health Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-14 13:39:39 -05:00
james.maesandClaude Opus 5.5	b13122f1b0	feat(health): deadline-aware read-only capability probe Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-14 13:39:38 -05:00