CHANGELOG.md

@@ -27,6 +27,7 @@ All notable changes to this project are documented here. The format follows Keep
 - Conformance suite against MinIO and Garage (Testcontainers), service, index and action tests, and the index and actions on PostgreSQL.
 
 ### Removed
+- The QRun-IO/qqq#1165 stand-ins: `SessionCredentialsClient` (session tokens now go through core `S3BackendMetaData.sessionToken` / `S3ClientFactory`, including for the presigner) and the file reading in `SecretRefs` (core `QMetaDataVariableInterpreter` resolves `${file.PATH}`). **Behavior change:** core reads `${file.PATH}` only inside its allowed roots, which are `/var/run/secrets` and `/run/secrets` by default. A host that mounts storage secrets elsewhere must set `qqq.fileReference.roots` or `QQQ_FILE_REFERENCE_ROOTS`. The qbit's default allow-list (`file./var/run/secrets/storage/*`) is already inside the default roots.
 - None for the health migration; both Test connection record actions, including on-demand write tests, are retained.
 
 ### Changed

docs/S3-FACTORY-MIGRATION.md

@@ -26,16 +26,21 @@ The public qbit configuration API is unchanged.
 
 ## Remaining local code
 
-- Core RC.5 has no session-token field or credentials-provider customization
-  in `S3BackendMetaData` / `S3ClientFactory.buildClient`. The narrow
-  `SessionCredentialsClient` bridge adds session credentials to requests while
-  retaining their overrides; core still creates the client and HTTP transport.
-  Core needs session-token metadata (including enrichment and cache-key support)
-  and `AwsSessionCredentials` construction, or a caller-provided credentials hook.
-  Marker: `STAND-IN(core=QRun-IO/qqq#1165, removeBy=4.1.0-RC.7)`.
-- Core has no presigner factory. The local presigner retains its separate public
-  endpoint and least-privilege keys, along with existing session-token handling.
-- `SecretRefs` file resolution is unchanged and carries the same RC.7 marker.
+- Core has no presigner factory. The local presigner keeps its separate public
+  endpoint and least-privilege keys; its credentials (including the session
+  token) come from core `S3ClientFactory.buildCredentialsProvider`.
+- `SecretRefs` keeps only the qbit's allow-list; resolution, including
+  `${file.PATH}`, is core `QMetaDataVariableInterpreter`.
+
+## Stand-ins removed (QRun-IO/qqq#1165)
+
+- `SessionCredentialsClient` is gone: core `S3BackendMetaData.sessionToken`
+  makes `S3ClientFactory` sign with `AwsSessionCredentials`.
+- `SecretRefs` no longer reads files itself. Core reads `${file.PATH}`
+  read-only, and only inside its allowed roots: `/var/run/secrets` and
+  `/run/secrets` by default, or `qqq.fileReference.roots` /
+  `QQQ_FILE_REFERENCE_ROOTS` (comma-separated). The qbit's default allow-list
+  (`file./var/run/secrets/storage/*`) is inside the default roots.
 
 ## Resolved dependency diff
 

pom.xml

@@ -283,14 +283,14 @@
    <profiles>
       <!-- Built against qqq 4.1 (QBitMetaDataProducer). Until qbit-build-parent  -->
       <!-- imports the released 4.1 BOM, this profile is active by default and     -->
-      <!-- imports qqq-bom-pom 4.1.0-RC.7 ahead of the parent's 4.0.0 BOM.         -->
+      <!-- imports qqq-bom-pom 4.1.0-RC.15 ahead of the parent's 4.0.0 BOM.        -->
       <profile>
          <id>qqq-snapshot</id>
          <activation>
             <activeByDefault>true</activeByDefault>
          </activation>
          <properties>
-            <qqq.snapshot.version>4.1.0-RC.8</qqq.snapshot.version>
+            <qqq.snapshot.version>4.1.0-RC.15</qqq.snapshot.version>
          </properties>
          <dependencyManagement>
             <dependencies>

src/main/java/com/kingsrook/qbits/objectstorage/internal/ConnectionRuntime.java

@@ -22,7 +22,6 @@ package com.kingsrook.qbits.objectstorage.internal;
 
 
 import java.net.URI;
-import java.util.function.Function;
 import com.kingsrook.qbits.objectstorage.config.AddressingStyle;
 import com.kingsrook.qbits.objectstorage.config.CredentialsMode;
 import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig;
@@ -30,17 +29,10 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3ChecksumMode;
 import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3ClientFactory;
-import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
 import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
-import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
-import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration;
 import software.amazon.awssdk.regions.Region;
-import software.amazon.awssdk.services.s3.DelegatingS3Client;
 import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.S3Configuration;
-import software.amazon.awssdk.services.s3.model.S3Request;
 import software.amazon.awssdk.services.s3.presigner.S3Presigner;
 
 
@@ -198,16 +190,7 @@ public class ConnectionRuntime
     *******************************************************************************/
    private S3Client buildClient()
    {
-      S3BackendMetaData backend = backendMetaData();
-      String token = config.getCredentialsMode() == CredentialsMode.STATIC_REFS ? SecretRefs.resolve(config.getSessionTokenRef()) : null;
-      AwsCredentialsProvider sessionCredentials = null;
-      if(token != null && backend.getAccessKey() != null && backend.getSecretKey() != null)
-      {
-         sessionCredentials = StaticCredentialsProvider.create(AwsSessionCredentials.create(backend.getAccessKey(), backend.getSecretKey(), token));
-      }
-
-      S3Client coreClient = S3ClientFactory.buildClient(backend);
-      return (sessionCredentials == null ? coreClient : new SessionCredentialsClient(coreClient, sessionCredentials));
+      return (S3ClientFactory.buildClient(backendMetaData()));
    }
 
 
@@ -229,8 +212,7 @@ public class ConnectionRuntime
          .withSocketTimeoutMillis(config.getReadTimeoutMs());
       if(config.getCredentialsMode() == CredentialsMode.STATIC_REFS)
       {
-         backend.withAccessKey(SecretRefs.resolve(config.getAccessKeyRef()))
-            .withSecretKey(SecretRefs.resolve(config.getSecretKeyRef()));
+         withStaticCredentials(backend, config.getAccessKeyRef(), config.getSecretKeyRef());
       }
       return (backend);
    }
@@ -270,63 +252,33 @@ public class ConnectionRuntime
 
 
    /*******************************************************************************
-    ** Static keys from the references, or the SDK default chain.
+    ** Static keys (with the session token, when set) from the references, or the
+    ** SDK default chain; core builds the provider.
     *******************************************************************************/
    private AwsCredentialsProvider credentials(String accessRef, String secretRef)
    {
+      S3BackendMetaData credentials = new S3BackendMetaData();
       if(config.getCredentialsMode() == CredentialsMode.STATIC_REFS)
       {
-         String accessKey = SecretRefs.resolve(accessRef);
-         String secretKey = SecretRefs.resolve(secretRef);
-         if(accessKey != null && secretKey != null)
-         {
-            String token = SecretRefs.resolve(config.getSessionTokenRef());
-            return (StaticCredentialsProvider.create(token == null ? AwsBasicCredentials.create(accessKey, secretKey)
-               : AwsSessionCredentials.create(accessKey, secretKey, token)));
-         }
+         withStaticCredentials(credentials, accessRef, secretRef);
       }
-      return (DefaultCredentialsProvider.builder().build());
+      return (S3ClientFactory.buildCredentialsProvider(credentials));
    }
 
 
 
    /*******************************************************************************
-    ** STAND-IN(core=QRun-IO/qqq#1165, removeBy=4.1.0-RC.9)
-    ** RC.8 S3BackendMetaData and S3ClientFactory still lack a session token or
-    ** credentials-provider hook (verified against the released JAR).
-    ** Override request credentials only; core still owns the client and transport.
+    ** Resolve static keys onto core meta data. The session token applies only
+    ** with both keys, as before (core rejects a token without keys).
     *******************************************************************************/
-   private static class SessionCredentialsClient extends DelegatingS3Client
+   private void withStaticCredentials(S3BackendMetaData backend, String accessRef, String secretRef)
    {
-      private final AwsCredentialsProvider credentialsProvider;
-
-
-
-      /*******************************************************************************
-       ** Wrap a core-built client with the connection's session credentials.
-       *******************************************************************************/
-      private SessionCredentialsClient(S3Client delegate, AwsCredentialsProvider credentialsProvider)
+      String accessKey = SecretRefs.resolve(accessRef);
+      String secretKey = SecretRefs.resolve(secretRef);
+      backend.withAccessKey(accessKey).withSecretKey(secretKey);
+      if(accessKey != null && secretKey != null)
       {
-         super(delegate);
-         this.credentialsProvider = credentialsProvider;
-      }
-
-
-
-      /*******************************************************************************
-       ** Preserve request overrides and the concrete request type while adding the
-       ** session token to signing, including streaming and paginated operations.
-       *******************************************************************************/
-      @Override
-      @SuppressWarnings("unchecked") // toBuilder().build() retains the concrete request type T.
-      protected <T extends S3Request, ReturnT> ReturnT invokeOperation(T request, Function<T, ReturnT> operation)
-      {
-         AwsRequestOverrideConfiguration.Builder overrides = request.overrideConfiguration()
-            .map(AwsRequestOverrideConfiguration::toBuilder).orElseGet(AwsRequestOverrideConfiguration::builder);
-         T signedRequest = (T) request.toBuilder()
-            .overrideConfiguration(overrides.credentialsProvider(credentialsProvider).build())
-            .build();
-         return (operation.apply(signedRequest));
+         backend.withSessionToken(SecretRefs.resolve(config.getSessionTokenRef()));
       }
    }
 }

src/main/java/com/kingsrook/qbits/objectstorage/internal/SecretRefs.java

@@ -21,10 +21,6 @@
 package com.kingsrook.qbits.objectstorage.internal;
 
 
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
 import java.util.List;
 import java.util.regex.Pattern;
 import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
@@ -32,14 +28,12 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
 /*******************************************************************************
- ** Resolves secret references (${env.X}, ${prop.X}, ${file.PATH}) and enforces
- ** the allow-list. Resolved values live only in memory, never in a record, a log
- ** line or a process value.
- **
- ** STAND-IN(core=QRun-IO/qqq#1165, removeBy=4.1.0-RC.9)
- ** RC.8 QMetaDataVariableInterpreter still supports env/prop, not file references.
- ** ${file.PATH} moves to QMetaDataVariableInterpreter in QQQ core
- ** (spec section 8.2); then this class delegates entirely.
+ ** Enforces the qbit's secret-reference allow-list and resolves references
+ ** (${env.X}, ${prop.X}, ${file.PATH}) through core QMetaDataVariableInterpreter.
+ ** Core reads ${file.PATH} read-only from its allowed roots (default
+ ** /var/run/secrets and /run/secrets; qqq.fileReference.roots or
+ ** QQQ_FILE_REFERENCE_ROOTS). Resolved values live only in memory, never in a
+ ** record, a log line or a process value.
  *******************************************************************************/
 public final class SecretRefs
 {
@@ -93,19 +87,7 @@ public final class SecretRefs
       {
          return (null);
       }
-      String trimmed = ref.trim();
-      if(trimmed.startsWith("${file.") && trimmed.endsWith("}"))
-      {
-         try
-         {
-            return (Files.readString(Path.of(trimmed.substring("${file.".length(), trimmed.length() - 1)), StandardCharsets.UTF_8).trim());
-         }
-         catch(IOException e)
-         {
-            return (null);
-         }
-      }
-      String value = new QMetaDataVariableInterpreter().interpret(trimmed);
+      String value = new QMetaDataVariableInterpreter().interpret(ref.trim());
       return (value == null || value.startsWith("${") ? null : value);
    }
 

src/test/java/com/kingsrook/qbits/objectstorage/internal/ConnectionRuntimeTest.java

@@ -38,6 +38,7 @@ import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMe
 import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3ChecksumMode;
 import com.sun.net.httpserver.Headers;
 import com.sun.net.httpserver.HttpServer;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
 import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
@@ -58,9 +59,21 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
  *******************************************************************************/
 class ConnectionRuntimeTest
 {
+   /*******************************************************************************
+    ** Core reads ${file.PATH} only inside its allowed roots; reference() points
+    ** them at each test's temporary directory.
+    *******************************************************************************/
+   @AfterEach
+   void afterEach()
+   {
+      System.clearProperty("qqq.fileReference.roots");
+   }
+
+
+
    /*******************************************************************************
     ** Every core-supported setting is mapped after resolving references. Session
-    ** tokens and presign settings have no core fields and are exercised below.
+    ** tokens and presign settings are exercised below.
     *******************************************************************************/
    @Test
    void testBackendMetaDataCarriesEverySetting(@TempDir Path dir) throws Exception
@@ -199,8 +212,8 @@ class ConnectionRuntimeTest
 
 
    /*******************************************************************************
-    ** Session credentials sign streaming reads/writes and paginator requests.
-    ** Existing request headers survive the bridge's credentials override.
+    ** Session credentials (core sessionToken) sign streaming reads/writes and
+    ** paginator requests; request overrides are untouched.
     *******************************************************************************/
    @Test
    void testSessionSigningAndRequestOverrides(@TempDir Path dir) throws Exception
@@ -313,6 +326,7 @@ class ConnectionRuntimeTest
     *******************************************************************************/
    private static String reference(Path dir, String name, String value) throws Exception
    {
+      System.setProperty("qqq.fileReference.roots", dir.toString());
       Path file = dir.resolve(name);
       Files.writeString(file, value);
       return ("${file." + file + "}");

src/test/java/com/kingsrook/qbits/objectstorage/internal/InternalsTest.java

@@ -83,10 +83,23 @@ class InternalsTest
       assertThat(SecretRefs.resolve(null)).isNull();
       assertThat(SecretRefs.resolveSetting("")).isNull();
 
+      //////////////////////////////////////////////////////////////////////
+      // core reads ${file.PATH} only inside its allowed roots (by default //
+      // /var/run/secrets and /run/secrets); point them at the temp dir.   //
+      //////////////////////////////////////////////////////////////////////
       Path file = dir.resolve("key");
       Files.writeString(file, "  from-file\n");
-      assertThat(SecretRefs.resolve("${file." + file + "}")).isEqualTo("from-file");
-      assertThat(SecretRefs.resolve("${file." + dir.resolve("missing") + "}")).isNull();
+      assertThat(SecretRefs.resolve("${file." + file + "}")).as("outside the allowed roots").isNull();
+      System.setProperty("qqq.fileReference.roots", dir.toString());
+      try
+      {
+         assertThat(SecretRefs.resolve("${file." + file + "}")).isEqualTo("from-file");
+         assertThat(SecretRefs.resolve("${file." + dir.resolve("missing") + "}")).isNull();
+      }
+      finally
+      {
+         System.clearProperty("qqq.fileReference.roots");
+      }
    }