From 2b5a84e805b8e3dcf3b28a6ccb4bcd0d053e7a4e Mon Sep 17 00:00:00 2001 From: James Maes Date: Thu, 15 Oct 2026 13:16:47 -0500 Subject: [PATCH 1/2] refactor: drop the qqq#1165 stand-ins for core session credentials and file refs SessionCredentialsClient is gone: the session token now rides core S3BackendMetaData.sessionToken, and the presigner takes its credentials from core S3ClientFactory.buildCredentialsProvider. SecretRefs keeps only the qbit's allow-list; ${file.PATH} resolution is core QMetaDataVariableInterpreter (read-only, allowed roots /var/run/secrets and /run/secrets by default). Tests point qqq.fileReference.roots at their temp dirs. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + docs/S3-FACTORY-MIGRATION.md | 25 +++--- .../internal/ConnectionRuntime.java | 78 ++++--------------- .../objectstorage/internal/SecretRefs.java | 32 ++------ .../internal/ConnectionRuntimeTest.java | 20 ++++- .../objectstorage/internal/InternalsTest.java | 17 +++- 6 files changed, 70 insertions(+), 103 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96519cd..6508d77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ All notable changes to this project are documented here. The format follows Keep - Conformance suite against MinIO and Garage (Testcontainers), service, index and action tests, and the index and actions on PostgreSQL. ### Removed +- The QRun-IO/qqq#1165 stand-ins: `SessionCredentialsClient` (session tokens now go through core `S3BackendMetaData.sessionToken` / `S3ClientFactory`, including for the presigner) and the file reading in `SecretRefs` (core `QMetaDataVariableInterpreter` resolves `${file.PATH}`). **Behavior change:** core reads `${file.PATH}` only inside its allowed roots, which are `/var/run/secrets` and `/run/secrets` by default. A host that mounts storage secrets elsewhere must set `qqq.fileReference.roots` or `QQQ_FILE_REFERENCE_ROOTS`. The qbit's default allow-list (`file./var/run/secrets/storage/*`) is already inside the default roots. - None for the health migration; both Test connection record actions, including on-demand write tests, are retained. ### Changed diff --git a/docs/S3-FACTORY-MIGRATION.md b/docs/S3-FACTORY-MIGRATION.md index d4a97ea..4eee75c 100644 --- a/docs/S3-FACTORY-MIGRATION.md +++ b/docs/S3-FACTORY-MIGRATION.md @@ -26,16 +26,21 @@ The public qbit configuration API is unchanged. ## Remaining local code -- Core RC.5 has no session-token field or credentials-provider customization - in `S3BackendMetaData` / `S3ClientFactory.buildClient`. The narrow - `SessionCredentialsClient` bridge adds session credentials to requests while - retaining their overrides; core still creates the client and HTTP transport. - Core needs session-token metadata (including enrichment and cache-key support) - and `AwsSessionCredentials` construction, or a caller-provided credentials hook. - Marker: `STAND-IN(core=QRun-IO/qqq#1165, removeBy=4.1.0-RC.7)`. -- Core has no presigner factory. The local presigner retains its separate public - endpoint and least-privilege keys, along with existing session-token handling. -- `SecretRefs` file resolution is unchanged and carries the same RC.7 marker. +- Core has no presigner factory. The local presigner keeps its separate public + endpoint and least-privilege keys; its credentials (including the session + token) come from core `S3ClientFactory.buildCredentialsProvider`. +- `SecretRefs` keeps only the qbit's allow-list; resolution, including + `${file.PATH}`, is core `QMetaDataVariableInterpreter`. + +## Stand-ins removed (QRun-IO/qqq#1165) + +- `SessionCredentialsClient` is gone: core `S3BackendMetaData.sessionToken` + makes `S3ClientFactory` sign with `AwsSessionCredentials`. +- `SecretRefs` no longer reads files itself. Core reads `${file.PATH}` + read-only, and only inside its allowed roots: `/var/run/secrets` and + `/run/secrets` by default, or `qqq.fileReference.roots` / + `QQQ_FILE_REFERENCE_ROOTS` (comma-separated). The qbit's default allow-list + (`file./var/run/secrets/storage/*`) is inside the default roots. ## Resolved dependency diff diff --git a/src/main/java/com/kingsrook/qbits/objectstorage/internal/ConnectionRuntime.java b/src/main/java/com/kingsrook/qbits/objectstorage/internal/ConnectionRuntime.java index 84aae98..4ac8e8e 100644 --- a/src/main/java/com/kingsrook/qbits/objectstorage/internal/ConnectionRuntime.java +++ b/src/main/java/com/kingsrook/qbits/objectstorage/internal/ConnectionRuntime.java @@ -22,7 +22,6 @@ package com.kingsrook.qbits.objectstorage.internal; import java.net.URI; -import java.util.function.Function; import com.kingsrook.qbits.objectstorage.config.AddressingStyle; import com.kingsrook.qbits.objectstorage.config.CredentialsMode; import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig; @@ -30,17 +29,10 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3ChecksumMode; import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3ClientFactory; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; -import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration; import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.DelegatingS3Client; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.S3Configuration; -import software.amazon.awssdk.services.s3.model.S3Request; import software.amazon.awssdk.services.s3.presigner.S3Presigner; @@ -198,16 +190,7 @@ public class ConnectionRuntime *******************************************************************************/ private S3Client buildClient() { - S3BackendMetaData backend = backendMetaData(); - String token = config.getCredentialsMode() == CredentialsMode.STATIC_REFS ? SecretRefs.resolve(config.getSessionTokenRef()) : null; - AwsCredentialsProvider sessionCredentials = null; - if(token != null && backend.getAccessKey() != null && backend.getSecretKey() != null) - { - sessionCredentials = StaticCredentialsProvider.create(AwsSessionCredentials.create(backend.getAccessKey(), backend.getSecretKey(), token)); - } - - S3Client coreClient = S3ClientFactory.buildClient(backend); - return (sessionCredentials == null ? coreClient : new SessionCredentialsClient(coreClient, sessionCredentials)); + return (S3ClientFactory.buildClient(backendMetaData())); } @@ -229,8 +212,7 @@ public class ConnectionRuntime .withSocketTimeoutMillis(config.getReadTimeoutMs()); if(config.getCredentialsMode() == CredentialsMode.STATIC_REFS) { - backend.withAccessKey(SecretRefs.resolve(config.getAccessKeyRef())) - .withSecretKey(SecretRefs.resolve(config.getSecretKeyRef())); + withStaticCredentials(backend, config.getAccessKeyRef(), config.getSecretKeyRef()); } return (backend); } @@ -270,63 +252,33 @@ public class ConnectionRuntime /******************************************************************************* - ** Static keys from the references, or the SDK default chain. + ** Static keys (with the session token, when set) from the references, or the + ** SDK default chain; core builds the provider. *******************************************************************************/ private AwsCredentialsProvider credentials(String accessRef, String secretRef) { + S3BackendMetaData credentials = new S3BackendMetaData(); if(config.getCredentialsMode() == CredentialsMode.STATIC_REFS) { - String accessKey = SecretRefs.resolve(accessRef); - String secretKey = SecretRefs.resolve(secretRef); - if(accessKey != null && secretKey != null) - { - String token = SecretRefs.resolve(config.getSessionTokenRef()); - return (StaticCredentialsProvider.create(token == null ? AwsBasicCredentials.create(accessKey, secretKey) - : AwsSessionCredentials.create(accessKey, secretKey, token))); - } + withStaticCredentials(credentials, accessRef, secretRef); } - return (DefaultCredentialsProvider.builder().build()); + return (S3ClientFactory.buildCredentialsProvider(credentials)); } /******************************************************************************* - ** STAND-IN(core=QRun-IO/qqq#1165, removeBy=4.1.0-RC.9) - ** RC.8 S3BackendMetaData and S3ClientFactory still lack a session token or - ** credentials-provider hook (verified against the released JAR). - ** Override request credentials only; core still owns the client and transport. + ** Resolve static keys onto core meta data. The session token applies only + ** with both keys, as before (core rejects a token without keys). *******************************************************************************/ - private static class SessionCredentialsClient extends DelegatingS3Client + private void withStaticCredentials(S3BackendMetaData backend, String accessRef, String secretRef) { - private final AwsCredentialsProvider credentialsProvider; - - - - /******************************************************************************* - ** Wrap a core-built client with the connection's session credentials. - *******************************************************************************/ - private SessionCredentialsClient(S3Client delegate, AwsCredentialsProvider credentialsProvider) + String accessKey = SecretRefs.resolve(accessRef); + String secretKey = SecretRefs.resolve(secretRef); + backend.withAccessKey(accessKey).withSecretKey(secretKey); + if(accessKey != null && secretKey != null) { - super(delegate); - this.credentialsProvider = credentialsProvider; - } - - - - /******************************************************************************* - ** Preserve request overrides and the concrete request type while adding the - ** session token to signing, including streaming and paginated operations. - *******************************************************************************/ - @Override - @SuppressWarnings("unchecked") // toBuilder().build() retains the concrete request type T. - protected ReturnT invokeOperation(T request, Function operation) - { - AwsRequestOverrideConfiguration.Builder overrides = request.overrideConfiguration() - .map(AwsRequestOverrideConfiguration::toBuilder).orElseGet(AwsRequestOverrideConfiguration::builder); - T signedRequest = (T) request.toBuilder() - .overrideConfiguration(overrides.credentialsProvider(credentialsProvider).build()) - .build(); - return (operation.apply(signedRequest)); + backend.withSessionToken(SecretRefs.resolve(config.getSessionTokenRef())); } } } diff --git a/src/main/java/com/kingsrook/qbits/objectstorage/internal/SecretRefs.java b/src/main/java/com/kingsrook/qbits/objectstorage/internal/SecretRefs.java index 8081658..2a37132 100644 --- a/src/main/java/com/kingsrook/qbits/objectstorage/internal/SecretRefs.java +++ b/src/main/java/com/kingsrook/qbits/objectstorage/internal/SecretRefs.java @@ -21,10 +21,6 @@ package com.kingsrook.qbits.objectstorage.internal; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.List; import java.util.regex.Pattern; import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; @@ -32,14 +28,12 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* - ** Resolves secret references (${env.X}, ${prop.X}, ${file.PATH}) and enforces - ** the allow-list. Resolved values live only in memory, never in a record, a log - ** line or a process value. - ** - ** STAND-IN(core=QRun-IO/qqq#1165, removeBy=4.1.0-RC.9) - ** RC.8 QMetaDataVariableInterpreter still supports env/prop, not file references. - ** ${file.PATH} moves to QMetaDataVariableInterpreter in QQQ core - ** (spec section 8.2); then this class delegates entirely. + ** Enforces the qbit's secret-reference allow-list and resolves references + ** (${env.X}, ${prop.X}, ${file.PATH}) through core QMetaDataVariableInterpreter. + ** Core reads ${file.PATH} read-only from its allowed roots (default + ** /var/run/secrets and /run/secrets; qqq.fileReference.roots or + ** QQQ_FILE_REFERENCE_ROOTS). Resolved values live only in memory, never in a + ** record, a log line or a process value. *******************************************************************************/ public final class SecretRefs { @@ -93,19 +87,7 @@ public final class SecretRefs { return (null); } - String trimmed = ref.trim(); - if(trimmed.startsWith("${file.") && trimmed.endsWith("}")) - { - try - { - return (Files.readString(Path.of(trimmed.substring("${file.".length(), trimmed.length() - 1)), StandardCharsets.UTF_8).trim()); - } - catch(IOException e) - { - return (null); - } - } - String value = new QMetaDataVariableInterpreter().interpret(trimmed); + String value = new QMetaDataVariableInterpreter().interpret(ref.trim()); return (value == null || value.startsWith("${") ? null : value); } diff --git a/src/test/java/com/kingsrook/qbits/objectstorage/internal/ConnectionRuntimeTest.java b/src/test/java/com/kingsrook/qbits/objectstorage/internal/ConnectionRuntimeTest.java index 2285c34..0db49a0 100644 --- a/src/test/java/com/kingsrook/qbits/objectstorage/internal/ConnectionRuntimeTest.java +++ b/src/test/java/com/kingsrook/qbits/objectstorage/internal/ConnectionRuntimeTest.java @@ -38,6 +38,7 @@ import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMe import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3ChecksumMode; import com.sun.net.httpserver.Headers; import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; @@ -58,9 +59,21 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; *******************************************************************************/ class ConnectionRuntimeTest { + /******************************************************************************* + ** Core reads ${file.PATH} only inside its allowed roots; reference() points + ** them at each test's temporary directory. + *******************************************************************************/ + @AfterEach + void afterEach() + { + System.clearProperty("qqq.fileReference.roots"); + } + + + /******************************************************************************* ** Every core-supported setting is mapped after resolving references. Session - ** tokens and presign settings have no core fields and are exercised below. + ** tokens and presign settings are exercised below. *******************************************************************************/ @Test void testBackendMetaDataCarriesEverySetting(@TempDir Path dir) throws Exception @@ -199,8 +212,8 @@ class ConnectionRuntimeTest /******************************************************************************* - ** Session credentials sign streaming reads/writes and paginator requests. - ** Existing request headers survive the bridge's credentials override. + ** Session credentials (core sessionToken) sign streaming reads/writes and + ** paginator requests; request overrides are untouched. *******************************************************************************/ @Test void testSessionSigningAndRequestOverrides(@TempDir Path dir) throws Exception @@ -313,6 +326,7 @@ class ConnectionRuntimeTest *******************************************************************************/ private static String reference(Path dir, String name, String value) throws Exception { + System.setProperty("qqq.fileReference.roots", dir.toString()); Path file = dir.resolve(name); Files.writeString(file, value); return ("${file." + file + "}"); diff --git a/src/test/java/com/kingsrook/qbits/objectstorage/internal/InternalsTest.java b/src/test/java/com/kingsrook/qbits/objectstorage/internal/InternalsTest.java index 76c0367..5b14898 100644 --- a/src/test/java/com/kingsrook/qbits/objectstorage/internal/InternalsTest.java +++ b/src/test/java/com/kingsrook/qbits/objectstorage/internal/InternalsTest.java @@ -83,10 +83,23 @@ class InternalsTest assertThat(SecretRefs.resolve(null)).isNull(); assertThat(SecretRefs.resolveSetting("")).isNull(); + ////////////////////////////////////////////////////////////////////// + // core reads ${file.PATH} only inside its allowed roots (by default // + // /var/run/secrets and /run/secrets); point them at the temp dir. // + ////////////////////////////////////////////////////////////////////// Path file = dir.resolve("key"); Files.writeString(file, " from-file\n"); - assertThat(SecretRefs.resolve("${file." + file + "}")).isEqualTo("from-file"); - assertThat(SecretRefs.resolve("${file." + dir.resolve("missing") + "}")).isNull(); + assertThat(SecretRefs.resolve("${file." + file + "}")).as("outside the allowed roots").isNull(); + System.setProperty("qqq.fileReference.roots", dir.toString()); + try + { + assertThat(SecretRefs.resolve("${file." + file + "}")).isEqualTo("from-file"); + assertThat(SecretRefs.resolve("${file." + dir.resolve("missing") + "}")).isNull(); + } + finally + { + System.clearProperty("qqq.fileReference.roots"); + } } -- 2.54.0 From 4261382567ea39990642bcfefb60cb18db76fc59 Mon Sep 17 00:00:00 2001 From: James Maes Date: Thu, 15 Oct 2026 16:00:18 -0500 Subject: [PATCH 2/2] build: pin QQQ 4.1.0-RC.15 (core S3 session credentials and ${file.PATH}, qqq#1165) Co-Authored-By: Claude Opus 5.5 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 919eb97..2b763b9 100644 --- a/pom.xml +++ b/pom.xml @@ -283,14 +283,14 @@ - + qqq-snapshot true - 4.1.0-RC.8 + 4.1.0-RC.15 -- 2.54.0